def run_tasks(APIS):
config.ECHO_NEST_API_KEY = ECHO_NEST_KEY
csvDatastreamTasks = CSVDatastreamTasks()
celery = Celery('bittrails_tasks', broker='amqp://guest@localhost//')
tasks = {
'twitter': TwitterTasks,
'lastfm': LastfmTasks,
'google': GoogleTasks
}
users = User.get_collection().find()
for user in users:
uids = UID.get_collection().find({'user_id': ObjectId(user['_id'])})
for uid in uids:
if uid['datastream'] in tasks:
task = tasks[uid['datastream']](user,
uid['uid'],
api=APIS[uid['datastream']])
task.run()
# Custom datastreams
custom_streams = CustomTimeSeriesPath.find({
'url': {
'$exists': True
},
'user_id':
ObjectId(user['_id'])
})
for stream in custom_streams:
csvDatastreamTasks.run(stream)
def get_top_level_directory(token, url_prefix):
user = User.find_one({'_id': token['user_id']})
links = {
'self': {
'href': request.base_url,
'title': 'API root'
},
'dimensions.json': {
'href': '%s/dimensions.json' % (url_prefix),
'title': 'dimensions for aggregating leaf data'
}
}
links.update({
datastream: {
'href': '%s/%s.json' % (url_prefix, datastream)
}
for datastream in user['external_tokens'].keys()
})
links.update({
datastream['name']: {
'href': '%s/%s.json' % (url_prefix, datastream['name'])
}
for datastream in TimeSeriesPath.get_collection().find(
{
'parent_path': None,
'user_id': token['user_id'],
'client_id': token['client_id']
})
})
return json.dumps({'_links': links})
def setUp(self):
self.user = User(_id=ObjectId("50e209f8fb5d1b6d96ad37b7"))
self.Mockmodel_aspects = {
'mockDatastream': ['song_mockmodel', 'task_mockmodel']
}
self.Mockmodel2_aspects = {
'mockDatastream': ['song_mockmodel2', 'task_mockmodel2']
}
self.window_size = 3
def setUp(self):
self.user = User(
_id=ObjectId('50e3da15ab0ddcff7dd3c187'),
external_tokens={
"twitter": [
"14847576-NtVpk6iONznNMC7AQmYuI138nf9bualJZG0Jpd5Q0",
"JbmAHGyE2n485Yp7hs6dpTT8eFSn5AFAiiwJ52OHetw"
]
})
self.username = '******'
def run_tasks():
celery = Celery('bittrails_tasks', broker='amqp://guest@localhost//')
users = User.get_collection().find()
for user in users:
uids = UID.get_collection().find({'user_id': ObjectId(user['_id'])})
if uids.count() > 0:
posts = LastPostRetrieved.get_collection().find({
'$or': [{'datastream': row['datastream'],
'uid': row['uid']} for row in uids]
})
available_datastreams = [post['datastream'] for post in posts]
# Cycle through every class that inherits from CorrelationTask
# in tasks.py, instantiate it, and run it.
for task_class in CorrelationTask.__subclasses__():
task = task_class(user, available_datastreams)
task.run()
def run_tasks():
celery = Celery('bittrails_tasks', broker='amqp://guest@localhost//')
users = User.get_collection().find()
for user in users:
uids = UID.get_collection().find({'user_id': ObjectId(user['_id'])})
if uids.count() > 0:
posts = LastPostRetrieved.get_collection().find({
'$or': [{
'datastream': row['datastream'],
'uid': row['uid']
} for row in uids]
})
available_datastreams = [post['datastream'] for post in posts]
# Cycle through every class that inherits from CorrelationTask
# in tasks.py, instantiate it, and run it.
for task_class in CorrelationTask.__subclasses__():
task = task_class(user, available_datastreams)
task.run()
def run_tasks(APIS):
config.ECHO_NEST_API_KEY = ECHO_NEST_KEY
csvDatastreamTasks = CSVDatastreamTasks()
celery = Celery('bittrails_tasks', broker='amqp://guest@localhost//')
tasks = { 'twitter': TwitterTasks, 'lastfm': LastfmTasks, 'google': GoogleTasks }
users = User.get_collection().find()
for user in users:
uids = UID.get_collection().find({'user_id': ObjectId(user['_id'])})
for uid in uids:
if uid['datastream'] in tasks:
task = tasks[uid['datastream']](
user, uid['uid'], api = APIS[uid['datastream']])
task.run()
# Custom datastreams
custom_streams = CustomTimeSeriesPath.find(
{'url': {'$exists': True}, 'user_id': ObjectId(user['_id'])})
for stream in custom_streams:
csvDatastreamTasks.run(stream)
def get_top_level_directory(token, url_prefix):
user = User.find_one({'_id': token['user_id']})
links = {
'self': {'href': request.base_url, 'title': 'API root'},
'dimensions.json': {
'href': '%s/dimensions.json' % (url_prefix),
'title': 'dimensions for aggregating leaf data'
}
}
links.update({datastream: {
'href': '%s/%s.json' % (url_prefix, datastream)
} for datastream in user['external_tokens'].keys()})
links.update({datastream['name']: {
'href': '%s/%s.json' % (url_prefix, datastream['name'])
} for datastream in TimeSeriesPath.get_collection().find(
{'parent_path': None,
'user_id': token['user_id'],
'client_id': token['client_id']
}
)
})
return json.dumps({'_links': links})
def _provide_user(token, *args, **kwargs):
user = User.find_one({'_id': token['user_id']})
return _f(user, *args, **kwargs)
def setUp(self):
self.user = User.find_one(ObjectId("50e3da15ab0ddcff7dd3c187"), as_obj=True)
self.username = "******"
self.counter = TwitterPostCounter(self.user)
def request(self, *args, **kwargs):
response = super(GoogleOAuth, self).request(*args, **kwargs)
# If the token has expired, we need to request a new one and try again.
# We can only do this, though, if a user was passed in.
if ("error" in response.content
and "message" in response.content["error"] and
response.content["error"]["message"] == u'Invalid Credentials'
and kwargs.get('user', False)):
user = kwargs.get('user')
self._logger.info("Refreshing expired token for user %s." %
user['_id'])
# Is there a refresh token for this user?
if ('refresh_tokens' not in user
or self.name not in user['refresh_tokens']):
self._logger.error(
"User %s does not have a refresh token. Abandoning request."
% (user['_id']))
self._logger.info("Abandoned request for user %s was %s" %
(user['_id'], args))
else:
# Request a new access token.
refresh = requests.post(self.refresh_token_url,
data={
'refresh_token':
user['refresh_tokens'][self.name],
'grant_type':
'refresh_token',
'client_id':
self.consumer_key,
'client_secret':
self.consumer_secret
})
refresh = json.loads(refresh.content)
# If the user variable is not a User object, retrieve the
# corresponding object.
if ((not hasattr(user, 'save') or not callable(user.save))
and '_id' in user):
user = User.find_one({'_id': user['_id']}, as_obj=True)
# Was the request for a new token successful?
if 'access_token' in refresh and refresh['access_token']:
# Update the user's Google access token and save it.
user['external_tokens'][
self.name] = refresh['access_token']
user.save()
# Retry the original request wit the new token.
kwargs['user'] = user
response = super(GoogleOAuth,
self).request(*args, **kwargs)
else:
# If the request for a new token was not successful, log
# Google's response so we can debug later.
self._logger.error(
"Tried to refresh token for user %s. Instead got: %s" %
(user['_id'], response.content))
return response
def setUp(self):
self.user = User(_id=ObjectId("50e209f8fb5d1b6d96ad37b7"))
def setUp(self):
self.user = User.find_one(ObjectId("50e3da15ab0ddcff7dd3c187"),
as_obj=True)
self.username = '******'
self.counter = TwitterPostCounter(self.user)
def request(self, *args, **kwargs):
response = super(GoogleOAuth, self).request(*args, **kwargs)
# If the token has expired, we need to request a new one and try again.
# We can only do this, though, if a user was passed in.
if ("error" in response.content
and "message" in response.content["error"]
and response.content["error"]["message"] == u'Invalid Credentials'
and kwargs.get('user', False)):
user = kwargs.get('user')
self._logger.info(
"Refreshing expired token for user %s." % user['_id'])
# Is there a refresh token for this user?
if ('refresh_tokens' not in user
or self.name not in user['refresh_tokens']):
self._logger.error(
"User %s does not have a refresh token. Abandoning request." % (
user['_id'])
)
self._logger.info("Abandoned request for user %s was %s" % (
user['_id'], args))
else:
# Request a new access token.
refresh = requests.post(
self.refresh_token_url,
data = { 'refresh_token': user['refresh_tokens'][self.name],
'grant_type': 'refresh_token',
'client_id': self.consumer_key,
'client_secret': self.consumer_secret }
)
refresh = json.loads(refresh.content)
# If the user variable is not a User object, retrieve the
# corresponding object.
if ((not hasattr(user, 'save') or not callable(user.save))
and '_id' in user):
user = User.find_one({'_id': user['_id']}, as_obj = True)
# Was the request for a new token successful?
if 'access_token' in refresh and refresh['access_token']:
# Update the user's Google access token and save it.
user['external_tokens'][self.name] = refresh['access_token']
user.save()
# Retry the original request wit the new token.
kwargs['user'] = user
response = super(GoogleOAuth, self).request(*args, **kwargs)
else:
# If the request for a new token was not successful, log
# Google's response so we can debug later.
self._logger.error(
"Tried to refresh token for user %s. Instead got: %s" % (
user['_id'], response.content)
)
return response
def _provide_user(token, *args, **kwargs):
user = User.find_one({'_id': token['user_id']})
return _f(user, *args, **kwargs)