def dispatch_request(self, *args, **kwargs):
self.user = kwargs.pop('user', None)
http_status = {
'GET': 200,
'POST': 201,
'DELETE': 204
}.get(request.method, 200)
try:
kwargs.update(self.get_parameters())
response = self.do_dispatch(*args, **kwargs)
except AppException as exc:
LOGGER.exception(__name__)
resp = exc.to_dict()
resp['ok'] = False
http_status = exc.http_code
except:
LOGGER.exception(__name__)
http_status = 423
resp = {'ok': False, 'error': 'System error'}
else:
ok = True
if isinstance(response, HttpResponse):
http_status = response.status
ok = response.ok
response = response.data
resp = {"data": response, "ok": ok}
if response is None and request.method == 'GET':
http_status = 404
resp = jsonify_response(resp)
resp.status_code = http_status
return resp
def post_authenticated(self, url, json=None, number_of_retries=None):
assert self.options and self.options.username and self.options.password
authtoken = None
expired = None
while True:
if not authtoken:
authtoken = self.get_authtoken(expired)
expired = None
auth_url = self.apply_token(url, json, authtoken)
try:
return self.post(auth_url, json=json)
except ApplicationError as err:
if str(err) == 'Auth token %s is expired' % authtoken:
LOGGER.info(err)
expired = authtoken
authtoken = None
else:
LOGGER.exception(err)
break
except UnauthorizedRequestError as err:
LOGGER.warning(err, exc_info=True)
expired = authtoken
authtoken = None
except InfrastructureError as err:
LOGGER.exception(err)
if number_of_retries is None:
time.sleep(self.sleep_timeout)
elif isinstance(number_of_retries,
int) and number_of_retries > 0:
number_of_retries -= 1
else:
break
def run(self):
try:
requests = list(self.gen_requests())
for idx, request in enumerate(requests):
if self.stopped():
break
for tweets in request:
if self.stopped():
break
self.push_posts(tweets,
post_data_format=request.post_data_format,
post_data_dump=request.post_data_dump)
self.update_status(SUBSCRIPTION_RUNNING)
self.historic_loader.load()
if self.stopped():
self.update_status(SUBSCRIPTION_STOPPED)
# update status, then wait for creator's work done
self.historic_loader.creator.join()
self.aggregate_state(self.historic_loader, {'finished': True})
else:
self.update_status(SUBSCRIPTION_FINISHED)
except TwitterApiRateLimitError as e:
LOGGER.warning(
'Waiting rate limits reset. Restart is needed in: %s sec.' %
e.wait_for)
self.update_status(SUBSCRIPTION_WAITING_LIMITS_RESET)
raise
except Exception:
LOGGER.exception('Subscriber exception:', exc_info=True)
self.update_status(SUBSCRIPTION_ERROR)
raise
def put(self, id_, **filters):
try:
item = self.manager.get_by_user(self.user, id=id_)
except self.model.DoesNotExist:
LOGGER.exception(__name__)
raise ResourceDoesNotExist("Not found")
self.check_duplicate_display_name(id=id_, **filters)
item.update(**filters)
return item
def send_mail(msg, email_description='email'):
# avoiding a circular module dependency
from solariat_bottle.app import MAIL_SENDER
try:
return MAIL_SENDER.send(msg)
except Exception, err:
LOGGER.exception("Sending %r failed because %r." %
(email_description, str(err)))
raise EmailSendError("Sending %r failed because %r." %
(email_description, str(err)))
def _prepare_post_data(**data):
post_data = data.copy()
if not isinstance(post_data['channels'][0], str):
post_data['channels'] = [str(c.id) for c in post_data['channels']]
if 'lang' in post_data and isinstance(post_data['lang'], Language):
post_data['lang'] = post_data['lang'].lang
try:
return json.dumps(post_data)
except Exception:
from pprint import pformat
LOGGER.exception(pformat(post_data))
raise
def sanity_check(CHANNEL_KEY_MAP, keywords_skipwords_pairs):
try:
details = analyze_channel_keywords(CHANNEL_KEY_MAP,
keywords_skipwords_pairs)
except:
LOGGER.exception(
u'analyze_channel_keywords failed with params:\n%s\n%s' %
(CHANNEL_KEY_MAP, keywords_skipwords_pairs))
else:
for channel, logs in details.iteritems():
for (log_level, msg) in logs:
message = u"%s[%s]: %s" % (channel.id, channel.title, msg)
getattr(LOGGER, log_level, 'info')(message)
def run(self):
for message in self.consumer:
LOGGER.debug('kafka message consumed: %s', message.value)
try:
object = kafka_serializer.deserialize(message.value)
if 'task' in object:
self.kafka_handler.handle_create_post(
object['task'], object['username'], object['kwargs'])
if 'event' in object:
self.kafka_handler.on_event_handler(
object['event'], object['stream_data'])
except Exception as e:
LOGGER.exception(e)
def get(self, id_=None, **filters):
limit = filters.pop('limit')
offset = filters.pop('offset')
if id_:
filters['id'] = id_
try:
return self.manager.get_by_user(self.user, **filters)
except self.model.DoesNotExist:
LOGGER.exception(__name__)
raise ResourceDoesNotExist("Not found")
else:
filters['account_id'] = self.user.account.id
res = self.manager.find_by_user(
self.user, **filters).limit(limit).skip(offset)[:]
return res
def __start_historic_load(self):
"""
Go through all the required steps in order to create a datasift subscription.
1. Compile a CSDL based on keywords / usernames
2. Hit the /historics/prepare endpoint in the Historics API with the CSDL from step 1 along with start/end dates
3. Hit the /push/create endpoint with the id returned from step 2, this actually creates the subscription
4. Hit the /historics/start endpoint to actually start the subscription
"""
try:
csdl_data = self.compute_csdl()
self.prepare_subscription_data(csdl_data['hash'])
self.subscribe_historics_data()
self.start_subscription()
except Exception as e:
LOGGER.exception('__start_historic_load')
self.subscription.update(
status=SUBSCRIPTION_ERROR,
status_data_historics={"exception": unicode(e)})
def on_connect(self):
"""Emitted by Stream when connected successfully"""
self.log_event('connect')
self.last_keep_alive = now()
was_offline = self.db.last_online(self.stream_id)
since = None
if was_offline:
_, _, since, self.last_status_id = was_offline
self.set_event(Events.EVENT_ONLINE)
MAX_OFFLINE_TIME = 1.5 * 24 * 60 * 60 # 36 hours
too_old = not since or (now() -
utc(since)).total_seconds() > MAX_OFFLINE_TIME
if not was_offline or too_old:
return
filters = {
"start_date": since and utc(since),
"end_date": self.last_keep_alive and utc(self.last_keep_alive),
"since_id": self.last_status_id
}
LOGGER.info(u'[%s] fetch offline direct messages with filters %s' %
(self.stream_id, filters))
from solariat_bottle.daemons.twitter.historics.timeline_request import DirectMessagesFetcher
from solariat_bottle.utils.tweet import TwitterApiWrapper
from solariat_bottle.daemons.twitter.parsers import DMTweetParser
try:
api = TwitterApiWrapper(auth_tuple=self.auth).api
fetcher = DirectMessagesFetcher(api, **filters)
parse = DMTweetParser()
for dm in fetcher.fetch():
self.on_direct_message(parse(dm))
else:
LOGGER.info(u'[%s] no offline direct messages' %
(self.stream_id, ))
except:
LOGGER.exception(u"[%s] fetch offline direct messages failed" %
self.stream_id)
def generate_csv(cls, iterable_data, context):
header = [
'Channel Type', 'Channel Name', 'Inbound/Outbound', 'Post/Comment',
'Initial Post', 'Date/Time (UTC)', 'Smart Tags', 'Intentions',
'Post Status', 'Topics'
]
def get_post_content(p):
_get = lambda p, attr: \
p.get(attr) if isinstance(p, dict) else getattr(p, attr)
attr = 'url' if context.get('platform') == 'Twitter' else 'content'
return unwrap_hidden(_get(p, attr))
def get_initial_post(p):
if p['parent']:
return get_post_content(p['parent'])
return ''
value_getters = [
cls.PrefetchedValue(context.get('platform')),
cls.PrefetchedValue(context.get('channel_name')),
cls.PrefetchedValue(context.get('channel_direction')),
get_post_content, get_initial_post, 'created_at', 'smart_tags',
'intentions', 'filter_status', 'topics'
]
stream = StringIO()
writer = UnicodeWriter(stream)
writer.writerow(header)
from solariat_bottle.settings import LOGGER
for item in iterable_data:
try:
row = cls.csv_row(value_getters, item)
except Exception:
LOGGER.exception(u"Cannot generate csv tuple for %s" % item)
else:
LOGGER.debug(row)
writer.writerow(row)
stream.seek(0)
return stream
def push_posts(self, tweets, post_data_format=None, post_data_dump=None):
insert_data = dict(subscription=self.subscription,
post_data_format=post_data_format)
for tweet in tweets:
log_state(self.subscription.channel.id, str(tweet['id']),
PostState.ARRIVED_IN_RECOVERY)
try:
insert_data.update({
"post_data":
post_data_dump(tweet),
"timestamp":
datetime_to_timestamp(parse_datetime(tweet['created_at']))
})
QueuedHistoricData.objects.create(**insert_data)
log_state(self.subscription.channel.id, str(tweet['id']),
PostState.ADDED_TO_WORKER_QUEUE)
except:
LOGGER.exception(u'QueuedHistoricData.objects.create: %s' %
insert_data)
return len(tweets)
def run(self):
consumer = self.consumer
for job_status in consumer:
try:
executor = JobExecutor(job_status, self.jobs_manager)
except NoRunnableJobFound as ex:
LOGGER.error(ex)
continue
except Exception as ex:
LOGGER.error('Error creating JobExecutor: %s',
ex,
exc_info=True)
continue
try:
executor.execute()
except Exception:
LOGGER.exception("Exception while executing job")
try:
consumer.commit()
except Exception:
LOGGER.exception("Could not commit offset")
def __gen_authtoken(self):
post_data = {
'username': self.options.username,
'password': self.options.password,
}
#url = '/api/v1.2/authtokens'
url = '/api/%s/authenticate' % get_var('API_VERSION')
while True:
try:
url_str = '%s%s' % (self.options.url, url)
LOGGER.debug('Trying to authenticate using url: %s %s',
url_str, self.options)
response = self.post(url_str,
json=post_data,
headers={
'Content-Type': 'application/json',
'User-Agent': self.user_agent
})
LOGGER.debug('Got auth token: %s' % response)
except FeedAPIError as err:
LOGGER.warning(err, exc_info=True)
time.sleep(self.sleep_timeout)
else:
if not ('token' in response or 'item' in response):
LOGGER.error("Bad auth response %s", response)
time.sleep(self.sleep_timeout)
continue
if 'item' in response:
response = response['item']
try:
authtoken = response['token'].encode('utf-8')
except KeyError:
LOGGER.exception(response)
time.sleep(self.sleep_timeout)
else:
self.__class__.authtoken = authtoken
return authtoken
def __request(self, url, params):
"""
A generic request for datasift
:param params: The request parameters
:param url: The url of the request
:return: The response of the request
"""
params.update({
"username": get_var('DATASIFT_USERNAME'),
"api_key": get_var('DATASIFT_API_KEY')
})
base_url = get_var('DATASIFT_BASE_URL')
if not url.startswith(base_url):
url = base_url + url
try:
resp = requests.get(url, urllib.urlencode(params))
LOGGER.debug(u"url: %s\nparams: %s\nresponse: %s" %
(url, params, resp))
except:
error_msg = "Error requesting '%s' with params '%s'" % (url,
params)
LOGGER.exception(error_msg)
raise Exception(error_msg)
else:
try:
data = resp.json()
except:
if resp:
LOGGER.info(u"Datasift response: %s" % resp)
return resp
else:
if 'error' in data:
LOGGER.error("Datasift returned error '%s'\n"
"url: %s\nparams: %s" %
(data['error'], url, params))
raise Exception(data['error'])
return data
def get_conversation_id(self, post):
"""
Compute a unique 50 bits id for a given inbound post.
"""
try:
root_post_id = post.get_conversation_root_id()
# Extract unique id from post to use as root post id.
match = self.conversation_id_re.search(root_post_id)
if match:
conv_id = long(match.group(2))
elif '_' in root_post_id:
# Public post, format <page_id>_<unique_post_id>
conv_id = long(root_post_id.split('_')[1])
else:
LOGGER.warning(
"Could not infer conversation id from root post {}. Post: {} Channel: {}"
"Defaulting to base one.".format(root_post_id, post, self))
return super(FacebookChannel, self).get_conversation_id(post)
except:
LOGGER.exception("Exception while getting conv_id from post: {}".format(post))
raise
return conv_id
def retrain_function(predictor, models):
from copy import deepcopy
for model in models:
backup_model_data = deepcopy(model.data)
predictor.select_model(model)
predictor.features_space_size(model)
try:
manager.produce_state_update({
'model': model.display_name,
'progress': model.task_data.progress
})
predictor.train_models(model=model)
manager.produce_state_update({
'model': model.display_name,
'progress': model.task_data.progress
})
predictor.save_model(model)
except Exception, ex:
# TODO: This needs some user notification somehow!
LOGGER.exception(ex)
model.data = backup_model_data
model.save()
# auth token expiration and other auth errors
except api_exc.AuthorizationError, exc:
LOGGER.info(exc)
return view.format_api_error_response(exc, msg=str(exc), description=exc.description)
#
# A defined API exception occurred
#
except api_exc.BaseAPIException, exc:
LOGGER.exception(exc)
return view.format_api_error_response(exc, msg=str(exc), description=exc.description)
#
# Some error while communicating with twitter happened
#
except TwitterCommunicationException, exc:
LOGGER.exception(exc)
return view.format_twitter_communication_exception(exc)
#
# Some error while communicating with facebook happened
#
except FacebookCommunicationException, exc:
LOGGER.exception(exc)
return view.format_api_error_response(exc, msg=str(exc))
#
# Something more sinister went wrong
#
except AppException, exc:
import traceback
LOGGER.error(traceback.format_exc())
msg = "{}: {} | {}".format(DEFAULT_ERROR_MSG.format('application'), exc.__class__.__name__, str(exc))
def api_request(func, allow_basic_auth=False):
"""
Meat and Bones of the REST API. All implemented API methods should be decorated with this,
with a couple exceptions. This decorator adds the authenticated user as a parameter
passed to the decorated method
1. Parse parameters from the Request
2. Validate Authentication
3. Execute API Method
4. Filter and format results
5. Handle Error's RESTfully
"""
@wraps(func)
def wrapper(*args, **kwargs):
#LOGGER.debug("API Request Args: {} | Params: {}".format(args, kwargs))
assert args
assert isinstance(args[0], BaseAPIView)
view = args[0] # Method decorator
try:
args = args[1:]
except IndexError:
args = ()
params = _get_request_data()
params.update(kwargs) # Pass URL variables to the view function
start_execution = datetime.utcnow()
# Execute API method
try:
# Assert authentication
LOGGER.debug("Started executing API call: %s.%s(%s, %s) " % (
view.__class__.__name__, func.__name__, args, kwargs))
if allow_basic_auth is True:
user = _get_user()
if user is None:
user = authenticate_api_user(params)
if user is None:
raise api_exc.AuthorizationError("User is not authenticated. Parameters for call: " + str(params))
else:
user = authenticate_api_user(params)
# Set user in thread local storage
set_user(user)
resp = func(view, user, *args, **params)
elapsed = datetime.utcnow() - start_execution
if elapsed.total_seconds() >= 2:
log = LOGGER.info
else:
log = LOGGER.debug
log("API call: %s.%s(%s,%s) finished after %s Parameters: %s" % (view.__class__.__name__, func.__name__,
str(args)[:500], str(kwargs)[:500],
elapsed, str(params)[:500]))
# auth token expiration and other auth errors
except api_exc.AuthorizationError, exc:
LOGGER.info(exc)
return view.format_api_error_response(exc, msg=str(exc), description=exc.description)
#
# A defined API exception occurred
#
except api_exc.BaseAPIException, exc:
LOGGER.exception(exc)
return view.format_api_error_response(exc, msg=str(exc), description=exc.description)
