def dispatch_request(self, *args, **kwargs): self.user = kwargs.pop('user', None) http_status = { 'GET': 200, 'POST': 201, 'DELETE': 204 }.get(request.method, 200) try: kwargs.update(self.get_parameters()) response = self.do_dispatch(*args, **kwargs) except AppException as exc: LOGGER.exception(__name__) resp = exc.to_dict() resp['ok'] = False http_status = exc.http_code except: LOGGER.exception(__name__) http_status = 423 resp = {'ok': False, 'error': 'System error'} else: ok = True if isinstance(response, HttpResponse): http_status = response.status ok = response.ok response = response.data resp = {"data": response, "ok": ok} if response is None and request.method == 'GET': http_status = 404 resp = jsonify_response(resp) resp.status_code = http_status return resp
def post_authenticated(self, url, json=None, number_of_retries=None): assert self.options and self.options.username and self.options.password authtoken = None expired = None while True: if not authtoken: authtoken = self.get_authtoken(expired) expired = None auth_url = self.apply_token(url, json, authtoken) try: return self.post(auth_url, json=json) except ApplicationError as err: if str(err) == 'Auth token %s is expired' % authtoken: LOGGER.info(err) expired = authtoken authtoken = None else: LOGGER.exception(err) break except UnauthorizedRequestError as err: LOGGER.warning(err, exc_info=True) expired = authtoken authtoken = None except InfrastructureError as err: LOGGER.exception(err) if number_of_retries is None: time.sleep(self.sleep_timeout) elif isinstance(number_of_retries, int) and number_of_retries > 0: number_of_retries -= 1 else: break
def run(self): try: requests = list(self.gen_requests()) for idx, request in enumerate(requests): if self.stopped(): break for tweets in request: if self.stopped(): break self.push_posts(tweets, post_data_format=request.post_data_format, post_data_dump=request.post_data_dump) self.update_status(SUBSCRIPTION_RUNNING) self.historic_loader.load() if self.stopped(): self.update_status(SUBSCRIPTION_STOPPED) # update status, then wait for creator's work done self.historic_loader.creator.join() self.aggregate_state(self.historic_loader, {'finished': True}) else: self.update_status(SUBSCRIPTION_FINISHED) except TwitterApiRateLimitError as e: LOGGER.warning( 'Waiting rate limits reset. Restart is needed in: %s sec.' % e.wait_for) self.update_status(SUBSCRIPTION_WAITING_LIMITS_RESET) raise except Exception: LOGGER.exception('Subscriber exception:', exc_info=True) self.update_status(SUBSCRIPTION_ERROR) raise
def put(self, id_, **filters): try: item = self.manager.get_by_user(self.user, id=id_) except self.model.DoesNotExist: LOGGER.exception(__name__) raise ResourceDoesNotExist("Not found") self.check_duplicate_display_name(id=id_, **filters) item.update(**filters) return item
def send_mail(msg, email_description='email'): # avoiding a circular module dependency from solariat_bottle.app import MAIL_SENDER try: return MAIL_SENDER.send(msg) except Exception, err: LOGGER.exception("Sending %r failed because %r." % (email_description, str(err))) raise EmailSendError("Sending %r failed because %r." % (email_description, str(err)))
def _prepare_post_data(**data): post_data = data.copy() if not isinstance(post_data['channels'][0], str): post_data['channels'] = [str(c.id) for c in post_data['channels']] if 'lang' in post_data and isinstance(post_data['lang'], Language): post_data['lang'] = post_data['lang'].lang try: return json.dumps(post_data) except Exception: from pprint import pformat LOGGER.exception(pformat(post_data)) raise
def sanity_check(CHANNEL_KEY_MAP, keywords_skipwords_pairs): try: details = analyze_channel_keywords(CHANNEL_KEY_MAP, keywords_skipwords_pairs) except: LOGGER.exception( u'analyze_channel_keywords failed with params:\n%s\n%s' % (CHANNEL_KEY_MAP, keywords_skipwords_pairs)) else: for channel, logs in details.iteritems(): for (log_level, msg) in logs: message = u"%s[%s]: %s" % (channel.id, channel.title, msg) getattr(LOGGER, log_level, 'info')(message)
def run(self): for message in self.consumer: LOGGER.debug('kafka message consumed: %s', message.value) try: object = kafka_serializer.deserialize(message.value) if 'task' in object: self.kafka_handler.handle_create_post( object['task'], object['username'], object['kwargs']) if 'event' in object: self.kafka_handler.on_event_handler( object['event'], object['stream_data']) except Exception as e: LOGGER.exception(e)
def get(self, id_=None, **filters): limit = filters.pop('limit') offset = filters.pop('offset') if id_: filters['id'] = id_ try: return self.manager.get_by_user(self.user, **filters) except self.model.DoesNotExist: LOGGER.exception(__name__) raise ResourceDoesNotExist("Not found") else: filters['account_id'] = self.user.account.id res = self.manager.find_by_user( self.user, **filters).limit(limit).skip(offset)[:] return res
def __start_historic_load(self): """ Go through all the required steps in order to create a datasift subscription. 1. Compile a CSDL based on keywords / usernames 2. Hit the /historics/prepare endpoint in the Historics API with the CSDL from step 1 along with start/end dates 3. Hit the /push/create endpoint with the id returned from step 2, this actually creates the subscription 4. Hit the /historics/start endpoint to actually start the subscription """ try: csdl_data = self.compute_csdl() self.prepare_subscription_data(csdl_data['hash']) self.subscribe_historics_data() self.start_subscription() except Exception as e: LOGGER.exception('__start_historic_load') self.subscription.update( status=SUBSCRIPTION_ERROR, status_data_historics={"exception": unicode(e)})
def on_connect(self): """Emitted by Stream when connected successfully""" self.log_event('connect') self.last_keep_alive = now() was_offline = self.db.last_online(self.stream_id) since = None if was_offline: _, _, since, self.last_status_id = was_offline self.set_event(Events.EVENT_ONLINE) MAX_OFFLINE_TIME = 1.5 * 24 * 60 * 60 # 36 hours too_old = not since or (now() - utc(since)).total_seconds() > MAX_OFFLINE_TIME if not was_offline or too_old: return filters = { "start_date": since and utc(since), "end_date": self.last_keep_alive and utc(self.last_keep_alive), "since_id": self.last_status_id } LOGGER.info(u'[%s] fetch offline direct messages with filters %s' % (self.stream_id, filters)) from solariat_bottle.daemons.twitter.historics.timeline_request import DirectMessagesFetcher from solariat_bottle.utils.tweet import TwitterApiWrapper from solariat_bottle.daemons.twitter.parsers import DMTweetParser try: api = TwitterApiWrapper(auth_tuple=self.auth).api fetcher = DirectMessagesFetcher(api, **filters) parse = DMTweetParser() for dm in fetcher.fetch(): self.on_direct_message(parse(dm)) else: LOGGER.info(u'[%s] no offline direct messages' % (self.stream_id, )) except: LOGGER.exception(u"[%s] fetch offline direct messages failed" % self.stream_id)
def generate_csv(cls, iterable_data, context): header = [ 'Channel Type', 'Channel Name', 'Inbound/Outbound', 'Post/Comment', 'Initial Post', 'Date/Time (UTC)', 'Smart Tags', 'Intentions', 'Post Status', 'Topics' ] def get_post_content(p): _get = lambda p, attr: \ p.get(attr) if isinstance(p, dict) else getattr(p, attr) attr = 'url' if context.get('platform') == 'Twitter' else 'content' return unwrap_hidden(_get(p, attr)) def get_initial_post(p): if p['parent']: return get_post_content(p['parent']) return '' value_getters = [ cls.PrefetchedValue(context.get('platform')), cls.PrefetchedValue(context.get('channel_name')), cls.PrefetchedValue(context.get('channel_direction')), get_post_content, get_initial_post, 'created_at', 'smart_tags', 'intentions', 'filter_status', 'topics' ] stream = StringIO() writer = UnicodeWriter(stream) writer.writerow(header) from solariat_bottle.settings import LOGGER for item in iterable_data: try: row = cls.csv_row(value_getters, item) except Exception: LOGGER.exception(u"Cannot generate csv tuple for %s" % item) else: LOGGER.debug(row) writer.writerow(row) stream.seek(0) return stream
def push_posts(self, tweets, post_data_format=None, post_data_dump=None): insert_data = dict(subscription=self.subscription, post_data_format=post_data_format) for tweet in tweets: log_state(self.subscription.channel.id, str(tweet['id']), PostState.ARRIVED_IN_RECOVERY) try: insert_data.update({ "post_data": post_data_dump(tweet), "timestamp": datetime_to_timestamp(parse_datetime(tweet['created_at'])) }) QueuedHistoricData.objects.create(**insert_data) log_state(self.subscription.channel.id, str(tweet['id']), PostState.ADDED_TO_WORKER_QUEUE) except: LOGGER.exception(u'QueuedHistoricData.objects.create: %s' % insert_data) return len(tweets)
def run(self): consumer = self.consumer for job_status in consumer: try: executor = JobExecutor(job_status, self.jobs_manager) except NoRunnableJobFound as ex: LOGGER.error(ex) continue except Exception as ex: LOGGER.error('Error creating JobExecutor: %s', ex, exc_info=True) continue try: executor.execute() except Exception: LOGGER.exception("Exception while executing job") try: consumer.commit() except Exception: LOGGER.exception("Could not commit offset")
def __gen_authtoken(self): post_data = { 'username': self.options.username, 'password': self.options.password, } #url = '/api/v1.2/authtokens' url = '/api/%s/authenticate' % get_var('API_VERSION') while True: try: url_str = '%s%s' % (self.options.url, url) LOGGER.debug('Trying to authenticate using url: %s %s', url_str, self.options) response = self.post(url_str, json=post_data, headers={ 'Content-Type': 'application/json', 'User-Agent': self.user_agent }) LOGGER.debug('Got auth token: %s' % response) except FeedAPIError as err: LOGGER.warning(err, exc_info=True) time.sleep(self.sleep_timeout) else: if not ('token' in response or 'item' in response): LOGGER.error("Bad auth response %s", response) time.sleep(self.sleep_timeout) continue if 'item' in response: response = response['item'] try: authtoken = response['token'].encode('utf-8') except KeyError: LOGGER.exception(response) time.sleep(self.sleep_timeout) else: self.__class__.authtoken = authtoken return authtoken
def __request(self, url, params): """ A generic request for datasift :param params: The request parameters :param url: The url of the request :return: The response of the request """ params.update({ "username": get_var('DATASIFT_USERNAME'), "api_key": get_var('DATASIFT_API_KEY') }) base_url = get_var('DATASIFT_BASE_URL') if not url.startswith(base_url): url = base_url + url try: resp = requests.get(url, urllib.urlencode(params)) LOGGER.debug(u"url: %s\nparams: %s\nresponse: %s" % (url, params, resp)) except: error_msg = "Error requesting '%s' with params '%s'" % (url, params) LOGGER.exception(error_msg) raise Exception(error_msg) else: try: data = resp.json() except: if resp: LOGGER.info(u"Datasift response: %s" % resp) return resp else: if 'error' in data: LOGGER.error("Datasift returned error '%s'\n" "url: %s\nparams: %s" % (data['error'], url, params)) raise Exception(data['error']) return data
def get_conversation_id(self, post): """ Compute a unique 50 bits id for a given inbound post. """ try: root_post_id = post.get_conversation_root_id() # Extract unique id from post to use as root post id. match = self.conversation_id_re.search(root_post_id) if match: conv_id = long(match.group(2)) elif '_' in root_post_id: # Public post, format <page_id>_<unique_post_id> conv_id = long(root_post_id.split('_')[1]) else: LOGGER.warning( "Could not infer conversation id from root post {}. Post: {} Channel: {}" "Defaulting to base one.".format(root_post_id, post, self)) return super(FacebookChannel, self).get_conversation_id(post) except: LOGGER.exception("Exception while getting conv_id from post: {}".format(post)) raise return conv_id
def retrain_function(predictor, models): from copy import deepcopy for model in models: backup_model_data = deepcopy(model.data) predictor.select_model(model) predictor.features_space_size(model) try: manager.produce_state_update({ 'model': model.display_name, 'progress': model.task_data.progress }) predictor.train_models(model=model) manager.produce_state_update({ 'model': model.display_name, 'progress': model.task_data.progress }) predictor.save_model(model) except Exception, ex: # TODO: This needs some user notification somehow! LOGGER.exception(ex) model.data = backup_model_data model.save()
# auth token expiration and other auth errors except api_exc.AuthorizationError, exc: LOGGER.info(exc) return view.format_api_error_response(exc, msg=str(exc), description=exc.description) # # A defined API exception occurred # except api_exc.BaseAPIException, exc: LOGGER.exception(exc) return view.format_api_error_response(exc, msg=str(exc), description=exc.description) # # Some error while communicating with twitter happened # except TwitterCommunicationException, exc: LOGGER.exception(exc) return view.format_twitter_communication_exception(exc) # # Some error while communicating with facebook happened # except FacebookCommunicationException, exc: LOGGER.exception(exc) return view.format_api_error_response(exc, msg=str(exc)) # # Something more sinister went wrong # except AppException, exc: import traceback LOGGER.error(traceback.format_exc()) msg = "{}: {} | {}".format(DEFAULT_ERROR_MSG.format('application'), exc.__class__.__name__, str(exc))
def api_request(func, allow_basic_auth=False): """ Meat and Bones of the REST API. All implemented API methods should be decorated with this, with a couple exceptions. This decorator adds the authenticated user as a parameter passed to the decorated method 1. Parse parameters from the Request 2. Validate Authentication 3. Execute API Method 4. Filter and format results 5. Handle Error's RESTfully """ @wraps(func) def wrapper(*args, **kwargs): #LOGGER.debug("API Request Args: {} | Params: {}".format(args, kwargs)) assert args assert isinstance(args[0], BaseAPIView) view = args[0] # Method decorator try: args = args[1:] except IndexError: args = () params = _get_request_data() params.update(kwargs) # Pass URL variables to the view function start_execution = datetime.utcnow() # Execute API method try: # Assert authentication LOGGER.debug("Started executing API call: %s.%s(%s, %s) " % ( view.__class__.__name__, func.__name__, args, kwargs)) if allow_basic_auth is True: user = _get_user() if user is None: user = authenticate_api_user(params) if user is None: raise api_exc.AuthorizationError("User is not authenticated. Parameters for call: " + str(params)) else: user = authenticate_api_user(params) # Set user in thread local storage set_user(user) resp = func(view, user, *args, **params) elapsed = datetime.utcnow() - start_execution if elapsed.total_seconds() >= 2: log = LOGGER.info else: log = LOGGER.debug log("API call: %s.%s(%s,%s) finished after %s Parameters: %s" % (view.__class__.__name__, func.__name__, str(args)[:500], str(kwargs)[:500], elapsed, str(params)[:500])) # auth token expiration and other auth errors except api_exc.AuthorizationError, exc: LOGGER.info(exc) return view.format_api_error_response(exc, msg=str(exc), description=exc.description) # # A defined API exception occurred # except api_exc.BaseAPIException, exc: LOGGER.exception(exc) return view.format_api_error_response(exc, msg=str(exc), description=exc.description)