def refresh_token(self, suid): auth_info = yield get_user_auth(suid, CACHED_PREFIX_KEY) if auth_info is None: _logger.warn("There is no login user info.") raise FlamesError(USER_NOT_FOUND, user_id=suid) url = self._OAUTH_ACCESS_TOKEN_URL httpc = httpclient.AsyncHTTPClient() post_args = { "grant_type": "refresh_token", "client_id": _app_settings['client_id'], "client_secret": _app_settings['consumer_secret'], "refresh_token": auth_info['refresh_token'] } headers = {'Content-Type': 'application/x-www-form-urlencoded'} try : response = yield httpc.fetch(url, method="POST", body=urllib_parse.urlencode(post_args), headers=headers) result = json2dict(response.body) update_user_auth(suid, result, CACHED_PREFIX_KEY) except HTTPError as e: _logger.error("Request %s %s" % (url, e)) raise e _logger.info("Request %s done." % (url)) raise Return(result)
def _wrapper(*args, **kwargs): cached_key = _get_cached_key(func, args, kwargs, prefix, key) data = get_redis_client().get(cached_key) if data: _logger.debug("get cached result by key '%s'" % cached_key) try: result = json2dict(data) except Exception: _logger.warn("Fail to parse cached result to json: %s. Call function %s directly." % (data, func.__name__), exc_info=True) result = yield func(*args, **kwargs) raise Return(result) else: result = yield func(*args, **kwargs) if result is not None: if isinstance(expires_in, basestring): cached_expires_in = eval("_config.%s" % expires_in) else: cached_expires_in = expires_in _logger.debug("cache result by key '%s' and expired period %d" % (cached_key, cached_expires_in)) get_redis_client().setex(cached_key, cached_expires_in, dict2json(result)) raise Return(result)
def api_request(self, suid, url, post_args=None, resend=False, **args): auth_info = yield get_user_auth(suid, CACHED_PREFIX_KEY) if auth_info is None: _logger.warn("There is no login user info.") raise FlamesError(USER_NOT_FOUND, user_id=suid) if args: args.update({'access_token': auth_info['access_token']}) else : args = {'access_token': auth_info['access_token']} url += "?" + urllib_parse.urlencode(args) httpc = self.get_auth_http_client() try: if post_args is not None: response = yield httpc.fetch(url, method="POST", body=urllib_parse.urlencode(post_args)) else: response = yield httpc.fetch(url) result = json2dict(response.body) except HTTPError as e: if e.code == 401 and resend == False: auth_info = yield self.refresh_token(suid) # Update the auth token, and send request again. yield update_user_auth(suid, auth_info, CACHED_PREFIX_KEY); result = yield self.api_request(suid, url, post_args, resend=True, **args) raise Return(result) raise e _logger.info("Request %s done." % (url)) raise Return(result)
def get_authenticated_user(self, code): """Handles the login for the user, returning a user object. """ url = self._OAUTH_ACCESS_TOKEN_URL client_id = _app_settings['client_id'] client_secret = _app_settings['consumer_secret'] http = self.get_auth_http_client() headers={'Content-type': 'application/x-www-form-urlencoded'} scope = " ".join(_app_settings['scope'].split()) body = urllib_parse.urlencode({ "grant_type": "authorization_code", "client_id": client_id, "client_secret": client_secret, "code": code, "redirect_uri": _app_settings['redirect_uri'], "scope": scope }) try : response = yield http.fetch(url, method="POST", headers=headers, body=body) except Exception as e: _logger.error("Request %s %s" % (url, e)) raise e _logger.debug(response.body) #TODO: check the response status code. raise Return(json2dict(response.body))
def refresh_token(self, suid, extr_headers={}): auth_info = yield get_user_auth(suid, self._SERVC_PREFIX) if auth_info is None: self._logger.warn("There is no login user info.") raise FlamesError(USER_NOT_FOUND, user_id=suid) url = self._OAUTH_ACCESS_TOKEN_URL httpc = httpclient.AsyncHTTPClient() post_args = { "grant_type": "refresh_token", "client_id": self._APP_SETTINGS['client_id'], "client_secret": self._APP_SETTINGS['consumer_secret'], "refresh_token": auth_info['refresh_token'] } headers = {'Content-Type': 'application/x-www-form-urlencoded'} for hdr_name, hdr_value in extr_headers.iteritems(): headers[hdr_name] = hdr_value try : response = yield httpc.fetch(url, method="POST", body=urllib_parse.urlencode(post_args), headers=headers) result = json2dict(response.body) update_user_auth(suid, result, self._SERVC_PREFIX) except HTTPError as e: self._logger.error("Request %s %s" % (url, e)) raise e self._logger.info("Request %s done." % (url)) raise Return(result)
def get_cache(prefix, user_id, field): cached_key = _get_key(prefix, user_id, field) data = get_redis_client().get(cached_key) result = None if data: _logger.debug("get cached result by key '%s'" % cached_key) try: result = json2dict(data) except Exception as e: _logger.warn("Fail to parse cached result to json: %s." % (data), exc_info=True) raise e raise Return(result)
def api_request(self, suid, url, hdr_auth=True, post_args=None, resend=False, json_schema=None, auth_info=None, **args): if auth_info is None: auth_info = yield get_user_auth(suid, self._SERVC_PREFIX) if auth_info is None: self._logger.warn("There is no login user info.") raise FlamesError(USER_NOT_FOUND, user_id=suid) hdr = {} all_args = {} if hdr_auth: oauth = "Bearer " + auth_info['access_token'] hdr = {'Authorization': oauth} else : all_args = {'access_token': auth_info['access_token']} if args: all_args.update(args) if all_args: url += "?" + urllib_parse.urlencode(all_args) self._logger.debug(url) httpc = self.get_auth_http_client() result = {} try: if post_args is not None: response = yield httpc.fetch(url, method="POST", body=urllib_parse.urlencode(post_args), headers=hdr) else: response = yield httpc.fetch(url,headers=hdr) result = json2dict(response.body, schema=json_schema, validated=True) self._logger.info("Request %s done." % (url)) except HTTPError as e: if e.code == 401 and resend == False: auth_info = yield self.refresh_token(suid) # Update the auth token, and send request again. yield update_user_auth(suid, auth_info, self._SERVC_PREFIX); result = yield self.api_request(suid, url, post_args, resend=True, **args) result={'code': e.code, "message": e.message} finally: raise Return(result)
def run(self): """ Read messages from queue and set task to worker thread. TODO: Remove the kafka related stuff from this function. """ # Brokers should be in the uri path # path.strip returns type 'unicode' and pykafka expects a string, so # converting unicode to str brokers = Consumer._Brokers.strip('/').encode('ascii', 'ignore') self.kafka_client = PyKafkaClient(hosts=brokers) kafka_topic = self.kafka_client.topics[self.topic] self.kafka_consumer = kafka_topic.get_balanced_consumer( consumer_group=Consumer._Consumer_group.encode('ascii', 'ignore'), **Consumer._Args) produce_topic = self.kafka_client.topics[("%s-data" % self.topic)] self.kafka_producer = produce_topic.get_producer() # Create the worker object. handler_desc = _handler_dict[self.topic] worker_class = get_class(handler_desc[0]) worker = worker_class() # Enable timer to consume the worker tasks. worker.enable_task_routine(task_callback=self.task_callback) while True: msg_list = kafka_queue_read(self.kafka_consumer) if len(msg_list) == 0: # No data, try again later. sleep(1) continue _logger.debug(len(msg_list)) for msg in msg_list: task_info = json2dict(msg.value) worker.add_task(task_info) # Check the worker thread working status; and commit the offset. # Wait for the worker consuming all the tasks. worker.wait_all_tasks_done() # Commit the kafka reading offset. self.kafka_consumer.commit_offsets()
def api_request(self, user_id, path, post_args=None, **kwargs): if path.startswith('http:') or path.startswith('https:'): # Raw urls are useful for e.g. search which doesn't follow the # usual pattern: http://search.twitter.com/search.json url = path else: url = self._API_URL + path # Add the OAuth resource request signature if we have credentials access_token = yield get_user_auth(user_id, CACHED_PREFIX_WITHINGS) if kwargs: kwargs.update({'userid': access_token['userid']}) if access_token: all_args = {} all_args.update(kwargs) all_args.update(post_args or {}) method = "POST" if post_args is not None else "GET" oauth = self._oauth_request_parameters(url, access_token, all_args, method=method) kwargs.update(oauth) if kwargs: url += "?" + urllib_parse.urlencode(kwargs) http = self.get_auth_http_client() try: if post_args is not None: response = yield http.fetch(url, method="POST", body=urllib_parse.urlencode(post_args)) else: response = yield http.fetch(url) result = json2dict(response.body) except HTTPError as e: if response.error: raise AuthError( "Error response %s fetching %s" % (response.error, response.request.url)) else : raise e _logger.info("Request %s done." % (url)) raise Return(result)