def build_data(self, args, kwargs): if len(args) == 1 and isinstance(args[0], Model): # explode model.to_state() of model instance into kwargs, clear args kwargs.update(args[0].to_state(self.action)) args = list() else: for k, v in kwargs.items(): if isinstance(v, Model): kwargs[k] = v.to_state(self.action) # filter kwargs for allowed_param and not in query_only_param kwargs = dict([(k, v) for k, v in kwargs.items() if k in self.allowed_param]) if self.use_json: self.json_data = dict([(k, v) for k, v in kwargs.items() if k not in self.query_only_param]) self.session.params = OrderedDict() for idx, arg in enumerate(args): if arg is None: continue try: self.session.params[ self.allowed_param[idx]] = convert_to_utf8_str(arg) except IndexError: raise SenseTError('Too many parameters supplied!') for k, arg in kwargs.items(): if arg is None: continue if k in self.session.params: raise SenseTError( 'Multiple values for parameter %s supplied!' % k) self.session.params[k] = convert_to_utf8_str(arg)
def build_query_params(self, kwargs): for param in self.query_only_param: try: self.query_params[param] = kwargs.get(param) except KeyError: raise SenseTError( "A required API.bind() method query_param was missing from the kwargs." )
def __getstate_create__(self, pickled): """ :param pickled: dict of object kay, values :return: API weirdly requires a single organisationid on creation/update but returns a list """ if not self.organisations: raise SenseTError("Platform creation requires an organisationid.") pickled["organisationid"] = self.organisations[0].id return pickled
def __getstate_create__(self, pickled): """ :param pickled: dict of object kay, values :return: """ if not self.type: raise SenseTError("Stream creation requires an type.") if self.type is not None: pickled["type"] = self._type.value return pickled
def parse(self, method, payload): try: json = self.json_lib.loads(payload) except Exception as e: raise SenseTError('Failed to parse JSON payload: %s' % e) needs_cursors = 'cursor' in method.session.params if needs_cursors and isinstance(json, dict): if 'previous_cursor' in json: if 'next_cursor' in json: cursors = json['previous_cursor'], json['next_cursor'] return json, cursors else: return json
def __init__(self, args, kwargs): api = self.api self.api_root = api.api_root self.session.verify = api.verify # If authentication is required and no credentials # are provided, throw an error. if self.require_auth and not api.auth: raise SenseTError('Authentication required!') self.post_data = kwargs.pop('post_data', None) self.json_data = kwargs.pop('json_data', {}) self.use_json = kwargs.pop('use_json', True) self.query_params = kwargs.pop('query_params', {}) self.retry_count = kwargs.pop('retry_count', api.retry_count) self.retry_delay = kwargs.pop('retry_delay', api.retry_delay) self.retry_errors = kwargs.pop('retry_errors', api.retry_errors) self.wait_on_rate_limit = kwargs.pop('wait_on_rate_limit', api.wait_on_rate_limit) self.wait_on_rate_limit_notify = kwargs.pop( 'wait_on_rate_limit_notify', api.wait_on_rate_limit_notify) self.parser = kwargs.pop('parser', api.parser) self.session.headers = kwargs.pop('headers', {}) self.build_data(args, kwargs) self.build_query_params(kwargs) # Perform any path variable substitution self.build_path() self.host = api.host # TODO: test and remove below. # Manually set Host header to fix an issue in python 2.5 # or older where Host is set including the 443 port. # This causes Twitter to issue 301 redirect. # See Issue https://github.com/tweepy/tweepy/issues/12 # self.session.headers['Host'] = self.host # Monitoring rate limits self._remaining_calls = None self._reset_time = None
def build_path(self): for variable in re_path_template.findall(self.path): name = variable.strip('{}') if name == 'user' and 'user' not in self.session.params and self.api.auth: # No 'user' parameter provided, fetch it from Auth instead. value = self.api.auth.get_username() else: try: value = quote(self.session.params[name]) except KeyError: raise SenseTError( 'No parameter value found for path variable: %s' % name) del self.session.params[name] self.path = self.path.replace(variable, value) log.info("PATH: %r", self.path)
def parse(self, method, payload): # Validate media type. media_type = method.query_params.get('media', None) if media_type != 'csv': raise SenseTError('PandasObservationParser requires CSV media type (media type "{}" is not supported).'.format(media_type)) # Skip header information. stream_ids = method.query_params['streamid'].split(',') # NOTE: this WILL break if stream IDs contain commas (need to properly parse as CSV). column_headers = frozenset(['timestamp'] + stream_ids) lines = payload.splitlines() for i, row in enumerate(lines): if set(s.strip() for s in row.split(',')) == column_headers: break # Parse CSV payload. df = self.pandas.read_csv(StringIO('\n'.join(lines[i:])), parse_dates=True, index_col='timestamp') # SensorCloud returns columns in random (alphabetic?) order - reorder to # match the order the stream IDs were originally given in. if len(stream_ids) > 1: df = df[stream_ids] return df
def parse(self, method, payload): try: if method.payload_type is None: return model = getattr(self.model_factory, method.payload_type) except AttributeError: raise SenseTError('No model for this payload type: ' '%s' % method.payload_type) json = JSONParser.parse(self, method, payload) if isinstance(json, tuple): json, cursors = json else: cursors = None if method.payload_list: result = model.parse_list(method.api, json) else: result = model.parse(method.api, json) if cursors: return result, cursors else: return result
def execute(self): self.api.cached_result = False # Build the request URL url = self.api_root + self.path full_url = 'https://' + self.host + url # Query the cache if one is available # and this request uses a GET method. if self.use_cache and self.api.cache and self.method == 'GET': cache_result = self.api.cache.get(url) # if cache result found and not expired, return it if cache_result: # must restore api reference if isinstance(cache_result, list): for result in cache_result: if isinstance(result, Model): result._api = self.api else: if isinstance(cache_result, Model): cache_result._api = self.api self.api.cached_result = True return cache_result # Continue attempting request until successful # or maximum number of retries is reached. retries_performed = 0 while retries_performed < self.retry_count + 1: # handle running out of api calls if self.wait_on_rate_limit: if self._reset_time is not None: if self._remaining_calls is not None: if self._remaining_calls < 1: sleep_time = self._reset_time - int( time.time()) if sleep_time > 0: if self.wait_on_rate_limit_notify: print( "Rate limit reached. Sleeping for:", sleep_time) time.sleep(sleep_time + 5) # sleep for few extra sec # if self.wait_on_rate_limit and self._reset_time is not None and \ # self._remaining_calls is not None and self._remaining_calls < 1: # sleep_time = self._reset_time - int(time.time()) # if sleep_time > 0: # if self.wait_on_rate_limit_notify: # print("Rate limit reached. Sleeping for: " + str(sleep_time)) # time.sleep(sleep_time + 5) # sleep for few extra sec # Apply authentication if self.api.auth: auth = self.api.auth.apply_auth() # Request compression if configured if self.api.compression: self.session.headers['Accept-encoding'] = 'gzip' # Execute request try: if self.use_json: self.session.params = OrderedDict() resp = self.session.request(self.method, full_url, json=self.json_data, params=self.query_params, timeout=self.api.timeout, auth=auth, proxies=self.api.proxy) else: resp = self.session.request(self.method, full_url, data=self.post_data, params=self.query_params, timeout=self.api.timeout, auth=auth, proxies=self.api.proxy) except Exception as e: raise SenseTError('Failed to send request: %s' % e) rem_calls = resp.headers.get('x-rate-limit-remaining') if rem_calls is not None: self._remaining_calls = int(rem_calls) elif isinstance(self._remaining_calls, int): self._remaining_calls -= 1 reset_time = resp.headers.get('x-rate-limit-reset') if reset_time is not None: self._reset_time = int(reset_time) if self.wait_on_rate_limit and self._remaining_calls == 0 and ( # if ran out of calls before waiting switching retry last call resp.status_code == 429 or resp.status_code == 420): continue retry_delay = self.retry_delay # Exit request loop if non-retry error code if resp.status_code == 200: break elif (resp.status_code == 429 or resp.status_code == 420) and self.wait_on_rate_limit: if 'retry-after' in resp.headers: retry_delay = float(resp.headers['retry-after']) elif self.retry_errors and resp.status_code not in self.retry_errors: break retries_performed += 1 # Sleep before retrying request again if retries_performed < self.retry_count + 1: # Only sleep when not on the last retry time.sleep(retry_delay) # If an error was returned, throw an exception self.api.last_response = resp if resp.status_code and not 200 <= resp.status_code < 300: try: error_msg, api_error_code = \ self.parser.parse_error(resp.text) except Exception as ex: error_msg = "SenseT error response: status code = %s" % resp.status_code api_error_code = None if is_rate_limit_error_message(error_msg): raise RateLimitError(error_msg, resp) else: raise SenseTError(error_msg, resp, api_code=api_error_code) # Parse the response payload result = self.parser.parse(self, resp.text) # Store result into cache if one is available. if self.use_cache and self.api.cache and self.method == 'GET' and result: self.api.cache.store(url, result) return result