def filter(self, follow=None, track=None, is_async=False, locations=None, stall_warnings=False, languages=None, encoding='utf8', filter_level=None): self.body = {} self.session.headers[ 'Content-type'] = "application/x-www-form-urlencoded" if self.running: raise TweepError('Stream object already connected!') self.url = '/%s/statuses/filter.json' % STREAM_VERSION if follow: self.body['follow'] = u','.join(follow).encode(encoding) if track: self.body['track'] = u','.join(track).encode(encoding) if locations and len(locations) > 0: if len(locations) % 4 != 0: raise TweepError("Wrong number of locations points, " "it has to be a multiple of 4") self.body['locations'] = u','.join(['%.4f' % l for l in locations]) if stall_warnings: self.body['stall_warnings'] = stall_warnings if languages: self.body['language'] = u','.join(map(str, languages)) if filter_level: self.body['filter_level'] = filter_level.encode(encoding) self.session.params = {'delimited': 'length'} self._start(is_async)
def userstream(self, stall_warnings=False, _with=None, replies=None, track=None, locations=None, is_async=False, encoding='utf8'): self.session.params = {'delimited': 'length'} if self.running: raise TweepError('Stream object already connected!') self.url = '/%s/user.json' % STREAM_VERSION self.host = 'userstream.twitter.com' if stall_warnings: self.session.params['stall_warnings'] = stall_warnings if _with: self.session.params['with'] = _with if replies: self.session.params['replies'] = replies if locations and len(locations) > 0: if len(locations) % 4 != 0: raise TweepError("Wrong number of locations points, " "it has to be a multiple of 4") self.session.params['locations'] = ','.join( ['%.2f' % l for l in locations]) if track: self.session.params['track'] = u','.join(track).encode(encoding) self._start(is_async)
def prev(self): if self.current_page is None: raise TweepError('Can not go back more, at first page') if self.page_index == 0: # At the beginning of the current page, move to next... self.current_page = self.page_iterator.prev() self.page_index = len(self.current_page) if self.page_index == 0: raise TweepError('No more items') self.page_index -= 1 self.num_tweets -= 1 return self.current_page[self.page_index]
def __init__(self, method, *args, **kwargs): if hasattr(method, 'pagination_mode'): if method.pagination_mode == 'cursor': self.iterator = CursorIterator(method, *args, **kwargs) elif method.pagination_mode == 'id': self.iterator = IdIterator(method, *args, **kwargs) elif method.pagination_mode == 'page': self.iterator = PageIterator(method, *args, **kwargs) else: raise TweepError('Invalid pagination mode.') else: raise TweepError('This method does not perform pagination')
def _get_request_token(self, access_type=None): try: url = self._get_oauth_url('request_token') if access_type: url += '?x_auth_access_type=%s' % access_type return self.oauth.fetch_request_token(url) except Exception as e: raise TweepError(e)
def firehose(self, count=None, is_async=False): self.session.params = {'delimited': 'length'} if self.running: raise TweepError('Stream object already connected!') self.url = '/%s/statuses/firehose.json' % STREAM_VERSION if count: self.url += '&count=%s' % count self._start(is_async)
def prev(self): if self.prev_cursor == 0: raise TweepError('Can not page back more, at first page') data, self.next_cursor, self.prev_cursor = self.method(cursor=self.prev_cursor, *self.args, **self.kwargs) self.num_tweets -= 1 return data
def _read_loop(self, resp): charset = resp.headers.get('content-type', default='') enc_search = re.search(r'charset=(?P<enc>\S*)', charset) if enc_search is not None: encoding = enc_search.group('enc') else: encoding = 'utf-8' buf = ReadBuffer(resp.raw, self.chunk_size, encoding=encoding) while self.running and not resp.raw.closed: length = 0 while not resp.raw.closed: line = buf.read_line() stripped_line = line.strip( ) if line else line # line is sometimes None so we need to check here if not stripped_line: self.listener.keep_alive( ) # keep-alive new lines are expected elif stripped_line.isdigit(): length = int(stripped_line) break else: raise TweepError( 'Expecting length, unexpected value found') next_status_obj = buf.read_len(length) if self.running and next_status_obj: self._data(next_status_obj) # # Note: keep-alive newlines might be inserted before each length value. # # read until we get a digit... # c = b'\n' # for c in resp.iter_content(decode_unicode=True): # if c == b'\n': # continue # break # # delimited_string = c # # # read rest of delimiter length.. # d = b'' # for d in resp.iter_content(decode_unicode=True): # if d != b'\n': # delimited_string += d # continue # break # # # read the next twitter status object # if delimited_string.decode('utf-8').strip().isdigit(): # status_id = int(delimited_string) # next_status_obj = resp.raw.read(status_id) # if self.running: # self._data(next_status_obj.decode('utf-8')) if resp.raw.closed: self.on_closed(resp)
def get_username(self): if self.username is None: api = API(self) user = api.verify_credentials() if user: self.username = user.screen_name else: raise TweepError('Unable to get username,' ' invalid oauth token!') return self.username
def sample(self, is_async=False, languages=None, stall_warnings=False): self.session.params = {'delimited': 'length'} if self.running: raise TweepError('Stream object already connected!') self.url = '/%s/statuses/sample.json' % STREAM_VERSION if languages: self.session.params['language'] = ','.join(map(str, languages)) if stall_warnings: self.session.params['stall_warnings'] = 'true' self._start(is_async)
def build_parameters(self, args, kwargs): self.session.params = {} for idx, arg in enumerate(args): if arg is None: continue try: self.session.params[ self.allowed_param[idx]] = convert_to_utf8_str(arg) except IndexError: raise TweepError('Too many parameters supplied!') for k, arg in kwargs.items(): if arg is None: continue if k in self.session.params: raise TweepError( 'Multiple values for parameter %s supplied!' % k) self.session.params[k] = convert_to_utf8_str(arg) log.debug("PARAMS: %r", self.session.params)
def parse(self, method, payload): try: json = json_lib.loads(payload) except Exception as e: raise TweepError('Failed to parse JSON payload: %s' % e) needs_cursors = 'cursor' in method.session.params if needs_cursors and isinstance(json, dict) \ and 'previous_cursor' in json \ and 'next_cursor' in json: cursors = json['previous_cursor'], json['next_cursor'] return json, cursors else: return json
def __init__(self, consumer_key, consumer_secret): self.consumer_key = consumer_key self.consumer_secret = consumer_secret self._bearer_token = '' resp = requests.post(self._get_oauth_url('token'), auth=(self.consumer_key, self.consumer_secret), data={'grant_type': 'client_credentials'}) data = resp.json() if data.get('token_type') != 'bearer': raise TweepError('Expected token_type to equal "bearer", ' 'but got %s instead' % data.get('token_type')) self._bearer_token = data['access_token']
def switch_auth(self): """ Switch authentication from the current one which is depleted """ if max(self._remaining_calls) > 0: self.api.auth_idx = max(enumerate(self._remaining_calls), key=lambda x: x[1])[0] else: if not self.wait_on_rate_limit: raise TweepError('API calls depleted.') next_idx = min(enumerate(self._reset_times), key=lambda x: x[1])[0] sleep_time = self._reset_times[next_idx] - int(time.time()) if sleep_time > 0: time.sleep(sleep_time + 5) self.api.auth_idx = next_idx self.build_path()
def get_authorization_url(self, signin_with_twitter=False, access_type=None): """Get the authorization URL to redirect the user""" try: if signin_with_twitter: url = self._get_oauth_url('authenticate') if access_type: log.warning(WARNING_MESSAGE) else: url = self._get_oauth_url('authorize') self.request_token = self._get_request_token( access_type=access_type) return self.oauth.authorization_url(url) except Exception as e: raise TweepError(e)
def build_path(self): for variable in re_path_template.findall(self.path): name = variable.strip('{}') if name == 'user' and 'user' not in self.session.params and self.api.auth: # No 'user' parameter provided, fetch it from Auth instead. value = self.api.auth.get_username() else: try: value = quote(self.session.params[name]) except KeyError: raise TweepError( 'No parameter value found for path variable: %s' % name) del self.session.params[name] self.path = self.path.replace(variable, value)
def sitestream(self, follow, stall_warnings=False, with_='user', replies=False, is_async=False): self.body = {} if self.running: raise TweepError('Stream object already connected!') self.url = '/%s/site.json' % STREAM_VERSION self.body['follow'] = u','.join(map(six.text_type, follow)) self.body['delimited'] = 'length' if stall_warnings: self.body['stall_warnings'] = stall_warnings if with_: self.body['with'] = with_ if replies: self.body['replies'] = replies self._start(is_async)
def get_access_token(self, verifier=None): """ After user has authorized the request token, get access token with user supplied verifier. """ try: url = self._get_oauth_url('access_token') self.oauth = OAuth1Session( self.consumer_key, client_secret=self.consumer_secret, resource_owner_key=self.request_token['oauth_token'], resource_owner_secret=self.request_token['oauth_token_secret'], verifier=verifier, callback_uri=self.callback) resp = self.oauth.fetch_access_token(url) self.access_token = resp['oauth_token'] self.access_token_secret = resp['oauth_token_secret'] return self.access_token, self.access_token_secret except Exception as e: raise TweepError(e)
def parse(self, method, payload): try: if method.payload_type is None: return model = getattr(self.model_factory, method.payload_type) except AttributeError: raise TweepError('No model for this payload type: ' '%s' % method.payload_type) json = JSONParser.parse(self, method, payload) if isinstance(json, tuple): json, cursors = json else: cursors = None if method.payload_list: result = model.parse_list(method.api, json) else: result = model.parse(method.api, json) if cursors: return result, cursors else: return result
def get_xauth_access_token(self, username, password): """ Get an access token from an username and password combination. In order to get this working you need to create an app at http://twitter.com/apps, after that send a mail to [email protected] and request activation of xAuth for it. """ try: url = self._get_oauth_url('access_token') oauth = OAuth1(self.consumer_key, client_secret=self.consumer_secret) r = requests.post(url=url, auth=oauth, headers={ 'x_auth_mode': 'client_auth', 'x_auth_username': username, 'x_auth_password': password }) credentials = parse_qs(r.content) return credentials.get('oauth_token')[0], credentials.get( 'oauth_token_secret')[0] except Exception as e: raise TweepError(e)
def execute(self): self.api.cached_result = False # Build the request URL url = self.api_root + self.path full_url = 'https://' + self.host + url # Query the cache if one is available # and this request uses a GET method. if self.use_cache and self.api.cache and self.method == 'GET': cache_result = self.api.cache.get( '%s?%s' % (url, urlencode(self.session.params))) # if cache result found and not expired, return it if cache_result: # must restore api reference if isinstance(cache_result, list): for result in cache_result: if isinstance(result, Model): result._api = self.api else: if isinstance(cache_result, Model): cache_result._api = self.api self.api.cached_result = True return cache_result # Continue attempting request until successful # or maximum number of retries is reached. retries_performed = 0 while retries_performed < self.retry_count + 1: # handle running out of api calls if self.wait_on_rate_limit: if self._reset_times is not None: if self._remaining_calls is not None: if len( self._remaining_calls ) < 1: #<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< Addition sleep_time = self._reset_times[next_idx] - int( time.time() ) #<<<<<<<<<<<<<<<<<<<<<<< [next_id] addition if sleep_time > 0: if self.wait_on_rate_limit_notify: log.warning( "Rate limit reached. Sleeping for: %d" % sleep_time) time.sleep(sleep_time + 5) # sleep for few extra sec # if self.wait_on_rate_limit and self._reset_time is not None and \ # self._remaining_calls is not None and self._remaining_calls < 1: # sleep_time = self._reset_time - int(time.time()) # if sleep_time > 0: # if self.wait_on_rate_limit_notify: # log.warning("Rate limit reached. Sleeping for: %d" % sleep_time) # time.sleep(sleep_time + 5) # sleep for few extra sec # Apply authentication auth = None if self.api.auth: auth = self.api.auth.apply_auth() # Request compression if configured if self.api.compression: self.session.headers['Accept-encoding'] = 'gzip' # Execute request try: resp = self.session.request(self.method, full_url, data=self.post_data, json=self.json_payload, timeout=self.api.timeout, auth=auth, proxies=self.api.proxy) except Exception as e: six.reraise(TweepError, TweepError('Failed to send request: %s' % e), sys.exc_info()[2]) # rem_calls = resp.headers.get('x-rate-limit-remaining') # if rem_calls is not None: # self._remaining_calls = int(rem_calls) # elif isinstance(self._remaining_calls, int): # self._remaining_calls -= 1 ############################################################################ if self.monitor_rate_limit: rem_calls = resp.headers.get('x-rate-limit-remaining') rem_calls = int( rem_calls ) if rem_calls is not None else self._remaining_calls[ self.api.auth_idx] - 1 self._remaining_calls[self.api.auth_idx] = rem_calls reset_time = resp.headers.get('x-rate-limit-reset') if reset_time is not None: self._reset_times[self.api.auth_idx] = int(reset_time) if rem_calls == 0: self.switch_auth() if self.wait_on_rate_limit and self._remaining_calls == 0 and ( # if ran out of calls before waiting switching retry last call resp.status_code == 429 or resp.status_code == 420): continue #################################################################################################### retry_delay = self.retry_delay # Exit request loop if non-retry error code if resp.status_code in (200, 204): break elif (resp.status_code == 429 or resp.status_code == 420) and self.wait_on_rate_limit: if 'retry-after' in resp.headers: retry_delay = float(resp.headers['retry-after']) elif self.retry_errors and resp.status_code not in self.retry_errors: break # Sleep before retrying request again time.sleep(retry_delay) retries_performed += 1 # If an error was returned, throw an exception self.api.last_response = resp if resp.status_code and not 200 <= resp.status_code < 300: try: error_msg, api_error_code = \ self.parser.parse_error(resp.text) except Exception: error_msg = "Twitter error response: status code = %s" % resp.status_code api_error_code = None if is_rate_limit_error_message(error_msg): raise RateLimitError(error_msg, resp) else: raise TweepError(error_msg, resp, api_code=api_error_code) # Parse the response payload result = self.parser.parse(self, resp.text) # Store result into cache if one is available. if self.use_cache and self.api.cache and self.method == 'GET' and result: self.api.cache.store( '%s?%s' % (url, urlencode(self.session.params)), result) return result
def __init__(self, *args, **kwargs): api = self.api # If authentication is required and no credentials # are provided, throw an error. if self.require_auth and not api.auth: raise TweepError('Authentication required!') self.post_data = kwargs.pop('post_data', None) self.json_payload = kwargs.pop('json_payload', None) self.retry_count = kwargs.pop('retry_count', api.retry_count) self.retry_delay = kwargs.pop('retry_delay', api.retry_delay) self.retry_errors = kwargs.pop('retry_errors', api.retry_errors) self.monitor_rate_limit = kwargs.pop( 'monitor_rate_limit', #<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< Addition api.monitor_rate_limit) self.wait_on_rate_limit = kwargs.pop('wait_on_rate_limit', api.wait_on_rate_limit) self.wait_on_rate_limit_notify = kwargs.pop( 'wait_on_rate_limit_notify', api.wait_on_rate_limit_notify) ###################################################################################################### self._remaining_calls = None self._reset_times = None if self.monitor_rate_limit: self._path_category = path_category_pattern.findall( self.path)[0] if self._path_category == 'application': self.monitor_rate_limit = False self._path_without_ext = path_without_ext_pattern.findall( self.path)[0] self._remaining_calls = [sys.maxsize] * len(self.api.auths) self._reset_times = [sys.maxsize] * len(self.api.auths) # Monitoring rate limits when not using multi-auths & monitor_rate_limit param. ######################################################################################################### self.parser = kwargs.pop('parser', api.parser) self.session.headers = kwargs.pop('headers', {}) self.build_parameters(args, kwargs) # Pick correct URL root to use if self.search_api: self.api_root = api.search_root elif self.upload_api: self.api_root = api.upload_root else: self.api_root = api.api_root # Perform any path variable substitution self.build_path() if self.search_api: self.host = api.search_host elif self.upload_api: self.host = api.upload_host else: self.host = api.host # Manually set Host header to fix an issue in python 2.5 # or older where Host is set including the 443 port. # This causes Twitter to issue 301 redirect. # See Issue https://github.com/tweepy/tweepy/issues/12 self.session.headers['Host'] = self.host
def retweet(self, is_async=False): self.session.params = {'delimited': 'length'} if self.running: raise TweepError('Stream object already connected!') self.url = '/%s/statuses/retweet.json' % STREAM_VERSION self._start(is_async)
def prev(self): if self.current_page == 1: raise TweepError('Can not page back more, at first page') self.current_page -= 1 return self.method(page=self.current_page, *self.args, **self.kwargs)