def execute(self): self.api.cached_result = False # Build the request URL url = self.api_root + self.path full_url = 'https://' + self.host + url # Query the cache if one is available # and this request uses a GET method. if self.use_cache and self.api.cache and self.method == 'GET': cache_result = self.api.cache.get( '%s?%s' % (url, urlencode(self.session.params))) # if cache result found and not expired, return it if cache_result: # must restore api reference if isinstance(cache_result, list): for result in cache_result: if isinstance(result, Model): result._api = self.api else: if isinstance(cache_result, Model): cache_result._api = self.api self.api.cached_result = True return cache_result # Continue attempting request until successful # or maximum number of retries is reached. retries_performed = 0 while retries_performed < self.retry_count + 1: # handle running out of api calls if self.wait_on_rate_limit: if self._reset_time is not None: if self._remaining_calls is not None: if self._remaining_calls < 1: sleep_time = self._reset_time - int( time.time()) if sleep_time > 0: if self.wait_on_rate_limit_notify: log.warning( "Rate limit reached. Sleeping for: %d" % sleep_time) time.sleep(sleep_time + 5) # sleep for few extra sec # if self.wait_on_rate_limit and self._reset_time is not None and \ # self._remaining_calls is not None and self._remaining_calls < 1: # sleep_time = self._reset_time - int(time.time()) # if sleep_time > 0: # if self.wait_on_rate_limit_notify: # log.warning("Rate limit reached. Sleeping for: %d" % sleep_time) # time.sleep(sleep_time + 5) # sleep for few extra sec # Apply authentication auth = None if self.api.auth: auth = self.api.auth.apply_auth() # Request compression if configured if self.api.compression: self.session.headers['Accept-encoding'] = 'gzip' # Execute request try: resp = self.session.request(self.method, full_url, data=self.post_data, timeout=self.api.timeout, auth=auth, proxies=self.api.proxy) except Exception as e: six.reraise(TweepError, TweepError('Failed to send request: %s' % e), sys.exc_info()[2]) rem_calls = resp.headers.get('x-rate-limit-remaining') if rem_calls is not None: self._remaining_calls = int(rem_calls) elif isinstance(self._remaining_calls, int): self._remaining_calls -= 1 reset_time = resp.headers.get('x-rate-limit-reset') if reset_time is not None: self._reset_time = int(reset_time) if self.wait_on_rate_limit and self._remaining_calls == 0 and ( # if ran out of calls before waiting switching retry last call resp.status_code == 429 or resp.status_code == 420): continue retry_delay = self.retry_delay # Exit request loop if non-retry error code if resp.status_code == 200: break elif (resp.status_code == 429 or resp.status_code == 420) and self.wait_on_rate_limit: if 'retry-after' in resp.headers: retry_delay = float(resp.headers['retry-after']) elif self.retry_errors and resp.status_code not in self.retry_errors: break # Sleep before retrying request again time.sleep(retry_delay) retries_performed += 1 # If an error was returned, throw an exception self.api.last_response = resp if resp.status_code and not 200 <= resp.status_code < 300: try: error_msg, api_error_code = \ self.parser.parse_error(resp.text) except Exception: error_msg = "Twitter error response: status code = %s" % resp.status_code api_error_code = None if is_rate_limit_error_message(error_msg): raise RateLimitError(error_msg, resp) else: raise TweepError(error_msg, resp, api_code=api_error_code) # Parse the response payload result = self.parser.parse(self, resp.text) # Store result into cache if one is available. if self.use_cache and self.api.cache and self.method == 'GET' and result: self.api.cache.store( '%s?%s' % (url, urlencode(self.session.params)), result) return result
def prev(self): if (self.current_page == 1): raise TweepError('Can not page back more, at first page') self.current_page -= 1 return self.method(page=self.current_page, *self.args, **self.kargs)
data = data.strip() # read data and pass into listener if self.listener.on_data(data) is False: self.running = False def _start(self, async): self.running = True if async: Thread(target=self._run).start() else: self._run() def userstream(self, count=None, async=False, secure=True): if self.running: raise TweepError('Stream object already connected!') self.url = '/2/user.json' self.host='userstream.twitter.com' if count: self.url += '&count=%s' % count self._start(async) def firehose(self, count=None, async=False): self.parameters = {'delimited': 'length'} if self.running: raise TweepError('Stream object already connected!') self.url = '/%i/statuses/firehose.json?delimited=length' % STREAM_VERSION if count: self.url += '&count=%s' % count self._start(async)
def prev(self): raise TweepError('This method does not allow backwards pagination')
def execute(self): # Build the request URL url = self.api_root + self.path if len(self.parameters): url = '%s?%s' % (url, urllib.urlencode(self.parameters)) # Query the cache if one is available # and this request uses a GET method. if self.use_cache and self.api.cache and self.method == 'GET': cache_result = self.api.cache.get(url) # if cache result found and not expired, return it if cache_result: # must restore api reference if isinstance(cache_result, list): for result in cache_result: if isinstance(result, Model): result._api = self.api else: if isinstance(cache_result, Model): cache_result._api = self.api return cache_result # Continue attempting request until successful # or maximum number of retries is reached. retries_performed = 0 while retries_performed < self.retry_count + 1: # Open connection if self.api.secure: conn = httplib.HTTPSConnection(self.host, timeout=self.api.timeout) else: conn = httplib.HTTPConnection(self.host, timeout=self.api.timeout) # Apply authentication if self.api.auth: self.api.auth.apply_auth(self.scheme + self.host + url, self.method, self.headers, self.parameters) # Request compression if configured if self.api.compression: self.headers['Accept-encoding'] = 'gzip' # Execute request try: conn.request(self.method, url, headers=self.headers, body=self.post_data) resp = conn.getresponse() except Exception, e: raise TweepError('Failed to send request: %s' % e) # Exit request loop if non-retry error code if self.retry_errors: if resp.status not in self.retry_errors: break else: if resp.status == 200: break # Sleep before retrying request again time.sleep(self.retry_delay) retries_performed += 1
class APIMethod(object): path = config['path'] payload_type = config.get('payload_type', None) payload_list = config.get('payload_list', False) allowed_param = config.get('allowed_param', []) method = config.get('method', 'GET') require_auth = config.get('require_auth', False) search_api = config.get('search_api', False) use_cache = config.get('use_cache', True) def __init__(self, api, args, kargs): # If authentication is required and no credentials # are provided, throw an error. if self.require_auth and not api.auth: raise TweepError('Authentication required!') self.api = api self.post_data = kargs.pop('post_data', None) self.retry_count = kargs.pop('retry_count', api.retry_count) self.retry_delay = kargs.pop('retry_delay', api.retry_delay) self.retry_errors = kargs.pop('retry_errors', api.retry_errors) self.headers = kargs.pop('headers', {}) self.build_parameters(args, kargs) # Pick correct URL root to use if self.search_api: self.api_root = api.search_root else: self.api_root = api.api_root # Perform any path variable substitution self.build_path() if api.secure: self.scheme = 'https://' else: self.scheme = 'http://' if self.search_api: self.host = api.search_host else: self.host = api.host # Manually set Host header to fix an issue in python 2.5 # or older where Host is set including the 443 port. # This causes Twitter to issue 301 redirect. # See Issue https://github.com/tweepy/tweepy/issues/12 self.headers['Host'] = self.host def build_parameters(self, args, kargs): self.parameters = {} for idx, arg in enumerate(args): if arg is None: continue try: self.parameters[ self.allowed_param[idx]] = convert_to_utf8_str(arg) except IndexError: raise TweepError('Too many parameters supplied!') for k, arg in kargs.items(): if arg is None: continue if k in self.parameters: raise TweepError( 'Multiple values for parameter %s supplied!' % k) self.parameters[k] = convert_to_utf8_str(arg) def build_path(self): for variable in re_path_template.findall(self.path): name = variable.strip('{}') if name == 'user' and 'user' not in self.parameters and self.api.auth: # No 'user' parameter provided, fetch it from Auth instead. value = self.api.auth.get_username() else: try: value = urllib.quote(self.parameters[name]) except KeyError: raise TweepError( 'No parameter value found for path variable: %s' % name) del self.parameters[name] self.path = self.path.replace(variable, value) def execute(self): # Build the request URL url = self.api_root + self.path if len(self.parameters): url = '%s?%s' % (url, urllib.urlencode(self.parameters)) # Query the cache if one is available # and this request uses a GET method. if self.use_cache and self.api.cache and self.method == 'GET': cache_result = self.api.cache.get(url) # if cache result found and not expired, return it if cache_result: # must restore api reference if isinstance(cache_result, list): for result in cache_result: if isinstance(result, Model): result._api = self.api else: if isinstance(cache_result, Model): cache_result._api = self.api return cache_result # Continue attempting request until successful # or maximum number of retries is reached. retries_performed = 0 while retries_performed < self.retry_count + 1: # Open connection if self.api.secure: conn = httplib.HTTPSConnection(self.host, timeout=self.api.timeout) else: conn = httplib.HTTPConnection(self.host, timeout=self.api.timeout) # Apply authentication if self.api.auth: self.api.auth.apply_auth(self.scheme + self.host + url, self.method, self.headers, self.parameters) # Request compression if configured if self.api.compression: self.headers['Accept-encoding'] = 'gzip' # Execute request try: conn.request(self.method, url, headers=self.headers, body=self.post_data) resp = conn.getresponse() except Exception, e: raise TweepError('Failed to send request: %s' % e) # Exit request loop if non-retry error code if self.retry_errors: if resp.status not in self.retry_errors: break else: if resp.status == 200: break # Sleep before retrying request again time.sleep(self.retry_delay) retries_performed += 1 # If an error was returned, throw an exception self.api.last_response = resp if resp.status != 200: try: error_msg = self.api.parser.parse_error(resp.read()) except Exception: error_msg = "Twitter error response: status code = %s" % resp.status raise TweepError(error_msg, resp) # Parse the response payload body = resp.read() if resp.getheader('Content-Encoding', '') == 'gzip': try: zipper = gzip.GzipFile(fileobj=StringIO(body)) body = zipper.read() except Exception, e: raise TweepError('Failed to decompress data: %s' % e)
def execute(self): self.api.cached_result = False # Build the request URL url = self.api_root + self.path full_url = 'https://' + self.host + url # Query the cache if one is available # and this request uses a GET method. if self.use_cache and self.api.cache and self.method == 'GET': cache_result = self.api.cache.get( f'{url}?{urlencode(self.session.params)}') # if cache result found and not expired, return it if cache_result: # must restore api reference if isinstance(cache_result, list): for result in cache_result: if isinstance(result, Model): result._api = self.api else: if isinstance(cache_result, Model): cache_result._api = self.api self.api.cached_result = True return cache_result # Continue attempting request until successful # or maximum number of retries is reached. retries_performed = 0 while retries_performed < self.retry_count + 1: if (self.wait_on_rate_limit and self._reset_time is not None and self._remaining_calls is not None and self._remaining_calls < 1): # Handle running out of API calls sleep_time = self._reset_time - int(time.time()) if sleep_time > 0: if self.wait_on_rate_limit_notify: log.warning( f"Rate limit reached. Sleeping for: {sleep_time}" ) time.sleep(sleep_time + 1) # Sleep for extra sec # Apply authentication auth = None if self.api.auth: auth = self.api.auth.apply_auth() # Request compression if configured if self.api.compression: self.session.headers['Accept-encoding'] = 'gzip' # Execute request try: resp = self.session.request(self.method, full_url, data=self.post_data, json=self.json_payload, timeout=self.api.timeout, auth=auth, proxies=self.api.proxy) except Exception as e: raise TweepError( f'Failed to send request: {e}').with_traceback( sys.exc_info()[2]) if resp.status_code in (200, 204): break rem_calls = resp.headers.get('x-rate-limit-remaining') if rem_calls is not None: self._remaining_calls = int(rem_calls) elif self._remaining_calls is not None: self._remaining_calls -= 1 reset_time = resp.headers.get('x-rate-limit-reset') if reset_time is not None: self._reset_time = int(reset_time) retry_delay = self.retry_delay if resp.status_code in (420, 429) and self.wait_on_rate_limit: if self._remaining_calls == 0: # If ran out of calls before waiting switching retry last call continue if 'retry-after' in resp.headers: retry_delay = float(resp.headers['retry-after']) elif self.retry_errors and resp.status_code not in self.retry_errors: # Exit request loop if non-retry error code break # Sleep before retrying request again time.sleep(retry_delay) retries_performed += 1 # If an error was returned, throw an exception self.api.last_response = resp if resp.status_code and not 200 <= resp.status_code < 300: try: error_msg, api_error_code = \ self.parser.parse_error(resp.text) except Exception: error_msg = f"Twitter error response: status code = {resp.status_code}" api_error_code = None if is_rate_limit_error_message(error_msg): raise RateLimitError(error_msg, resp) else: raise TweepError(error_msg, resp, api_code=api_error_code) # Parse the response payload self.return_cursors = (self.return_cursors or 'cursor' in self.session.params or 'next' in self.session.params) result = self.parser.parse(self, resp.text, return_cursors=self.return_cursors) # Store result into cache if one is available. if self.use_cache and self.api.cache and self.method == 'GET' and result: self.api.cache.store(f'{url}?{urlencode(self.session.params)}', result) return result
class Stream(object): host = 'stream.twitter.com' def __init__(self, rest_endpoint, duration, listener, timeout=5.0, retry_count=None, retry_time=10.0, snooze_time=5.0, buffer_size=1500, headers=None): # self.auth = BasicAuthHandler(username, password) self.running = False self.timeout = timeout self.retry_count = retry_count self.retry_time = retry_time self.snooze_time = snooze_time self.buffer_size = buffer_size self.listener = listener self.api = API() self.headers = headers or {} self.body = None def _run(self): # setup # self.auth.apply_auth(None, None, self.headers, None) # enter loop error_counter = 0 conn = None while self.running: if self.retry_count and error_counter > self.retry_count: # quit if error count greater than retry count break try: # as of 10/1/11 only https is supported on streaming API conn = httplib.HTTPSConnection(self.host) conn.connect() conn.sock.settimeout(self.timeout) conn.request('GET', self.url, self.body, headers=self.headers) resp = conn.getresponse() if resp.status != 200: if self.listener.on_error(resp.status) is False: break error_counter += 1 sleep(self.retry_time) else: error_counter = 0 self._read_loop(resp) except timeout: if self.listener.on_timeout() == False: break if self.running is False: break conn.close() sleep(self.snooze_time) except Exception: # any other exception is fatal, so kill loop break # cleanup self.running = False if conn: conn.close() def _read_loop(self, resp): data = '' while self.running: if resp.isclosed(): break # read length length = '' while True: c = resp.read(1) if c == '\n': break length += c length = length.strip() if length.isdigit(): length = int(length) else: continue # read data and pass into listener data = resp.read(length) if self.listener.on_data(data) is False: self.running = False def _start(self, async): self.running = True if async: Thread(target=self._run).start() else: self._run() def firehose(self, count=None, async=False): if self.running: raise TweepError('Stream object already connected!') self.url = '/%i/statuses/firehose.json?delimited=length' % STREAM_VERSION if count: self.url += '&count=%s' % count self._start(async)
""" Internal use only """ @staticmethod def _pack_image(filename, max_size): """Pack image from file into multipart-formdata post body""" # image must be less than 700kb in size try: if os.path.getsize(filename) > (max_size * 1024): raise TweepError('File is too big, must be less than 700kb.') except os.error, e: raise TweepError('Unable to access file') # image must be gif, jpeg, or png file_type = mimetypes.guess_type(filename) if file_type is None: raise TweepError('Could not determine file type') file_type = file_type[0] if file_type not in ['image/gif', 'image/jpeg', 'image/png']: raise TweepError('Invalid file type for image: %s' % file_type) # build the mulitpart-formdata body fp = open(filename, 'rb') BOUNDARY = 'Tw3ePy' body = [] body.append('--' + BOUNDARY) body.append( 'Content-Disposition: form-data; name="image"; filename="%s"' % filename) body.append('Content-Type: %s' % file_type) body.append('') body.append(fp.read())
def execute(self): # Build the request URL url = self.api_root + self.path if len(self.parameters): url = '%s?%s' % (url, urllib.urlencode(self.parameters)) # Query the cache if one is available # and this request uses a GET method. if self.use_cache and self.api.cache and self.method == 'GET': cache_result = self.api.cache.get(url) # if cache result found and not expired, return it if cache_result: # must restore api reference if isinstance(cache_result, list): for result in cache_result: if isinstance(result, Model): result._api = self.api else: if isinstance(cache_result, Model): cache_result._api = self.api return cache_result # Continue attempting request until successful # or maximum number of retries is reached. retries_performed = 0 connection_args = {'host': self.host} # Python 2.6+ httplib has support for timeout via the API if self.timeout is not None and sys.hexversion >= 0x02060000: connection_args['timeout'] = self.timeout while retries_performed < self.retry_count + 1: # Open connection if self.api.secure: conn = httplib.HTTPSConnection(**connection_args) else: conn = httplib.HTTPConnection(**connection_args) # Apply authentication if self.api.auth: self.api.auth.apply_auth(self.scheme + self.host + url, self.method, self.headers, self.parameters) # Execute request try: conn.request(self.method, url, headers=self.headers, body=self.post_data) # Python <= 2.5 httplib doesn't support timeout via the api # so we have to set it manually on the socket if self.timeout is not None and sys.hexversion < 0x02060000: conn.sock.settimeout(self.timeout) resp = conn.getresponse() except socket.timeout, e: # Retry on timeout if we've been asked to if self.retry_timeout and retries_performed < self.retry_count: time.sleep(self.retry_delay) retries_performed += 1 continue raise TweepError('Failed to send request: %s' % e) except Exception, e: raise TweepError('Failed to send request: %s' % e)
def execute(self): self.api.cached_result = False # Build the request URL url = self.api_root + self.path if len(self.parameters): url = '%s?%s' % (url, urllib.urlencode(self.parameters)) # Query the cache if one is available # and this request uses a GET method. if self.use_cache and self.api.cache and self.method == 'GET': cache_result = self.api.cache.get(url) # if cache result found and not expired, return it if cache_result: # must restore api reference if isinstance(cache_result, list): for result in cache_result: if isinstance(result, Model): result._api = self.api else: if isinstance(cache_result, Model): cache_result._api = self.api self.api.cached_result = True return cache_result # Continue attempting request until successful # or maximum number of retries is reached. retries_performed = 0 while retries_performed < self.retry_count + 1: # Open connection if self.api.secure: conn = httplib.HTTPSConnection(self.host, timeout=self.api.timeout) else: conn = httplib.HTTPConnection(self.host, timeout=self.api.timeout) # Apply authentication if self.api.auth: self.api.auth.apply_auth(self.scheme + self.host + url, self.method, self.headers, self.parameters) # Request compression if configured if self.api.compression: self.headers['Accept-encoding'] = 'gzip' # Execute request try: conn.request(self.method, url, headers=self.headers, body=self.post_data) resp = conn.getresponse() except Exception as e: raise TweepError('Failed to send request: %s' % e) # Exit request loop if non-retry error code if self.retry_errors: if resp.status not in self.retry_errors: break else: if resp.status == 200: break # Sleep before retrying request again time.sleep(self.retry_delay) retries_performed += 1 # If an error was returned, throw an exception self.api.last_response = resp if resp.status and not 200 <= resp.status < 300: try: error_msg = self.api.parser.parse_error(resp.read()) except Exception: error_msg = "Twitter error response: status code = %s" % resp.status raise TweepError(error_msg, resp) # Parse the response payload body = resp.read() if resp.getheader('Content-Encoding', '') == 'gzip': try: zipper = gzip.GzipFile(fileobj=StringIO(body)) body = zipper.read() except Exception as e: raise TweepError('Failed to decompress data: %s' % e) result = self.api.parser.parse(self, body) conn.close() # Store result into cache if one is available. if self.use_cache and self.api.cache and self.method == 'GET' and result: self.api.cache.store(url, result) return result
def parse(self, method, payload): try: json = self.json_lib.loads(payload) except Exception, e: raise TweepError('Failed to parse JSON payload: %s' % e)
def get_status(self, status_id): status = next((s for s in self.statuses if s.id == status_id), None) if status: return status else: raise TweepError('No status with requested id')
def on_closed(self, resp): """ Called when the response has been closed by Twitter """ pass def userstream(self, stall_warnings=False, _with=None, replies=None, track=None, locations=None, async=False, encoding='utf8'): self.session.params = {'delimited': 'length'} if self.running: raise TweepError('Stream object already connected!') self.url = '/%s/user.json' % STREAM_VERSION self.host = 'userstream.twitter.com' if stall_warnings: self.session.params['stall_warnings'] = stall_warnings if _with: self.session.params['with'] = _with if replies: self.session.params['replies'] = replies if locations and len(locations) > 0: if len(locations) % 4 != 0: raise TweepError("Wrong number of locations points, " "it has to be a multiple of 4") self.session.params['locations'] = ','.join( ['%.2f' % l for l in locations]) if track:
def retweet(self, is_async=False): self.session.params = {'delimited': 'length'} if self.running: raise TweepError('Stream object already connected!') self.url = '/%s/statuses/retweet.json' % STREAM_VERSION self._start(is_async)
# Setup the Collector seed = parser.parse_args( ).seed # Swap seeds with another Twitter User ID if you like if seed == 1670174994: print("No seed given. Using default seed " + str(seed) + ".") else: print("Downloading and saving friends' details of user " + str(seed) + ".") con = Connection() collector = Collector(con, seed) # Get the friends and details of the specified seed try: friends = collector.get_friend_list() except TweepError as e: if "'code': 34" in e.reason: raise TweepError( "The seed you have given is not a valid Twitter user ID") friends_details = collector.get_details(friends) # Check for the relevant directory if not os.path.isdir(os.path.join("tests", "tweet_jsons")): os.mkdir(os.path.join("tests", "tweet_jsons")) # Write details in json files ct = 1 for friend_details in friends_details: with open( os.path.join("tests", "tweet_jsons", "user_" + str(ct) + ".json"), "w") as f: json.dump(friend_details._json, f) ct += 1
def _chunk_media(command, filename, max_size, form_field="media", chunk_size=4096, f=None, media_id=None, segment_index=0, is_direct_message=False): fp = None if command == 'init': file_size = getfilesize(filename, f) if file_size > (max_size * 1024): raise TweepError('File is too big, must be less than %skb.' % max_size) if f is None: # build the multipart-formdata body fp = open(filename, 'rb') else: fp = f elif command != 'finalize': if f is not None: fp = f else: raise TweepError('File input for APPEND is mandatory.') # video must be mp4 file_type, _ = mimetypes.guess_type(filename) if file_type is None: raise TweepError('Could not determine file type') if file_type not in CHUNKED_MIMETYPES: raise TweepError('Invalid file type for video: %s' % file_type) BOUNDARY = b'Tw3ePy' body = list() if command == 'init': query = { 'command': 'INIT', 'media_type': file_type, 'total_bytes': file_size, 'media_category': API._get_media_category(is_direct_message, file_type) } body.append(urlencode(query).encode('utf-8')) headers = { 'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8' } elif command == 'append': if media_id is None: raise TweepError('Media ID is required for APPEND command.') body.append(b'--' + BOUNDARY) body.append( 'Content-Disposition: form-data; name="command"'.encode( 'utf-8')) body.append(b'') body.append(b'APPEND') body.append(b'--' + BOUNDARY) body.append( 'Content-Disposition: form-data; name="media_id"'.encode( 'utf-8')) body.append(b'') body.append(str(media_id).encode('utf-8')) body.append(b'--' + BOUNDARY) body.append( 'Content-Disposition: form-data; name="segment_index"'.encode( 'utf-8')) body.append(b'') body.append(str(segment_index).encode('utf-8')) body.append(b'--' + BOUNDARY) body.append( 'Content-Disposition: form-data; name="{0}"; filename="{1}"'. format(form_field, os.path.basename(filename)).encode('utf-8')) body.append('Content-Type: {0}'.format(file_type).encode('utf-8')) body.append(b'') body.append(fp.read(chunk_size)) body.append(b'--' + BOUNDARY + b'--') headers = {'Content-Type': 'multipart/form-data; boundary=Tw3ePy'} elif command == 'finalize': if media_id is None: raise TweepError('Media ID is required for FINALIZE command.') body.append( urlencode({ 'command': 'FINALIZE', 'media_id': media_id }).encode('utf-8')) headers = { 'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8' } body = b'\r\n'.join(body) # build headers headers['Content-Length'] = str(len(body)) return headers, body, fp