def set_limit(self, url, limit, remaining, reset):
""" Set an endpoint's rate limits. The data used for each of the
args should come from Twitter's ``x-rate-limit`` headers.
Args:
url (str):
URL of the endpoint being fetched.
limit (int):
Max number of times a user or app can hit the endpoint
before being rate limited.
remaining (int):
Number of times a user or app can access the endpoint
before being rate limited.
reset (int):
Epoch time at which the rate limit window will reset.
"""
endpoint = self.url_to_resource(url)
resource_family = endpoint.split('/')[1]
try:
family_rates = self.resources.get(resource_family).get(endpoint)
except AttributeError:
self.set_unknown_limit(url, limit, remaining, reset)
family_rates = self.resources.get(resource_family).get(endpoint)
family_rates['limit'] = enf_type('limit', int, limit)
family_rates['remaining'] = enf_type('remaining', int, remaining)
family_rates['reset'] = enf_type('reset', int, reset)
return EndpointRateLimit(family_rates['limit'],
family_rates['remaining'],
family_rates['reset'])
def GetSentDirectMessages(self,
since_id=None,
max_id=None,
count=None,
page=None,
include_entities=True,
return_json=False):
"""
The upstream API for GetSentDirectMessages is broken.
"""
url = '%s/direct_messages/events/list.json' % self.base_url
parameters = {
'include_entities': bool(include_entities),
'max_id': max_id,
'since_id': since_id,
}
if count:
parameters['count'] = enf_type('count', int, count)
if page:
parameters['page'] = enf_type('page', int, page)
resp = self._RequestUrl(url, 'GET', data=parameters)
data = self._ParseAndCheckTwitter(resp.content.decode('utf-8'))
if return_json:
return data
else:
return [DirectMessage.NewFromJsonDict(x) for x in data]
def set_limit(self, url, limit, remaining, reset):
""" If a resource family is unknown, add it to the object's
dictionary. This is to deal with new endpoints being added to
the API, but not necessarily to the information returned by
``/account/rate_limit_status.json`` endpoint.
For example, if Twitter were to add an endpoint
``/puppies/lookup.json``, the RateLimit object would create a resource
family ``puppies`` and add ``/puppies/lookup`` as the endpoint, along
with whatever limit, remaining hits available, and reset time would be
applicable to that resource+endpoint pair.
Args:
url (str):
URL of the endpoint being fetched.
limit (int):
Max number of times a user or app can hit the endpoint
before being rate limited.
remaining (int):
Number of times a user or app can access the endpoint
before being rate limited.
reset (int):
Epoch time at which the rate limit window will reset.
"""
endpoint = self.url_to_resource(url)
resource_family = endpoint.split('/')[1]
new_endpoint = {
endpoint: {
"limit": enf_type('limit', int, limit),
"remaining": enf_type('remaining', int, remaining),
"reset": enf_type('reset', int, reset)
}
}
if not self.resources.get(resource_family, None):
self.resources[resource_family] = {}
self.__dict__['resources'][resource_family].update(new_endpoint)
return self.get_limit(url)
def set_limit(self, url, limit, remaining, reset):
""" If a resource family is unknown, add it to the object's
dictionary. This is to deal with new endpoints being added to
the API, but not necessarily to the information returned by
``/account/rate_limit_status.json`` endpoint.
For example, if Twitter were to add an endpoint
``/puppies/lookup.json``, the RateLimit object would create a resource
family ``puppies`` and add ``/puppies/lookup`` as the endpoint, along
with whatever limit, remaining hits available, and reset time would be
applicable to that resource+endpoint pair.
Args:
url (str):
URL of the endpoint being fetched.
limit (int):
Max number of times a user or app can hit the endpoint
before being rate limited.
remaining (int):
Number of times a user or app can access the endpoint
before being rate limited.
reset (int):
Epoch time at which the rate limit window will reset.
"""
endpoint = self.url_to_resource(url)
resource_family = endpoint.split('/')[1]
new_endpoint = {endpoint: {
"limit": enf_type('limit', int, limit),
"remaining": enf_type('remaining', int, remaining),
"reset": enf_type('reset', int, reset)
}}
if not self.resources.get(resource_family, None):
self.resources[resource_family] = {}
self.__dict__['resources'][resource_family].update(new_endpoint)
return self.get_limit(url)
def getMultiStatus(self,
status_ids,
trim_user=False,
include_my_retweet=True,
include_entities=True,
include_ext_alt_text=True):
"""
Function for retrieving up to 100 statuses
Will also filter non-english tweets
:param status_ids: list of status IDs
:param trim_user:
:param include_my_retweet:
:param include_entities:
:param include_ext_alt_text:
:return: Parsed list of statuses
"""
url = '%s/statuses/lookup.json' % (self.base_url)
stringIds = ','.join(status_ids)
parameters = {
'id':
stringIds,
'trim_user':
enf_type('trim_user', bool, trim_user),
'include_my_retweet':
enf_type('include_my_retweet', bool, include_my_retweet),
'include_entities':
enf_type('include_entities', bool, include_entities),
'include_ext_alt_text':
enf_type('include_ext_alt_text', bool, include_ext_alt_text)
}
try:
resp = self._RequestUrl(url, 'GET', data=parameters)
except TimeoutError:
time.sleep(60)
return self.getMultiStatus(status_ids, trim_user,
include_my_retweet, include_entities,
include_ext_alt_text)
if resp.status_code == 200:
data = self._ParseAndCheckTwitter(resp.content.decode('utf-8'))
if 'x-rate-limit-remaining' in resp.headers._store:
self.limitRemain = resp.headers._store[
'x-rate-limit-remaining']
if 'x-rate-limit-reset' in resp.headers._store:
self.limitReset = resp.headers._store['x-rate-limit-reset']
filteredData = []
for status in data:
if not (status['user']['id'] in filterSet):
continue
filteredText = re.sub(r"http\S+", "",
status['text']).replace('\n',
' ').strip()
try:
if (detect(status['text']) == 'en'):
status['text'] = self.filterASCII(filteredText)
dateFormatted = datetime.strptime(
status['created_at'], twitterDateTimeFormat)
status['created_at'] = dateFormatted.isoformat(sep=' ')
filteredData.append(status)
except LangDetectException:
pass
if (self.limitRemain == 0):
print('Waiting ', self.limitReset - time.time() + 15,
' seconds')
time.sleep(self.limitReset - time.time() + 15)
self.limitRemain == 1
return filteredData
else:
print('Response status code: ', resp.status_code)
print(resp.reason)
time.sleep(60)
return self.getMultiStatus(status_ids, trim_user,
include_my_retweet, include_entities,
include_ext_alt_text)
