def add_daterange(self, filters, timeformat):
current = datetime.now()
three_montsh_ago = current - timedelta(days=90)
current = current + timedelta(days=20)
filters['from'] = three_montsh_ago.strftime(timeformat)
filters['to'] = current.strftime(timeformat)
filters['timerange'] = [
datetime_to_timestamp(three_montsh_ago),
datetime_to_timestamp(current)
]
def get_csv_input_file(self, size=10):
now_ts = datetime_to_timestamp(utc(now()))
csv_file = tempfile.TemporaryFile('w+')
writer = csv.writer(csv_file, delimiter=',')
writer.writerow(self.TEST_ITEM_COLUMNS)
for _ in xrange(size):
writer.writerow(self._gen_item_values(now_ts))
csv_file.flush()
csv_file.seek(0)
return csv_file
def fake_send_dm_rate_limit_headers_zero_remains(api_proxy,
id_or_username,
text=None):
resp = requests.Response()
setattr(api_proxy.base_api, 'last_response', resp)
resp.headers['X-Rate-Limit-Limit'] = '15'
resp.headers['X-Rate-Limit-Remaining'] = '0'
resp.headers['X-Rate-Limit-Reset'] = str(
datetime_to_timestamp(now()) + QUARTER_HOUR_SEC)
raise tweepy.TweepError('Test rate limits (from headers) hit.', resp)
def make_id(cls, kw):
components = []
for id_part in id_pattern:
if id_part['type'] == 'long':
value = long(kw[id_part['name']])
elif id_part['type'] == 'timestamp':
value = datetime_to_timestamp(datetime.strptime(kw[id_part['name']], "%d/%m/%Y %I:%M:%S %p"))
else:
value = str(kw[id_pattern['name']])
components.append((value, id_part['bits']))
return pack_components(*components)
def prepare_subscription_data(self, CSDL):
""" Prepare the subsciption through the appropriate datasift API call.
Requires at minimum a start date, end date and csdl along with the credentials.
:param CSDL: The compiled CSDL required to start a datasift subscription
:returns The subscription ID from datasift
"""
from solariat.utils.timeslot import datetime_to_timestamp
params = {
"start": datetime_to_timestamp(self.from_date),
"end": datetime_to_timestamp(self.to_date),
"hash": CSDL,
"name": self.datasift_subscription_name,
"sources": 'twitter'
}
data = self.__request("/v1/historics/prepare", params)
self.subscription.update(datasift_historic_id=data['id'])
return data['id']
def test_headers_rate_limits_hit(self):
with patch('tweepy.API.%s' % self.METHOD_NAME,
new_callable=PropertyMock) as send_dm_mock:
api_proxy = RateLimitedTwitterApiWrapper.init_with_settings(
get_test_auth_settings())
send_dm_mock.return_value = partial(
fake_send_dm_rate_limit_headers_zero_remains, api_proxy)
with self.assertRaises(TwitterApiRateLimitError) as assert_cm:
api_proxy.send_direct_message('username', text='some text')
rate_limit_err = assert_cm.exception
self.assertTrue(
0 < rate_limit_err.wait_for <= QUARTER_HOUR_SEC,
"TwitterApiRateLimitError.wait_for param is incorrect")
self.assertTrue(
hasattr(api_proxy.base_api, 'last_response'),
"api methods patched incorrect, api should have :last_response attribute"
)
# - check there is DB stored limit
limits = api_proxy.get_rate_limit(self.METHOD_NAME)
self.assertTrue(limits, "Rate limits must be stored")
self.assertFalse(limits.is_manual(),
"This is headers rate limits test")
# call api again:
# - check api does not called, and TwitterApiRateLimitError raised
# - check there is the same DB stored limit
reset = limits.reset
delattr(api_proxy.base_api, 'last_response')
with self.assertRaises(TwitterApiRateLimitError) as assert_cm:
api_proxy.send_direct_message('username', text='another text')
limits = api_proxy.get_rate_limit(self.METHOD_NAME)
self.assertFalse(
hasattr(api_proxy.base_api, 'last_response'),
"API method must not be called after a RateLimits is stored")
self.assertEqual(
reset, limits.reset,
"Limits should not be changed in near 15 minutes")
# check rate limits removed after successfull call
limits.reset = datetime_to_timestamp(now()) - 1
limits.save()
send_dm_mock.return_value = partial(
fake_send_dm_rate_limit_headers_full, api_proxy)
resp = api_proxy.send_direct_message('username', text='success')
self.assertTrue(isinstance(resp, tweepy.models.DirectMessage),
"Got DM, response is OK")
limits = api_proxy.get_rate_limit(self.METHOD_NAME)
self.assertEqual(
limits, None,
"Rate Limits must be removed after successfull api call")
def get_json_input_file(self, size=10, event_types=None):
# schema:
json_file = tempfile.TemporaryFile('w+')
now_ts = datetime_to_timestamp(utc(now()))
res = []
for _ in xrange(size):
data_item = dict(zip(self.TEST_ITEM_COLUMNS, self._gen_item_values(now_ts)))
data_item.update({self.EVENT_TYPE_DATA_FIELD: random.choice(event_types)})
res.append(data_item)
json.dump(res, json_file)
# json_file.write(json.dump(res))
json_file.flush()
json_file.seek(0)
return json_file
def store_manual_rate_limit_info(self, method_name, limits):
''' create manual rate limit in DB if such was found in response
or increase delay if we catch limits again
returns seconds remaining to reset limit
'''
if limits:
if limits.delay < 125 * MINUTE: # stop increasing delay after 125 min
limits.delay *= 5
limits.reset = datetime_to_timestamp(now()) + limits.delay
else:
delay = 5 * MINUTE
reset = datetime_to_timestamp(now()) + delay
limits = TwitterRateLimit(id=self.make_request_id(method_name),
remaining=0,
limit=-1,
reset=reset,
delay=delay)
limits.save()
self.logger.debug(
'[RateLimitedTwitterProxy: %s] Store manual rate limit delay: %s',
method_name, limits.delay)
return limits.delay
def apply_shema_type(col_value, schema_type):
# TODO: Finish adding all types. Bring up to date with get_type by using constants
if col_value is None:
return col_value
if schema_type == TYPE_STRING:
return str(col_value)
if schema_type == TYPE_INTEGER:
try:
if isinstance(col_value, datetime.datetime):
return datetime_to_timestamp(col_value)
return int(col_value)
except Exception, ex:
raise SchemaValidationError(ex)
def fake_send_dm_rate_limit_headers_full(api_proxy, id_or_username, text=None):
api = api_proxy.base_api
resp = requests.Response()
setattr(api, 'last_response', resp)
resp.headers['X-Rate-Limit-Limit'] = '15'
resp.headers['X-Rate-Limit-Remaining'] = '15'
resp.headers['X-Rate-Limit-Reset'] = str(
datetime_to_timestamp(now()) + QUARTER_HOUR_SEC)
json_file = os.path.join(DATA_PATH, 'tw_dm.json')
with open(json_file) as f:
dm_json = json.load(f)
dm = tweepy.models.DirectMessage.parse(api, dm_json)
return dm
def push_posts(self, tweets, post_data_format=None, post_data_dump=None):
insert_data = dict(subscription=self.subscription,
post_data_format=post_data_format)
for tweet in tweets:
log_state(self.subscription.channel.id, str(tweet['id']),
PostState.ARRIVED_IN_RECOVERY)
try:
insert_data.update({
"post_data":
post_data_dump(tweet),
"timestamp":
datetime_to_timestamp(parse_datetime(tweet['created_at']))
})
QueuedHistoricData.objects.create(**insert_data)
log_state(self.subscription.channel.id, str(tweet['id']),
PostState.ADDED_TO_WORKER_QUEUE)
except:
LOGGER.exception(u'QueuedHistoricData.objects.create: %s' %
insert_data)
return len(tweets)
def test_media_fetching_rate_limits_hit(self):
with patch('requests.get') as get_resp_mock:
FETCH_MEDIA_METHOD_NAME = "fetch_media"
get_resp_mock.return_value = fake_download_media_err_response()
fetcher = TwitterMediaFetcher(auth_tuple=get_test_auth_settings())
media_url = 'https://ton.twitter.com/1.1/ton/data/dm/idfrom/idto/name.jpg'
with self.assertRaises(TwitterApiRateLimitError):
resp = fetcher.fetch_media(media_url)
limits = fetcher.api.get_rate_limit(FETCH_MEDIA_METHOD_NAME)
self.assertTrue(limits is not None, "Limits must be stored to DB")
limits.reset = datetime_to_timestamp(now()) - 1
limits.save()
get_resp_mock.return_value = fake_download_media_success_response()
resp = fetcher.fetch_media(media_url)
self.assertTrue('media_data' in resp, "Something goes wrong")
limits = fetcher.api.get_rate_limit(FETCH_MEDIA_METHOD_NAME)
self.assertTrue(limits is None,
"Limits must be deleted after successfull call")
def calc_wait(self, reset_at):
wait_for = reset_at - datetime_to_timestamp(now())
return wait_for if wait_for > 0 else 0
def test_error_resp_rate_limits_hit(self):
with patch('tweepy.API.%s' % self.METHOD_NAME,
new_callable=PropertyMock) as send_dm_mock:
api_proxy = RateLimitedTwitterApiWrapper.init_with_settings(
get_test_auth_settings())
send_dm_mock.return_value = partial(
fake_send_dm_rate_limit_in_err_body, api_proxy)
with self.assertRaises(TwitterApiRateLimitError) as assert_cm:
api_proxy.send_direct_message('username', text='some text')
rate_limit_err = assert_cm.exception
self.assertTrue(
0 < rate_limit_err.wait_for <= QUARTER_HOUR_SEC,
"TwitterApiRateLimitError wait_for params is incorrect")
# - check there is DB stored limit
limits = api_proxy.get_rate_limit(self.METHOD_NAME)
self.assertTrue(limits, "Rate limits must be stored")
self.assertTrue(limits.is_manual(),
"This is error code based rate limits test")
# call api again:
# - check api does not called, and TwitterApiRateLimitError raised
# - check there is the same DB stored limit
reset = limits.reset
delay = limits.delay
delattr(api_proxy.base_api, 'last_response')
with self.assertRaises(TwitterApiRateLimitError) as assert_cm:
api_proxy.send_direct_message('username', text='another text')
limits = api_proxy.get_rate_limit(self.METHOD_NAME)
self.assertFalse(
hasattr(api_proxy.base_api, 'last_response'),
"API method must not be called after a RateLimits is stored")
self.assertEqual(
reset, limits.reset,
"Limits should not be changed in near 15 minutes")
self.assertEqual(
delay, limits.delay,
"Limits should not be changed in near 15 minutes")
# emulate we had wait for limits reset and try to execute api with error:
# limits delay must be raised in 5 times
# in the same test, we try another - DAILY rate limits code
delay = limits.delay
limits.reset = datetime_to_timestamp(now()) - 1
limits.save()
send_dm_mock.return_value = partial(
fake_send_dm_rate_limit_in_err_body,
api_proxy,
code=DAILY_RATE_LIMIT_CODE)
with self.assertRaises(TwitterApiRateLimitError):
api_proxy.send_direct_message('username',
text='totally another text')
limits = api_proxy.get_rate_limit(self.METHOD_NAME)
self.assertEqual(delay * 5, limits.delay)
# check, limits.delay not increased > 125min
delattr(api_proxy.base_api, 'last_response')
for _ in xrange(5):
limits = api_proxy.get_rate_limit(self.METHOD_NAME)
limits.reset = datetime_to_timestamp(now()) - 1
limits.save()
with self.assertRaises(TwitterApiRateLimitError):
api_proxy.send_direct_message('username',
text='totally another text')
limits = api_proxy.get_rate_limit(self.METHOD_NAME)
self.assertEqual(
limits.delay, 125 * 60,
"Manual Rate Limits delay must not be > 125 min, but it is: %s sec"
% limits.delay)
# check is reset time updated
wait_for = api_proxy.calc_wait(limits.reset)
self.assertTrue(
float(wait_for) / (125 * 60) > 0.99,
"Manual Rate Limits reset time must be + 125 min now")
# check rate limits removed after successfull call
limits.reset = datetime_to_timestamp(now()) - 1
limits.save()
send_dm_mock.return_value = partial(
fake_send_dm_rate_limit_headers_full, api_proxy)
resp = api_proxy.send_direct_message('username', text='success')
self.assertTrue(isinstance(resp, tweepy.models.DirectMessage),
"Got DM, response is OK")
limits = api_proxy.get_rate_limit(self.METHOD_NAME)
self.assertEqual(
limits, None,
"Manual Rate Limits must be removed after successfull api call"
)