def test_rate_reset_timer_after_request(self):
github_session = GithubSession()
github_session.request_api('/')
rate_reset_time_no_cred_after_req = github_session.rate_reset_time
assert_that(rate_reset_time_no_cred_after_req, is_(not_none()))
assert_that(rate_reset_time_no_cred_after_req, is_(within_an_hour()))
def test_authorization_via_user_request(self):
github_session = GithubSession()
github_session.set_credentials(
personal_access_token=self.__personal_access_token)
body, headers, _ = github_session.request_api('/user')
assert_that(headers["Status"], is_("200 OK"))
assert_that(int(headers[RATE_REMAINING]), is_(greater_than(0)))
assert_that(body, has_entries())
assert_that(body['login'], is_(self.__username))
def test_request_current_sessions_user_unauthorized(self):
github_session = GithubSession()
_, headers, _ = github_session.request_api('/user')
assert_that(headers, not_none())
assert_that(headers["Status"], is_(any_of(UNAUTHORIZED, FORBIDDEN)))
if headers["Status"] == UNAUTHORIZED:
assert_that(int(headers[RATE_REMAINING]), is_(greater_than(0)))
if headers["Status"] == FORBIDDEN:
assert_that(int(headers[RATE_REMAINING]), is_(0))
def test_request_github_api_root(self):
github_session = GithubSession()
_, headers, _ = github_session.request_api('/')
assert_that(headers, not_none())
assert_that(headers["Status"], is_(any_of(OK, FORBIDDEN)))
if headers["Status"] == OK:
assert_that(int(headers[RATE_REMAINING]), is_(greater_than(0)))
if headers["Status"] == FORBIDDEN:
assert_that(int(headers[RATE_REMAINING]), is_(0))
def run_crawler_with_config(config, boto3_session):
"""Instantiates connections and instances, ties them together and runs the crawler."""
global GLOBAL
github_session = GithubSession()
github_session.set_credentials(
personal_access_token=config['github_token'])
msg_queue = SqsMessageQueue(botosession=boto3_session,
wait_time=20,
queue_address=config['msg_queue_address'],
msg_visibility_timeout=30)
my_prefix = 'raq_crawler_{}'.format(GLOBAL['RANDOM_ID'])
working_path = '/tmp/{}'.format(my_prefix)
results_path = working_path + '/results/'
os.makedirs(working_path)
os.makedirs(results_path)
GLOBAL['ORIGIN_DIR'] = working_path
while GLOBAL['SHOULD_RUN']:
os.chdir(GLOBAL['ORIGIN_DIR'])
message = msg_queue.pop_next_message()
if github_session.rate is not None:
info('Current rate left: {}'.format(github_session.rate))
info('Resets at {}'.format(
github_session.rate_reset_time.isoformat()))
debug('Received Msg')
debug(message.body_raw)
if 'task_type' not in message.body_dict:
warn(
"Received Message without task_type. '{}'\nIgnoring message and deleting it."
.format(message.body_raw))
message.delete()
if message.body_dict['task_type'] == 'repo':
handle_repo_task(github_session, message, working_path,
results_path)
message.delete()
elif message.body_dict['task_type'] == 'refill':
handle_refill_task(github_session, message, msg_queue)
message.delete()
elif message.body_dict['task_type'] == 'kill-15':
error('Received kill-15 task.')
GLOBAL['SHOULD_RUN'] = False
msg_queue.write_message(message_dict={'task_type': 'kill-15'})
def test_rate_tracker_decrements(self):
github_session = GithubSession()
github_session.set_credentials(
personal_access_token=self.__personal_access_token)
github_session.request_api('/')
rate_before = github_session.rate
github_session.request_api('/')
rate_after = github_session.rate
assert_that(rate_after, is_(less_than(rate_before)))
def test_rate_tracker_update_on_credential_change(self):
github_session = GithubSession()
github_session.request_api('/')
rate_no_cred_after_req = github_session.rate
github_session.set_credentials(
personal_access_token=self.__personal_access_token)
github_session.request_api('/')
rate_with_cred_after_req = github_session.rate
assert_that(rate_no_cred_after_req, is_(not_none()))
assert_that(rate_with_cred_after_req, is_(not_none()))
assert_that(rate_with_cred_after_req,
is_(not_(rate_no_cred_after_req)))