def test_get_token():
current_token = GithubAccount.get_token(0)
for i in range(1, 100):
new_token = GithubAccount.get_token(i)
assert new_token != None
assert new_token != current_token
current_token = new_token
def record(self, entity_item, start, end, size, timestamps):
self.seed += 1
timestamp = timestamps[0]
the_url = self.url.format(to_time_str(timestamp),
to_time_str(timestamp))
items = get_all_results(url=the_url,
token=GithubAccount.get_token(seed=self.seed))
current_time = now_pd_timestamp()
results = [{
'id': f'user_github_{item["login"]}',
'entity_id': f'user_github_{item["login"]}',
'timestamp': timestamp,
'exchange': 'github',
'entity_type': 'user',
'code': item['login'],
'node_id': item['node_id'],
'created_timestamp': current_time,
'updated_timestamp': None
} for item in items]
# for save faster
df = pd.DataFrame(data=results[:-1])
df_to_db(df=df,
data_schema=self.data_schema,
provider=self.provider,
force=True)
return results[-1:]
def follow_someone():
import random
year = random.randint(2012, 2014)
mon = random.randint(1, 12)
day = random.randint(1, 28)
start_timestamp = f'{year}-{mon}-{day}'
year = random.randint(2015, 2019)
mon = random.randint(1, 12)
day = random.randint(1, 28)
end_timestamp = f'{year}-{mon}-{day}'
users = get_data(provider='github',
data_schema=GithubUser,
start_timestamp=start_timestamp,
end_timestamp=end_timestamp,
return_type='domain',
limit=1000)
for seed in range(0, len(GithubAccount.tokens)):
for user in users:
resp = request_with_auth(url=url.format(user.code),
method='put',
token=GithubAccount.get_token(seed=seed),
headers={'Content-Length': '0'})
if resp.status_code == 204:
print('follow:{} ok'.format(user.code))
else:
print(resp.status_code)
def record(self, entity_item, start, end, size, timestamps):
self.seed += 1
the_url = self.url.format(entity_item.code)
repo_infos = get_result(url=the_url, token=GithubAccount.get_token(seed=self.seed))
for repo_info in repo_infos:
# get topics
topics_url = self.topics_url.format(entity_item.code, repo_info['name'])
topics = get_result(url=topics_url, token=GithubAccount.get_token(seed=self.seed),
headers={'Accept': 'application/vnd.github.mercy-preview+json'})
if topics:
repo_info['topics'] = topics['names']
repo_info['updated_timestamp'] = now_pd_timestamp()
repo_info['code'] = entity_item.code
repo_info['timestamp'] = to_pd_timestamp(repo_info['created_at'])
return repo_infos
def record(self, entity_item, start, end, size, timestamps):
self.seed += 1
the_url = self.url.format(entity_item.code)
user_info = get_result(url=the_url,
token=GithubAccount.get_token(seed=self.seed))
if user_info:
user_info['updated_timestamp'] = now_pd_timestamp()
return [user_info]
return []
def request_with_auth(url, method='get', data=None, token=None, headers=None):
if token is None:
token = GithubAccount.get_token(0)
if headers:
headers['Authorization'] = f'token {token}'
else:
headers = {
'Authorization': f'token {token}'
}
if method == 'get':
response = requests.get(url=url, headers=headers)
elif method == 'put':
response = requests.put(url=url, data=data, headers=headers)
else:
response = requests.post(url=url, data=data, headers=headers)
return response