def test_understand_repo_info(self):
responses.add(
responses.GET, 'https://api.github.com/repos/kkamkou/gitmostwanted.com', status=200,
body=json.dumps(self._body_repo_info()), content_type='application/json'
)
self.assertTupleEqual(
api.repo_info('kkamkou/gitmostwanted.com'), (self._body_repo_info(), 200)
)
responses.add(
responses.GET, 'https://api.github.com/repos/nobody/nothing', status=404,
body=json.dumps(self._body_not_found()), content_type='application/json'
)
self.assertTupleEqual(api.repo_info('nobody/nothing'), (None, 404))
def test_understand_repo_info(self):
responses.add(responses.GET,
'https://api.github.com/repos/kkamkou/gitmostwanted.com',
status=200,
body=json.dumps(self._body_repo_info()),
content_type='application/json')
self.assertTupleEqual(api.repo_info('kkamkou/gitmostwanted.com'),
(self._body_repo_info(), 200))
responses.add(responses.GET,
'https://api.github.com/repos/nobody/nothing',
status=404,
body=json.dumps(self._body_not_found()),
content_type='application/json')
self.assertTupleEqual(api.repo_info('nobody/nothing'), (None, 404))
def metadata_refresh(num_days):
repos = Repo.query\
.filter(
Repo.checked_at.is_(None) |
(Repo.checked_at <= datetime.now() + timedelta(days=num_days * -1))
)\
.yield_per(25)\
.limit(300) # GitHub allows only 3000 calls per day within a token
for repo in repos:
repo.checked_at = datetime.now()
details, code = api.repo_info(repo.full_name)
if not details:
if 400 <= code < 500:
repo.worth -= 1
app.logger.info(
'{0} is not found, the "worth" has been decreased by 1'.format(repo.full_name)
)
continue
for key in ['description', 'language', 'homepage', 'stargazers_count']:
if getattr(repo, key) != details[key]:
setattr(repo, key, details[key])
db.session.commit()
return repos.count()
def most_starred_sync(model_name: str, query: str):
app.logger.info('Importing repos of %s', model_name)
model = getattr(report, model_name)
service = bigquery.instance(app)
db.session.query(model).delete()
job = Job(service, query)
job.execute()
for row in results_of(job):
info, code = repo_info(row[1])
if not info:
continue
db.session.merge(
model(id=row[0],
cnt_watch=row[2],
repo=Repo(id=info['id'],
name=info['name'],
language=info['language'],
full_name=info['full_name'],
description=info['description'],
html_url=info['html_url'],
homepage=info['homepage'],
created_at=datetime.strptime(
info['created_at'], '%Y-%m-%dT%H:%M:%SZ'))))
db.session.commit()
def most_starred_sync(model_name: str, query: str):
app.logger.info('Importing repos of %s', model_name)
model = getattr(report, model_name)
service = bigquery.instance(app)
db.session.query(model).delete()
job = Job(service, query)
job.execute()
for row in results_of(job):
info, code = repo_info(row[1])
if not info:
continue
db.session.merge(
model(
id=row[0],
cnt_watch=row[2],
repo=Repo(
id=info['id'],
name=info['name'],
language=info['language'],
full_name=info['full_name'],
description=info['description'],
html_url=info['html_url'],
homepage=info['homepage'],
created_at=datetime.strptime(info['created_at'], '%Y-%m-%dT%H:%M:%SZ')
)
)
)
db.session.commit()
def metadata_refresh(num_days):
repos = Repo.query\
.filter(
(Repo.status != 'deleted') & (
Repo.checked_at.is_(None) |
(Repo.checked_at <= datetime.now() + timedelta(days=num_days * -1))
)
)\
.limit(300) # GitHub allows only 3000 calls per day within a token
for repo in repos:
details, code = api.repo_info(repo.full_name)
if not details:
if 400 <= code < 500:
repo.worth -= 1
db.session.commit()
log.info(
'{0} is not found (code={1}), the "worth" has been decreased by 1'
.format(repo.full_name, code))
continue
repo.checked_at = datetime.now()
for key in [
'description', 'forks_count', 'homepage', 'language',
'open_issues_count', 'size', 'stargazers_count',
'subscribers_count'
]:
if getattr(repo, key) != details[key]:
setattr(repo, key, details[key])
if 'license' in details and details['license'] is not None:
license = details['license'].get('key', 'unlicense')
if license != repo.license:
setattr(repo, 'license', license)
db.session.commit()
log.info('Repository {0}({1}) has been updated'.format(
repo.id, repo.full_name))
return repos.count()
