def user_repositories_scanned(username, requestor_id=None):
"""
Update the timestamp on the pull request object,
and delete old pull request files that weren't updated.
"""
user = User.get(username)
prev_scan_at = user.repos_last_scanned_at
user.repos_last_scanned_at = datetime.now()
db.session.add(user)
if prev_scan_at:
# delete any repos that the user owns that were not updated
# since the previous scan -- the user must have deleted those
# repos from Github
query = (
Repository.query.filter_by(owner_id=user.id)
.filter(Repository.last_replicated_at < prev_scan_at)
)
query.delete(synchronize_session=False)
# delete the mutex
lock_name = LOCK_TEMPLATE.format(username=username)
Mutex.query.filter_by(name=lock_name).delete()
logger.info("Lock {name} deleted".format(name=lock_name))
db.session.commit()
def pull_requests_scanned(owner, repo, requestor_id=None):
"""
Update the timestamp on the repository object,
and delete old pull request that weren't updated.
"""
repo_name = repo
repo = Repository.get(owner, repo_name)
prev_scan_at = repo.pull_requests_last_scanned_at
repo.pull_requests_last_scanned_at = datetime.now()
db.session.add(repo)
if prev_scan_at:
# delete any PRs that were not updated since the previous scan --
# they have been removed from Github
query = (
PullRequest.query.filter_by(base_repo_id=repo.id)
.filter(PullRequest.last_replicated_at < prev_scan_at)
)
query.delete()
# delete the mutex
lock_name = LOCK_TEMPLATE.format(owner=owner, repo=repo_name)
Mutex.query.filter_by(name=lock_name).delete()
logger.info("Lock {name} deleted".format(name=lock_name))
db.session.commit()
def issues_scanned(owner, repo, requestor_id=None):
"""
Update the timestamp on the repository object,
and delete old issues that weren't updated.
"""
repo_name = repo
repo = Repository.get(owner, repo_name)
prev_scan_at = repo.issues_last_scanned_at
repo.issues_last_scanned_at = datetime.now()
db.session.add(repo)
if prev_scan_at:
# delete any issues that were not updated since the previous scan --
# they have been removed from Github
query = (
Issue.query.filter_by(repo_id=repo.id)
.filter(Issue.last_replicated_at < prev_scan_at)
)
query.delete()
# delete the mutex
lock_name = LOCK_TEMPLATE.format(owner=owner, repo=repo_name)
Mutex.query.filter_by(name=lock_name).delete()
logger.info("Lock {name} deleted".format(name=lock_name))
db.session.commit()
def spawn_page_tasks_for_user_repositories(
username, type="all", children=False, requestor_id=None, per_page=100,
):
# acquire lock or fail (we're already in a transaction)
lock_name = LOCK_TEMPLATE.format(username=username)
existing = Mutex.query.get(lock_name)
if existing:
return False
lock = Mutex(name=lock_name, user_id=requestor_id)
db.session.add(lock)
try:
db.session.commit()
except IntegrityError:
return False
else:
logger.info("Lock {name} set by {requestor_id}".format(
name=lock_name, requestor_id=requestor_id,
))
repo_page_url = (
"/users/{username}/repos?type={type}&per_page={per_page}"
).format(
username=username, type=type, per_page=per_page,
)
if requestor_id:
requestor = User.query.get(int(requestor_id))
assert requestor
if requestor.login == username:
# we can use the API for getting your *own* repos
repo_page_url = (
"/user/repos?type={type}&per_page={per_page}"
).format(
type=type, per_page=per_page,
)
resp = fetch_url_from_github(
repo_page_url, method="HEAD", requestor_id=requestor_id,
headers={"Accept": "application/vnd.github.moondragon+json"},
)
last_page_url = URLObject(resp.links.get('last', {}).get('url', ""))
last_page_num = int(last_page_url.query.dict.get('page', 1))
g = group(
sync_page_of_repositories_for_user.s(
username=username, type=type,
children=children, requestor_id=requestor_id,
per_page=per_page, page=page,
) for page in xrange(1, last_page_num+1)
)
finisher = user_repositories_scanned.si(
username=username, requestor_id=requestor_id,
)
return (g | finisher).delay()
def spawn_page_tasks_for_milestones(owner,
repo,
state="all",
children=False,
requestor_id=None,
per_page=100):
# acquire lock or fail (we're already in a transaction)
lock_name = LOCK_TEMPLATE.format(owner=owner, repo=repo)
existing = Mutex.query.get(lock_name)
if existing:
return False
lock = Mutex(name=lock_name, user_id=requestor_id)
db.session.add(lock)
try:
db.session.commit()
except IntegrityError:
return False
else:
logger.info("Lock {name} set by {requestor_id}".format(
name=lock_name,
requestor_id=requestor_id,
))
milestone_list_url = ("/repos/{owner}/{repo}/pulls?"
"state={state}&per_page={per_page}").format(
owner=owner,
repo=repo,
state=state,
per_page=per_page,
)
resp = fetch_url_from_github(
milestone_list_url,
method="HEAD",
requestor_id=requestor_id,
)
last_page_url = URLObject(resp.links.get('last', {}).get('url', ""))
last_page_num = int(last_page_url.query.dict.get('page', 1))
g = group(
sync_page_of_milestones.s(
owner=owner,
repo=repo,
state=state,
requestor_id=requestor_id,
per_page=per_page,
page=page,
) for page in xrange(1, last_page_num + 1))
finisher = milestones_scanned.si(
owner=owner,
repo=repo,
requestor_id=requestor_id,
)
return (g | finisher).delay()
def spawn_page_tasks_for_pull_requests(owner, repo, state="all", children=False,
requestor_id=None, per_page=100):
# acquire lock or fail (we're already in a transaction)
lock_name = LOCK_TEMPLATE.format(owner=owner, repo=repo)
existing = Mutex.query.get(lock_name)
if existing:
return False
lock = Mutex(name=lock_name, user_id=requestor_id)
db.session.add(lock)
try:
db.session.commit()
except IntegrityError:
return False
else:
logger.info("Lock {name} set by {requestor_id}".format(
name=lock_name, requestor_id=requestor_id,
))
pr_list_url = (
"/repos/{owner}/{repo}/pulls?"
"state={state}&per_page={per_page}"
).format(
owner=owner, repo=repo,
state=state, per_page=per_page,
)
resp = fetch_url_from_github(
pr_list_url, method="HEAD", requestor_id=requestor_id,
)
last_page_url = URLObject(resp.links.get('last', {}).get('url', ""))
last_page_num = int(last_page_url.query.dict.get('page', 1))
g = group(
sync_page_of_pull_requests.s(
owner=owner, repo=repo, state=state,
children=children, requestor_id=requestor_id,
per_page=per_page, page=page
) for page in xrange(1, last_page_num+1)
)
finisher = pull_requests_scanned.si(
owner=owner, repo=repo, requestor_id=requestor_id,
)
return (g | finisher).delay()
def user_repositories_scanned(username, requestor_id=None):
"""
Update the timestamp on the pull request object,
and delete old pull request files that weren't updated.
"""
user = User.get(username)
prev_scan_at = user.repos_last_scanned_at
user.repos_last_scanned_at = datetime.now()
db.session.add(user)
if prev_scan_at:
# delete any repos that the user owns that were not updated
# since the previous scan -- the user must have deleted those
# repos from Github
query = (Repository.query.filter_by(owner_id=user.id).filter(
Repository.last_replicated_at < prev_scan_at))
query.delete(synchronize_session=False)
# delete the mutex
lock_name = LOCK_TEMPLATE.format(username=username)
Mutex.query.filter_by(name=lock_name).delete()
logger.info("Lock {name} deleted".format(name=lock_name))
db.session.commit()
def fetch_url_from_github(self,
url,
as_user=None,
requestor_id=None,
**kwargs):
if "method" in kwargs:
method = kwargs.pop("method")
else:
method = "GET"
if method.upper() == "HEAD":
kwargs.setdefault("allow_redirects", False)
username = "******"
if as_user:
github.blueprint.config["user"] = as_user
username = "******".format(login=as_user.login)
elif requestor_id:
github.blueprint.config["user_id"] = int(requestor_id)
username = "******".format(requestor_id)
logger.info("{method} {url} as {username}".format(
method=method,
url=url,
username=username,
))
try:
resp = github.request(method=method, url=url, **kwargs)
except RateLimited as exc:
logger.info("rate limited: {url}".format(url=url))
# if this task is being executed inline, let the exception raise
# so that Flask's error-handling mechanisms can catch it
if self.request.is_eager:
raise
# otherwise, schedule this task to retry when the rate limit is reset
else:
logger.warn("Retrying {url} at {reset}".format(url=url,
reset=exc.reset))
self.retry(exc=exc, eta=exc.reset)
if resp.status_code == 404:
logger.info("not found: {url}".format(url=url))
raise NotFound(url)
if not resp.ok:
raise RequestException(resp.text)
return resp
def spawn_page_tasks_for_user_repositories(
username,
type="all",
children=False,
requestor_id=None,
per_page=100,
):
# acquire lock or fail (we're already in a transaction)
lock_name = LOCK_TEMPLATE.format(username=username)
existing = Mutex.query.get(lock_name)
if existing:
return False
lock = Mutex(name=lock_name, user_id=requestor_id)
db.session.add(lock)
try:
db.session.commit()
except IntegrityError:
return False
else:
logger.info("Lock {name} set by {requestor_id}".format(
name=lock_name,
requestor_id=requestor_id,
))
repo_page_url = (
"/users/{username}/repos?type={type}&per_page={per_page}").format(
username=username,
type=type,
per_page=per_page,
)
if requestor_id:
requestor = User.query.get(int(requestor_id))
assert requestor
if requestor.login == username:
# we can use the API for getting your *own* repos
repo_page_url = (
"/user/repos?type={type}&per_page={per_page}").format(
type=type,
per_page=per_page,
)
resp = fetch_url_from_github(
repo_page_url,
method="HEAD",
requestor_id=requestor_id,
headers={"Accept": "application/vnd.github.moondragon+json"},
)
last_page_url = URLObject(resp.links.get('last', {}).get('url', ""))
last_page_num = int(last_page_url.query.dict.get('page', 1))
g = group(
sync_page_of_repositories_for_user.s(
username=username,
type=type,
children=children,
requestor_id=requestor_id,
per_page=per_page,
page=page,
) for page in xrange(1, last_page_num + 1))
finisher = user_repositories_scanned.si(
username=username,
requestor_id=requestor_id,
)
return (g | finisher).delay()
