def pull_requests_scanned(owner, repo, requestor_id=None):
"""
Update the timestamp on the repository object,
and delete old pull request that weren't updated.
"""
repo_name = repo
repo = Repository.get(owner, repo_name)
prev_scan_at = repo.pull_requests_last_scanned_at
repo.pull_requests_last_scanned_at = datetime.now()
db.session.add(repo)
if prev_scan_at:
# delete any PRs that were not updated since the previous scan --
# they have been removed from Github
query = (
PullRequest.query.filter_by(base_repo_id=repo.id)
.filter(PullRequest.last_replicated_at < prev_scan_at)
)
query.delete()
# delete the mutex
lock_name = LOCK_TEMPLATE.format(owner=owner, repo=repo_name)
Mutex.query.filter_by(name=lock_name).delete()
logger.info("Lock {name} deleted".format(name=lock_name))
db.session.commit()
def issues_scanned(owner, repo, requestor_id=None):
"""
Update the timestamp on the repository object,
and delete old issues that weren't updated.
"""
repo_name = repo
repo = Repository.get(owner, repo_name)
prev_scan_at = repo.issues_last_scanned_at
repo.issues_last_scanned_at = datetime.now()
db.session.add(repo)
if prev_scan_at:
# delete any issues that were not updated since the previous scan --
# they have been removed from Github
query = (
Issue.query.filter_by(repo_id=repo.id)
.filter(Issue.last_replicated_at < prev_scan_at)
)
query.delete()
# delete the mutex
lock_name = LOCK_TEMPLATE.format(owner=owner, repo=repo_name)
Mutex.query.filter_by(name=lock_name).delete()
logger.info("Lock {name} deleted".format(name=lock_name))
db.session.commit()
def process_repository(repo_data, via="webhook", fetched_at=None, commit=True,
requestor_id=None):
repo_id = repo_data.get("id")
if not repo_id:
raise MissingData("no repo ID")
# fetch the object from the database,
# or create it if it doesn't exist in the DB
repo = Repository.query.get(repo_id)
if not repo:
repo = Repository(id=repo_id)
# should we update the object?
fetched_at = fetched_at or datetime.now()
if repo.last_replicated_at > fetched_at:
raise StaleData()
# update the object
fields = (
"name", "private", "description", "fork", "homepage", "size",
"stargazers_count", "watchers_count", "language", "has_issues",
"has_downloads", "has_wiki", "has_pages", "forks_count",
"open_issues_count", "default_branch",
)
for field in fields:
if field in repo_data:
setattr(repo, field, repo_data[field])
dt_fields = ("created_at", "updated_at", "pushed_at")
for field in dt_fields:
if repo_data.get(field):
dt = parse_date(repo_data[field]).replace(tzinfo=None)
setattr(repo, field, dt)
# user references
user_fields = ("owner", "organization")
for user_field in user_fields:
if user_field not in repo_data:
continue
user_data = repo_data[user_field]
id_field = "{}_id".format(user_field)
login_field = "{}_login".format(user_field)
if user_data:
setattr(repo, id_field, user_data["id"])
if hasattr(repo, login_field):
setattr(repo, login_field, user_data["login"])
try:
process_user(user_data, via=via, fetched_at=fetched_at)
except StaleData:
pass
else:
setattr(repo, id_field, None)
if hasattr(repo, login_field):
setattr(repo, login_field, None)
# update replication timestamp
replicated_dt_field = "last_replicated_via_{}_at".format(via)
if hasattr(repo, replicated_dt_field):
setattr(repo, replicated_dt_field, fetched_at)
# add to DB session, so that it will be committed
db.session.add(repo)
# if we have requestor_id and permissions, update the permissions object
if requestor_id and repo_data.get("permissions"):
permissions_data = repo_data["permissions"]
assoc = UserRepoAssociation.query.get((requestor_id, repo_id))
if not assoc:
assoc = UserRepoAssociation(user_id=requestor_id, repo_id=repo_id)
for perm in ("admin", "push", "pull"):
if perm in permissions_data:
perm_attr = "can_{perm}".format(perm=perm)
setattr(assoc, perm_attr, permissions_data[perm])
db.session.add(assoc)
if commit:
db.session.commit()
return repo
def process_milestone(milestone_data,
via="webhook",
fetched_at=None,
commit=True,
repo_id=None):
number = milestone_data.get("number")
if not number:
raise MissingData("no milestone number")
if not repo_id:
url = milestone_data.get("url")
if not url:
raise MissingData("no milestone url")
# parse repo info from url
path = URLObject(url).path
assert path.segments[0] == "repos"
repo_owner = path.segments[1]
repo_name = path.segments[2]
# fetch repo from database
try:
repo = Repository.get(repo_owner, repo_name)
except MultipleResultsFound:
msg = "Repo {owner}/{repo} found multiple times!".format(
owner=repo_owner,
repo=repo_name,
)
raise DatabaseError(msg, {
"type": "milestone",
"owner": repo_owner,
"repo": repo_name,
})
if not repo:
msg = "Repo {owner}/{repo} not loaded in webhookdb".format(
owner=repo_owner,
repo=repo_name,
)
raise NotFound(msg, {
"type": "milestone",
"owner": repo_owner,
"repo": repo_name,
})
repo_id = repo.id
# fetch the object from the database,
# or create it if it doesn't exist in the DB
milestone = Milestone.query.get((repo_id, number))
if not milestone:
milestone = Milestone(repo_id=repo_id, number=number)
# should we update the object?
fetched_at = fetched_at or datetime.now()
if milestone.last_replicated_at > fetched_at:
raise StaleData()
# Most fields have the same name in our model as they do in Github's API.
# However, some are different. This mapping contains just the differences.
field_to_model = {
"open_issues": "open_issues_count",
"closed_issues": "closed_issues_count",
"due_on": "due_at",
}
# update the object
fields = (
"state",
"title",
"description",
"open_issues",
"closed_issues",
)
for field in fields:
if field in milestone_data:
mfield = field_to_model.get(field, field)
setattr(milestone, mfield, milestone_data[field])
dt_fields = ("created_at", "updated_at", "closed_at", "due_on")
for field in dt_fields:
if milestone_data.get(field):
dt = parse_date(milestone_data[field]).replace(tzinfo=None)
mfield = field_to_model.get(field, field)
setattr(milestone, mfield, dt)
# user references
user_fields = ("creator", )
for user_field in user_fields:
if user_field not in milestone_data:
continue
user_data = milestone_data[user_field]
id_field = "{}_id".format(user_field)
login_field = "{}_login".format(user_field)
if user_data:
setattr(milestone, id_field, user_data["id"])
if hasattr(milestone, login_field):
setattr(milestone, login_field, user_data["login"])
try:
process_user(user_data, via=via, fetched_at=fetched_at)
except StaleData:
pass
else:
setattr(milestone, id_field, None)
if hasattr(milestone, login_field):
setattr(milestone, login_field, None)
# update replication timestamp
replicated_dt_field = "last_replicated_via_{}_at".format(via)
if hasattr(milestone, replicated_dt_field):
setattr(milestone, replicated_dt_field, fetched_at)
# add to DB session, so that it will be committed
db.session.add(milestone)
if commit:
db.session.commit()
return milestone
def process_label(label_data,
via="webhook",
fetched_at=None,
commit=True,
repo_id=None):
name = label_data.get("name")
if not name:
raise MissingData("no label name")
if not repo_id:
url = label_data.get("url")
if not url:
raise MissingData("no label url")
# parse repo info from url
path = URLObject(url).path
assert path.segments[0] == "repos"
repo_owner = path.segments[1]
repo_name = path.segments[2]
# fetch repo from database
try:
repo = Repository.get(repo_owner, repo_name)
except MultipleResultsFound:
msg = "Repo {owner}/{repo} found multiple times!".format(
owner=repo_owner,
repo=repo_name,
)
raise DatabaseError(msg, {
"type": "label",
"owner": repo_owner,
"repo": repo_name,
})
if not repo:
msg = "Repo {owner}/{repo} not loaded in webhookdb".format(
owner=repo_owner,
repo=repo_name,
)
raise NotFound(msg, {
"type": "label",
"owner": repo_owner,
"repo": repo_name,
})
repo_id = repo.id
# fetch the object from the database,
# or create it if it doesn't exist in the DB
label = IssueLabel.query.get((repo_id, name))
if not label:
label = IssueLabel(repo_id=repo_id, name=name)
# should we update the object?
fetched_at = fetched_at or datetime.now()
if label.last_replicated_at > fetched_at:
raise StaleData()
# color reference
if "color" in label_data:
color_hex = label_data["color"]
if color_hex:
label.color = Color("#{hex}".format(hex=color_hex))
else:
label.color = None
# update replication timestamp
replicated_dt_field = "last_replicated_via_{}_at".format(via)
if hasattr(label, replicated_dt_field):
setattr(label, replicated_dt_field, fetched_at)
# add to DB session, so that it will be committed
db.session.add(label)
if commit:
db.session.commit()
return label