def fetch_resultset(self, url, repository, sha=None):
newrelic.agent.add_custom_parameter("sha", sha)
logger.info("fetching for {} {}".format(repository, url))
# there will only ever be one, with this url
push = fetch_json(url)["pushes"].values()[0]
commits = []
# TODO: Remove this when bug 1257602 is addressed
rev_hash_components = []
# we only want to ingest the last 200 commits for each push,
# to protect against the 5000+ commit merges on release day uplift.
for commit in push['changesets'][-200:]:
commits.append({
"revision": commit["node"],
"author": commit["author"],
"comment": commit["desc"],
})
rev_hash_components.append(commit['node'])
rev_hash_components.append(commit['branch'])
return {
"revision": commits[-1]["revision"],
'revision_hash': generate_revision_hash(rev_hash_components),
"author": push["user"],
"push_timestamp": push["date"],
"revisions": commits,
}
def fetch_push(self, url, repository, sha=None):
newrelic.agent.add_custom_parameter("sha", sha)
logger.info("fetching for {} {}".format(repository, url))
# there will only ever be one, with this url
push = fetch_json(url)["pushes"].values()[0]
commits = []
# TODO: Remove this when bug 1257602 is addressed
rev_hash_components = []
# we only want to ingest the last 200 commits for each push,
# to protect against the 5000+ commit merges on release day uplift.
for commit in push['changesets'][-200:]:
commits.append({
"revision": commit["node"],
"author": commit["author"],
"comment": commit["desc"],
})
rev_hash_components.append(commit['node'])
rev_hash_components.append(commit['branch'])
return {
"revision": commits[-1]["revision"],
'revision_hash': generate_revision_hash(rev_hash_components),
"author": push["user"],
"push_timestamp": push["date"],
"revisions": commits,
}
def transform(self, pushlog, repository):
# this contain the whole list of transformed pushes
result_sets = []
# last push available
if pushlog:
last_push = max(pushlog.keys())
else:
last_push = None
th_collections = {}
# iterate over the pushes
for push in pushlog.values():
result_set = dict()
result_set['push_timestamp'] = push['date']
result_set['revisions'] = []
# Author of the push/resultset
result_set['author'] = push['user']
rev_hash_components = []
# iterate over the revisions
for change in push['changesets']:
revision = dict()
# we need to get the short version of a revision
# because buildapi doesn't provide the long one
# and we need to match it
revision['revision'] = change['node'][0:12]
revision['files'] = change['files']
revision['author'] = change['author']
revision['branch'] = change['branch']
revision['comment'] = change['desc']
revision['repository'] = repository
rev_hash_components.append(change['node'])
rev_hash_components.append(change['branch'])
# append the revision to the push
result_set['revisions'].append(revision)
result_set['revision_hash'] = generate_revision_hash(
rev_hash_components)
if repository not in th_collections:
th_collections[repository] = TreeherderResultSetCollection()
th_resultset = th_collections[repository].get_resultset(result_set)
th_collections[repository].add(th_resultset)
# cache the last push seen
if last_push:
cache.set("{0}:last_push".format(repository), last_push)
return th_collections
def transform(self, pushlog, repository):
# this contain the whole list of transformed pushes
result_sets = []
# last push available
if pushlog:
last_push = max(pushlog.keys())
else:
last_push = None
th_collections = {}
# iterate over the pushes
for push in pushlog.values():
result_set = dict()
result_set['push_timestamp'] = push['date']
result_set['revisions'] = []
# Author of the push/resultset
result_set['author'] = push['user']
rev_hash_components = []
# iterate over the revisions
for change in push['changesets']:
revision = dict()
# we need to get the short version of a revision
# because buildapi doesn't provide the long one
# and we need to match it
revision['revision'] = change['node'][0:12]
revision['files'] = change['files']
revision['author'] = change['author']
revision['branch'] = change['branch']
revision['comment'] = change['desc']
revision['repository'] = repository
rev_hash_components.append(change['node'])
rev_hash_components.append(change['branch'])
# append the revision to the push
result_set['revisions'].append(revision)
result_set['revision_hash'] = generate_revision_hash(rev_hash_components)
if repository not in th_collections:
th_collections[ repository ] = TreeherderResultSetCollection()
th_resultset = th_collections[ repository ].get_resultset(result_set)
th_collections[ repository ].add(th_resultset)
# cache the last push seen
if last_push:
cache.set("{0}:last_push".format(repository), last_push)
return th_collections
def transform(self, pushlog, repository):
# this contain the whole list of transformed pushes
th_collections = {repository: TreeherderResultSetCollection()}
# iterate over the pushes
for push in pushlog.values():
if not push['changesets']:
# If a pushlog contains hidden changesets (changesets that are
# obsolete) then the call to json-pushes will return a push
# with no changesets. This was changed in bug 1286426.
# For us, if `changesets` is empty, we will not be able to get
# a revision, which is required for a resultset. So we
# need to just skip the push.
continue
result_set = dict()
result_set['push_timestamp'] = push['date']
result_set['revisions'] = []
# Author of the push/resultset
result_set['author'] = push['user']
result_set['active_status'] = push.get('active_status', 'active')
# TODO: Remove this with Bug 1257602 is addressed
rev_hash_components = []
# iterate over the revisions
# we only want to ingest the last 200 revisions.
for change in push['changesets'][-200:]:
revision = {
'revision': change['node'],
'author': change['author'],
'branch': change['branch'],
'comment': change['desc'],
'repository': repository
}
rev_hash_components.append(change['node'])
rev_hash_components.append(change['branch'])
# append the revision to the push
result_set['revisions'].append(revision)
result_set['revision_hash'] = generate_revision_hash(
rev_hash_components)
result_set['revision'] = result_set["revisions"][-1]["revision"]
th_resultset = th_collections[repository].get_resultset(result_set)
th_collections[repository].add(th_resultset)
return th_collections
def transform(self, pushlog, repository):
# this contain the whole list of transformed pushes
th_collections = {
repository: TreeherderResultSetCollection()
}
# iterate over the pushes
for push in pushlog.values():
if not push['changesets']:
# If a pushlog contains hidden changesets (changesets that are
# obsolete) then the call to json-pushes will return a push
# with no changesets. This was changed in bug 1286426.
# For us, if `changesets` is empty, we will not be able to get
# a revision, which is required for a resultset. So we
# need to just skip the push.
continue
result_set = dict()
result_set['push_timestamp'] = push['date']
result_set['revisions'] = []
# Author of the push/resultset
result_set['author'] = push['user']
result_set['active_status'] = push.get('active_status', 'active')
# TODO: Remove this with Bug 1257602 is addressed
rev_hash_components = []
# iterate over the revisions
# we only want to ingest the last 200 revisions.
for change in push['changesets'][-200:]:
revision = {
'revision': change['node'],
'author': change['author'],
'branch': change['branch'],
'comment': change['desc'],
'repository': repository}
rev_hash_components.append(change['node'])
rev_hash_components.append(change['branch'])
# append the revision to the push
result_set['revisions'].append(revision)
result_set['revision_hash'] = generate_revision_hash(rev_hash_components)
result_set['revision'] = result_set["revisions"][-1]["revision"]
th_resultset = th_collections[repository].get_resultset(result_set)
th_collections[repository].add(th_resultset)
return th_collections
def transform(self, pushlog, repository):
# this contain the whole list of transformed pushes
th_collections = {}
# iterate over the pushes
for push in pushlog.values():
result_set = dict()
result_set['push_timestamp'] = push['date']
result_set['revisions'] = []
# Author of the push/resultset
result_set['author'] = push['user']
result_set['active_status'] = push.get('active_status', 'active')
rev_hash_components = []
# iterate over the revisions
# we only want to ingest the last 200 revisions.
for change in push['changesets'][-200:]:
revision = dict()
# we need to get the short version of a revision
# because buildapi doesn't provide the long one
# and we need to match it
revision['revision'] = change['node'][0:12]
revision['files'] = change['files']
revision['author'] = change['author']
revision['branch'] = change['branch']
revision['comment'] = change['desc']
revision['repository'] = repository
rev_hash_components.append(change['node'])
rev_hash_components.append(change['branch'])
# append the revision to the push
result_set['revisions'].append(revision)
result_set['revision_hash'] = generate_revision_hash(
rev_hash_components)
if repository not in th_collections:
th_collections[repository] = TreeherderResultSetCollection()
th_resultset = th_collections[repository].get_resultset(result_set)
th_collections[repository].add(th_resultset)
return th_collections
def transform(self, pushlog, repository):
# this contain the whole list of transformed pushes
th_collections = {}
# iterate over the pushes
for push in pushlog.values():
result_set = dict()
result_set['push_timestamp'] = push['date']
result_set['revisions'] = []
# Author of the push/resultset
result_set['author'] = push['user']
result_set['active_status'] = push.get('active_status', 'active')
rev_hash_components = []
# iterate over the revisions
# we only want to ingest the last 200 revisions.
for change in push['changesets'][-200:]:
revision = dict()
# we need to get the short version of a revision
# because buildapi doesn't provide the long one
# and we need to match it
revision['revision'] = change['node'][0:12]
revision['author'] = change['author']
revision['branch'] = change['branch']
revision['comment'] = change['desc']
revision['repository'] = repository
rev_hash_components.append(change['node'])
rev_hash_components.append(change['branch'])
# append the revision to the push
result_set['revisions'].append(revision)
result_set['revision_hash'] = generate_revision_hash(rev_hash_components)
if repository not in th_collections:
th_collections[repository] = TreeherderResultSetCollection()
th_resultset = th_collections[repository].get_resultset(result_set)
th_collections[repository].add(th_resultset)
return th_collections
def transform(self, pushlog, repository):
# this contain the whole list of transformed pushes
th_collections = {}
# iterate over the pushes
for push in pushlog.values():
result_set = dict()
result_set['push_timestamp'] = push['date']
result_set['revisions'] = []
# Author of the push/resultset
result_set['author'] = push['user']
result_set['active_status'] = push.get('active_status', 'active')
# TODO: Remove this with Bug 1257602 is addressed
rev_hash_components = []
# iterate over the revisions
# we only want to ingest the last 200 revisions.
for change in push['changesets'][-200:]:
revision = dict()
revision['revision'] = change['node']
revision['author'] = change['author']
revision['branch'] = change['branch']
revision['comment'] = change['desc']
revision['repository'] = repository
rev_hash_components.append(change['node'])
rev_hash_components.append(change['branch'])
# append the revision to the push
result_set['revisions'].append(revision)
result_set['revision_hash'] = generate_revision_hash(
rev_hash_components)
result_set['revision'] = result_set["revisions"][-1]["revision"]
if repository not in th_collections:
th_collections[repository] = TreeherderResultSetCollection()
th_resultset = th_collections[repository].get_resultset(result_set)
th_collections[repository].add(th_resultset)
return th_collections
def transform(self, pushlog, repository):
# this contain the whole list of transformed pushes
th_collections = {}
# iterate over the pushes
for push in pushlog.values():
result_set = dict()
result_set['push_timestamp'] = push['date']
result_set['revisions'] = []
# Author of the push/resultset
result_set['author'] = push['user']
result_set['active_status'] = push.get('active_status', 'active')
# TODO: Remove this with Bug 1257602 is addressed
rev_hash_components = []
# iterate over the revisions
# we only want to ingest the last 200 revisions.
for change in push['changesets'][-200:]:
revision = dict()
revision['revision'] = change['node']
revision['author'] = change['author']
revision['branch'] = change['branch']
revision['comment'] = change['desc']
revision['repository'] = repository
rev_hash_components.append(change['node'])
rev_hash_components.append(change['branch'])
# append the revision to the push
result_set['revisions'].append(revision)
result_set['revision_hash'] = generate_revision_hash(rev_hash_components)
result_set['revision'] = result_set["revisions"][-1]["revision"]
if repository not in th_collections:
th_collections[repository] = TreeherderResultSetCollection()
th_resultset = th_collections[repository].get_resultset(result_set)
th_collections[repository].add(th_resultset)
return th_collections
def transform(self, pushlog, repository):
# this contain the whole list of transformed pushes
th_collections = {}
# iterate over the pushes
for push in pushlog.values():
result_set = dict()
result_set["push_timestamp"] = push["date"]
result_set["revisions"] = []
# Author of the push/resultset
result_set["author"] = push["user"]
result_set["active_status"] = push.get("active_status", "active")
rev_hash_components = []
# iterate over the revisions
# we only want to ingest the last 200 revisions.
for change in push["changesets"][-200:]:
revision = dict()
revision["revision"] = change["node"]
revision["author"] = change["author"]
revision["branch"] = change["branch"]
revision["comment"] = change["desc"]
revision["repository"] = repository
rev_hash_components.append(change["node"])
rev_hash_components.append(change["branch"])
# append the revision to the push
result_set["revisions"].append(revision)
result_set["revision_hash"] = generate_revision_hash(rev_hash_components)
if repository not in th_collections:
th_collections[repository] = TreeherderResultSetCollection()
th_resultset = th_collections[repository].get_resultset(result_set)
th_collections[repository].add(th_resultset)
return th_collections
def transform_push(self, push):
commits = []
# TODO: Remove this when bug 1257602 is addressed
rev_hash_components = []
# we only want to ingest the last 200 commits for each push,
# to protect against the 5000+ commit merges on release day uplift.
for commit in push['changesets'][-200:]:
commits.append({
'revision': commit['node'],
'author': commit['author'],
'comment': commit['desc'],
})
rev_hash_components.append(commit['node'])
rev_hash_components.append(commit['branch'])
return {
'revision': commits[-1]["revision"],
'revision_hash': generate_revision_hash(rev_hash_components),
'author': push['user'],
'push_timestamp': push['date'],
'revisions': commits,
}
def transform_push(self, push):
commits = []
# TODO: Remove this when bug 1257602 is addressed
rev_hash_components = []
# we only want to ingest the last 200 commits for each push,
# to protect against the 5000+ commit merges on release day uplift.
for commit in push['changesets'][-200:]:
commits.append({
'revision': commit['node'],
'author': commit['author'],
'comment': commit['desc'],
})
rev_hash_components.append(commit['node'])
rev_hash_components.append(commit['branch'])
return {
'revision': commits[-1]["revision"],
'revision_hash': generate_revision_hash(rev_hash_components),
'author': push['user'],
'push_timestamp': push['date'],
'revisions': commits,
}