Example #1
0
    def fetch_resultset(self, url, repository, sha=None):
        newrelic.agent.add_custom_parameter("sha", sha)

        logger.info("fetching for {} {}".format(repository, url))
        # there will only ever be one, with this url
        push = fetch_json(url)["pushes"].values()[0]

        commits = []
        # TODO: Remove this when bug 1257602 is addressed
        rev_hash_components = []
        # we only want to ingest the last 200 commits for each push,
        # to protect against the 5000+ commit merges on release day uplift.
        for commit in push['changesets'][-200:]:
            commits.append({
                "revision": commit["node"],
                "author": commit["author"],
                "comment": commit["desc"],
            })
            rev_hash_components.append(commit['node'])
            rev_hash_components.append(commit['branch'])

        return {
            "revision": commits[-1]["revision"],
            'revision_hash': generate_revision_hash(rev_hash_components),
            "author": push["user"],
            "push_timestamp": push["date"],
            "revisions": commits,
        }
Example #2
0
    def fetch_push(self, url, repository, sha=None):
        newrelic.agent.add_custom_parameter("sha", sha)

        logger.info("fetching for {} {}".format(repository, url))
        # there will only ever be one, with this url
        push = fetch_json(url)["pushes"].values()[0]

        commits = []
        # TODO: Remove this when bug 1257602 is addressed
        rev_hash_components = []
        # we only want to ingest the last 200 commits for each push,
        # to protect against the 5000+ commit merges on release day uplift.
        for commit in push['changesets'][-200:]:
            commits.append({
                "revision": commit["node"],
                "author": commit["author"],
                "comment": commit["desc"],
            })
            rev_hash_components.append(commit['node'])
            rev_hash_components.append(commit['branch'])

        return {
            "revision": commits[-1]["revision"],
            'revision_hash': generate_revision_hash(rev_hash_components),
            "author": push["user"],
            "push_timestamp": push["date"],
            "revisions": commits,
        }
Example #3
0
    def transform(self, pushlog, repository):

        # this contain the whole list of transformed pushes
        result_sets = []

        # last push available
        if pushlog:
            last_push = max(pushlog.keys())
        else:
            last_push = None

        th_collections = {}

        # iterate over the pushes
        for push in pushlog.values():
            result_set = dict()
            result_set['push_timestamp'] = push['date']

            result_set['revisions'] = []

            # Author of the push/resultset
            result_set['author'] = push['user']

            rev_hash_components = []

            # iterate over the revisions
            for change in push['changesets']:
                revision = dict()
                # we need to get the short version of a revision
                # because buildapi doesn't provide the long one
                # and we need to match it
                revision['revision'] = change['node'][0:12]
                revision['files'] = change['files']
                revision['author'] = change['author']
                revision['branch'] = change['branch']
                revision['comment'] = change['desc']
                revision['repository'] = repository
                rev_hash_components.append(change['node'])
                rev_hash_components.append(change['branch'])

                # append the revision to the push
                result_set['revisions'].append(revision)

            result_set['revision_hash'] = generate_revision_hash(
                rev_hash_components)

            if repository not in th_collections:
                th_collections[repository] = TreeherderResultSetCollection()

            th_resultset = th_collections[repository].get_resultset(result_set)
            th_collections[repository].add(th_resultset)

        # cache the last push seen
        if last_push:
            cache.set("{0}:last_push".format(repository), last_push)

        return th_collections
    def transform(self, pushlog,  repository):

        # this contain the whole list of transformed pushes
        result_sets = []

        # last push available
        if pushlog:
            last_push = max(pushlog.keys())
        else:
            last_push = None

        th_collections = {}

        # iterate over the pushes
        for push in pushlog.values():
            result_set = dict()
            result_set['push_timestamp'] = push['date']

            result_set['revisions'] = []

            # Author of the push/resultset
            result_set['author'] = push['user']

            rev_hash_components = []

            # iterate over the revisions
            for change in push['changesets']:
                revision = dict()
                # we need to get the short version of a revision
                # because buildapi doesn't provide the long one
                # and we need to match it
                revision['revision'] = change['node'][0:12]
                revision['files'] = change['files']
                revision['author'] = change['author']
                revision['branch'] = change['branch']
                revision['comment'] = change['desc']
                revision['repository'] = repository
                rev_hash_components.append(change['node'])
                rev_hash_components.append(change['branch'])

                # append the revision to the push
                result_set['revisions'].append(revision)

            result_set['revision_hash'] = generate_revision_hash(rev_hash_components)

            if repository not in th_collections:
                th_collections[ repository ] = TreeherderResultSetCollection()

            th_resultset = th_collections[ repository ].get_resultset(result_set)
            th_collections[ repository ].add(th_resultset)

        # cache the last push seen
        if last_push:
            cache.set("{0}:last_push".format(repository), last_push)

        return th_collections
Example #5
0
    def transform(self, pushlog, repository):

        # this contain the whole list of transformed pushes

        th_collections = {repository: TreeherderResultSetCollection()}

        # iterate over the pushes
        for push in pushlog.values():
            if not push['changesets']:
                # If a pushlog contains hidden changesets (changesets that are
                # obsolete) then the call to json-pushes will return a push
                # with no changesets.  This was changed in bug 1286426.
                # For us, if `changesets` is empty, we will not be able to get
                # a revision, which is required for a resultset.  So we
                # need to just skip the push.
                continue

            result_set = dict()
            result_set['push_timestamp'] = push['date']

            result_set['revisions'] = []

            # Author of the push/resultset
            result_set['author'] = push['user']

            result_set['active_status'] = push.get('active_status', 'active')

            # TODO: Remove this with Bug 1257602 is addressed
            rev_hash_components = []

            # iterate over the revisions
            # we only want to ingest the last 200 revisions.
            for change in push['changesets'][-200:]:
                revision = {
                    'revision': change['node'],
                    'author': change['author'],
                    'branch': change['branch'],
                    'comment': change['desc'],
                    'repository': repository
                }
                rev_hash_components.append(change['node'])
                rev_hash_components.append(change['branch'])

                # append the revision to the push
                result_set['revisions'].append(revision)

            result_set['revision_hash'] = generate_revision_hash(
                rev_hash_components)
            result_set['revision'] = result_set["revisions"][-1]["revision"]

            th_resultset = th_collections[repository].get_resultset(result_set)
            th_collections[repository].add(th_resultset)

        return th_collections
Example #6
0
    def transform(self, pushlog, repository):

        # this contain the whole list of transformed pushes

        th_collections = {
            repository: TreeherderResultSetCollection()
        }

        # iterate over the pushes
        for push in pushlog.values():
            if not push['changesets']:
                # If a pushlog contains hidden changesets (changesets that are
                # obsolete) then the call to json-pushes will return a push
                # with no changesets.  This was changed in bug 1286426.
                # For us, if `changesets` is empty, we will not be able to get
                # a revision, which is required for a resultset.  So we
                # need to just skip the push.
                continue

            result_set = dict()
            result_set['push_timestamp'] = push['date']

            result_set['revisions'] = []

            # Author of the push/resultset
            result_set['author'] = push['user']

            result_set['active_status'] = push.get('active_status', 'active')

            # TODO: Remove this with Bug 1257602 is addressed
            rev_hash_components = []

            # iterate over the revisions
            # we only want to ingest the last 200 revisions.
            for change in push['changesets'][-200:]:
                revision = {
                    'revision': change['node'],
                    'author': change['author'],
                    'branch': change['branch'],
                    'comment': change['desc'],
                    'repository': repository}
                rev_hash_components.append(change['node'])
                rev_hash_components.append(change['branch'])

                # append the revision to the push
                result_set['revisions'].append(revision)

            result_set['revision_hash'] = generate_revision_hash(rev_hash_components)
            result_set['revision'] = result_set["revisions"][-1]["revision"]

            th_resultset = th_collections[repository].get_resultset(result_set)
            th_collections[repository].add(th_resultset)

        return th_collections
Example #7
0
    def transform(self, pushlog, repository):

        # this contain the whole list of transformed pushes

        th_collections = {}

        # iterate over the pushes
        for push in pushlog.values():
            result_set = dict()
            result_set['push_timestamp'] = push['date']

            result_set['revisions'] = []

            # Author of the push/resultset
            result_set['author'] = push['user']

            result_set['active_status'] = push.get('active_status', 'active')

            rev_hash_components = []

            # iterate over the revisions
            # we only want to ingest the last 200 revisions.
            for change in push['changesets'][-200:]:
                revision = dict()
                # we need to get the short version of a revision
                # because buildapi doesn't provide the long one
                # and we need to match it
                revision['revision'] = change['node'][0:12]
                revision['files'] = change['files']
                revision['author'] = change['author']
                revision['branch'] = change['branch']
                revision['comment'] = change['desc']
                revision['repository'] = repository
                rev_hash_components.append(change['node'])
                rev_hash_components.append(change['branch'])

                # append the revision to the push
                result_set['revisions'].append(revision)

            result_set['revision_hash'] = generate_revision_hash(
                rev_hash_components)

            if repository not in th_collections:
                th_collections[repository] = TreeherderResultSetCollection()

            th_resultset = th_collections[repository].get_resultset(result_set)
            th_collections[repository].add(th_resultset)

        return th_collections
Example #8
0
    def transform(self, pushlog, repository):

        # this contain the whole list of transformed pushes

        th_collections = {}

        # iterate over the pushes
        for push in pushlog.values():
            result_set = dict()
            result_set['push_timestamp'] = push['date']

            result_set['revisions'] = []

            # Author of the push/resultset
            result_set['author'] = push['user']

            result_set['active_status'] = push.get('active_status', 'active')

            rev_hash_components = []

            # iterate over the revisions
            # we only want to ingest the last 200 revisions.
            for change in push['changesets'][-200:]:
                revision = dict()
                # we need to get the short version of a revision
                # because buildapi doesn't provide the long one
                # and we need to match it
                revision['revision'] = change['node'][0:12]
                revision['author'] = change['author']
                revision['branch'] = change['branch']
                revision['comment'] = change['desc']
                revision['repository'] = repository
                rev_hash_components.append(change['node'])
                rev_hash_components.append(change['branch'])

                # append the revision to the push
                result_set['revisions'].append(revision)

            result_set['revision_hash'] = generate_revision_hash(rev_hash_components)

            if repository not in th_collections:
                th_collections[repository] = TreeherderResultSetCollection()

            th_resultset = th_collections[repository].get_resultset(result_set)
            th_collections[repository].add(th_resultset)

        return th_collections
Example #9
0
    def transform(self, pushlog, repository):

        # this contain the whole list of transformed pushes

        th_collections = {}

        # iterate over the pushes
        for push in pushlog.values():
            result_set = dict()
            result_set['push_timestamp'] = push['date']

            result_set['revisions'] = []

            # Author of the push/resultset
            result_set['author'] = push['user']

            result_set['active_status'] = push.get('active_status', 'active')

            # TODO: Remove this with Bug 1257602 is addressed
            rev_hash_components = []

            # iterate over the revisions
            # we only want to ingest the last 200 revisions.
            for change in push['changesets'][-200:]:
                revision = dict()
                revision['revision'] = change['node']
                revision['author'] = change['author']
                revision['branch'] = change['branch']
                revision['comment'] = change['desc']
                revision['repository'] = repository
                rev_hash_components.append(change['node'])
                rev_hash_components.append(change['branch'])

                # append the revision to the push
                result_set['revisions'].append(revision)

            result_set['revision_hash'] = generate_revision_hash(
                rev_hash_components)
            result_set['revision'] = result_set["revisions"][-1]["revision"]

            if repository not in th_collections:
                th_collections[repository] = TreeherderResultSetCollection()

            th_resultset = th_collections[repository].get_resultset(result_set)
            th_collections[repository].add(th_resultset)

        return th_collections
Example #10
0
    def transform(self, pushlog, repository):

        # this contain the whole list of transformed pushes

        th_collections = {}

        # iterate over the pushes
        for push in pushlog.values():
            result_set = dict()
            result_set['push_timestamp'] = push['date']

            result_set['revisions'] = []

            # Author of the push/resultset
            result_set['author'] = push['user']

            result_set['active_status'] = push.get('active_status', 'active')

            # TODO: Remove this with Bug 1257602 is addressed
            rev_hash_components = []

            # iterate over the revisions
            # we only want to ingest the last 200 revisions.
            for change in push['changesets'][-200:]:
                revision = dict()
                revision['revision'] = change['node']
                revision['author'] = change['author']
                revision['branch'] = change['branch']
                revision['comment'] = change['desc']
                revision['repository'] = repository
                rev_hash_components.append(change['node'])
                rev_hash_components.append(change['branch'])

                # append the revision to the push
                result_set['revisions'].append(revision)

            result_set['revision_hash'] = generate_revision_hash(rev_hash_components)
            result_set['revision'] = result_set["revisions"][-1]["revision"]

            if repository not in th_collections:
                th_collections[repository] = TreeherderResultSetCollection()

            th_resultset = th_collections[repository].get_resultset(result_set)
            th_collections[repository].add(th_resultset)

        return th_collections
Example #11
0
    def transform(self, pushlog, repository):

        # this contain the whole list of transformed pushes

        th_collections = {}

        # iterate over the pushes
        for push in pushlog.values():
            result_set = dict()
            result_set["push_timestamp"] = push["date"]

            result_set["revisions"] = []

            # Author of the push/resultset
            result_set["author"] = push["user"]

            result_set["active_status"] = push.get("active_status", "active")

            rev_hash_components = []

            # iterate over the revisions
            # we only want to ingest the last 200 revisions.
            for change in push["changesets"][-200:]:
                revision = dict()
                revision["revision"] = change["node"]
                revision["author"] = change["author"]
                revision["branch"] = change["branch"]
                revision["comment"] = change["desc"]
                revision["repository"] = repository
                rev_hash_components.append(change["node"])
                rev_hash_components.append(change["branch"])

                # append the revision to the push
                result_set["revisions"].append(revision)

            result_set["revision_hash"] = generate_revision_hash(rev_hash_components)

            if repository not in th_collections:
                th_collections[repository] = TreeherderResultSetCollection()

            th_resultset = th_collections[repository].get_resultset(result_set)
            th_collections[repository].add(th_resultset)

        return th_collections
Example #12
0
    def transform_push(self, push):
        commits = []
        # TODO: Remove this when bug 1257602 is addressed
        rev_hash_components = []
        # we only want to ingest the last 200 commits for each push,
        # to protect against the 5000+ commit merges on release day uplift.
        for commit in push['changesets'][-200:]:
            commits.append({
                'revision': commit['node'],
                'author': commit['author'],
                'comment': commit['desc'],
            })
            rev_hash_components.append(commit['node'])
            rev_hash_components.append(commit['branch'])

        return {
            'revision': commits[-1]["revision"],
            'revision_hash': generate_revision_hash(rev_hash_components),
            'author': push['user'],
            'push_timestamp': push['date'],
            'revisions': commits,
        }
Example #13
0
    def transform_push(self, push):
        commits = []
        # TODO: Remove this when bug 1257602 is addressed
        rev_hash_components = []
        # we only want to ingest the last 200 commits for each push,
        # to protect against the 5000+ commit merges on release day uplift.
        for commit in push['changesets'][-200:]:
            commits.append({
                'revision': commit['node'],
                'author': commit['author'],
                'comment': commit['desc'],
            })
            rev_hash_components.append(commit['node'])
            rev_hash_components.append(commit['branch'])

        return {
            'revision': commits[-1]["revision"],
            'revision_hash': generate_revision_hash(rev_hash_components),
            'author': push['user'],
            'push_timestamp': push['date'],
            'revisions': commits,
        }