Exemple #1
0
    def create_job(self, guid, **kwargs):
        job = TreeherderJob()

        job.add_job_guid(guid)
        job.add_tier(self.settings['treeherder']['tier'])

        job.add_product_name('firefox')

        job.add_project(self.repository)
        job.add_revision_hash(self.retrieve_revision_hash())

        # Add platform and build information
        job.add_machine(socket.getfqdn())
        platform = self._get_treeherder_platform()
        job.add_machine_info(*platform)
        job.add_build_info(*platform)

        # TODO debug or others?
        job.add_option_collection({'opt': True})

        # TODO: Add e10s group once we run those tests
        job.add_group_name(self.settings['treeherder']['group_name'].format(**kwargs))
        job.add_group_symbol(self.settings['treeherder']['group_symbol'].format(**kwargs))

        # Bug 1174973 - for now we need unique job names even in different groups
        job.add_job_name(self.settings['treeherder']['job_name'].format(**kwargs))
        job.add_job_symbol(self.settings['treeherder']['job_symbol'].format(**kwargs))

        job.add_start_timestamp(int(time.time()))

        # Bug 1175559 - Workaround for HTTP Error
        job.add_end_timestamp(0)

        return job
    def test_job_sample_data(self):

        for job in self.job_data:

            tj = TreeherderJob()

            tj.add_revision_hash( job['revision_hash'] )
            tj.add_project( job['project'] )
            tj.add_coalesced_guid( job['coalesced'] )
            tj.add_job_guid( job['job']['job_guid'] )
            tj.add_job_name( job['job']['name'] )
            tj.add_job_symbol( job['job']['job_symbol'] )
            tj.add_group_name( job['job']['group_name'] )
            tj.add_group_symbol( job['job']['group_symbol'] )
            tj.add_description( job['job']['desc'] )
            tj.add_product_name( job['job']['product_name'] )
            tj.add_state( job['job']['state'] )
            tj.add_result( job['job']['result'] )
            tj.add_reason( job['job']['reason'] )
            tj.add_who( job['job']['who'] )
            tj.add_submit_timestamp( job['job']['submit_timestamp'] )
            tj.add_start_timestamp( job['job']['start_timestamp'] )
            tj.add_end_timestamp( job['job']['end_timestamp'] )
            tj.add_machine( job['job']['machine'] )
            tj.add_build_url( job['job']['build_url'] )

            tj.add_build_info(
                job['job']['build_platform']['os_name'],
                job['job']['build_platform']['platform'],
                job['job']['build_platform']['architecture']
                )

            tj.add_machine_info(
                job['job']['machine_platform']['os_name'],
                job['job']['machine_platform']['platform'],
                job['job']['machine_platform']['architecture']
                )

            tj.add_option_collection( job['job']['option_collection'] )

            tj.add_log_reference(
                'builds-4h', job['job']['log_references'][0]['url'] )

            # if the blob is empty, TreeherderJob will ignore the insertion
            job['job']['artifacts'][0]['blob'] = "some value"

            tj.add_artifact(
                job['job']['artifacts'][0]['name'],
                job['job']['artifacts'][0]['type'],
                job['job']['artifacts'][0]['blob'])

            self.compare_structs(tj.data, job)

            # Confirm we get the same dict if we initialize from
            # a job dict
            tj_dict = TreeherderJob(job)
            self.compare_structs(tj.data, tj_dict.data)
    def test_job_sample_data(self):

        for job in self.job_data:

            tj = TreeherderJob()

            tj.add_revision_hash(job['revision_hash'])
            tj.add_project(job['project'])
            tj.add_coalesced_guid(job['coalesced'])
            tj.add_job_guid(job['job']['job_guid'])
            tj.add_job_name(job['job']['name'])
            tj.add_job_symbol(job['job']['job_symbol'])
            tj.add_group_name(job['job']['group_name'])
            tj.add_group_symbol(job['job']['group_symbol'])
            tj.add_description(job['job']['desc'])
            tj.add_product_name(job['job']['product_name'])
            tj.add_state(job['job']['state'])
            tj.add_result(job['job']['result'])
            tj.add_reason(job['job']['reason'])
            tj.add_who(job['job']['who'])
            tj.add_submit_timestamp(job['job']['submit_timestamp'])
            tj.add_start_timestamp(job['job']['start_timestamp'])
            tj.add_end_timestamp(job['job']['end_timestamp'])
            tj.add_machine(job['job']['machine'])
            tj.add_build_url(job['job']['build_url'])

            tj.add_build_info(job['job']['build_platform']['os_name'],
                              job['job']['build_platform']['platform'],
                              job['job']['build_platform']['architecture'])

            tj.add_machine_info(job['job']['machine_platform']['os_name'],
                                job['job']['machine_platform']['platform'],
                                job['job']['machine_platform']['architecture'])

            tj.add_option_collection(job['job']['option_collection'])

            tj.add_log_reference('builds-4h',
                                 job['job']['log_references'][0]['url'])

            # if the blob is empty, TreeherderJob will ignore the insertion
            job['job']['artifacts'][0]['blob'] = "some value"

            tj.add_artifact(job['job']['artifacts'][0]['name'],
                            job['job']['artifacts'][0]['type'],
                            job['job']['artifacts'][0]['blob'])

            self.compare_structs(tj.data, job)

            # Confirm we get the same dict if we initialize from
            # a job dict
            tj_dict = TreeherderJob(job)
            self.compare_structs(tj.data, tj_dict.data)
Exemple #4
0
    def create_job(self, data=None, **kwargs):
        """Creates a new instance of a Treeherder job for submission.

        :param data: Job data to use for initilization, e.g. from a previous submission, optional
        :param kwargs: Dictionary of necessary values to build the job details. The
            properties correlate to the placeholders in config.py.

        """
        data = data or {}

        job = TreeherderJob(data=data)

        # If no data is available we have to set all properties
        if not data:
            job.add_job_guid(str(uuid.uuid4()))
            job.add_tier(self.settings['treeherder']['tier'])

            job.add_product_name('firefox')

            job.add_project(self.repository)
            job.add_revision_hash(self.retrieve_revision_hash())

            # Add platform and build information
            job.add_machine(socket.getfqdn())
            platform = self.settings["treeherder"]["platform"]
            job.add_machine_info(*platform)
            job.add_build_info(*platform)

            # TODO debug or others?
            job.add_option_collection({'opt': True})

            # TODO: Add e10s group once we run those tests
            job.add_group_name(
                self.settings['treeherder']['group_name'].format(**kwargs))
            job.add_group_symbol(
                self.settings['treeherder']['group_symbol'].format(**kwargs))

            # Bug 1174973 - for now we need unique job names even in different groups
            job.add_job_name(
                self.settings['treeherder']['job_name'].format(**kwargs))
            job.add_job_symbol(
                self.settings['treeherder']['job_symbol'].format(**kwargs))

            job.add_start_timestamp(int(time.time()))

            # Bug 1175559 - Workaround for HTTP Error
            job.add_end_timestamp(0)

        return job
Exemple #5
0
    def create_job(self, data=None, **kwargs):
        """Creates a new instance of a Treeherder job for submission.

        :param data: Job data to use for initilization, e.g. from a previous submission, optional
        :param kwargs: Dictionary of necessary values to build the job details. The
            properties correlate to the placeholders in config.py.

        """
        data = data or {}

        job = TreeherderJob(data=data)

        # If no data is available we have to set all properties
        if not data:
            job.add_job_guid(str(uuid.uuid4()))
            job.add_tier(self.settings['treeherder']['tier'])

            job.add_product_name('firefox')

            job.add_project(self.repository)
            job.add_revision_hash(self.retrieve_revision_hash())

            # Add platform and build information
            job.add_machine(socket.getfqdn())
            platform = self._get_treeherder_platform()
            job.add_machine_info(*platform)
            job.add_build_info(*platform)

            # TODO debug or others?
            job.add_option_collection({'opt': True})

            # TODO: Add e10s group once we run those tests
            job.add_group_name(self.settings['treeherder']['group_name'].format(**kwargs))
            job.add_group_symbol(self.settings['treeherder']['group_symbol'].format(**kwargs))

            # Bug 1174973 - for now we need unique job names even in different groups
            job.add_job_name(self.settings['treeherder']['job_name'].format(**kwargs))
            job.add_job_symbol(self.settings['treeherder']['job_symbol'].format(**kwargs))

            job.add_start_timestamp(int(time.time()))

            # Bug 1175559 - Workaround for HTTP Error
            job.add_end_timestamp(0)

        return job
    def create_job(self, guid, **kwargs):
        job = TreeherderJob()

        job.add_job_guid(guid)

        job.add_product_name('raptor')

        job.add_project(self.repository)
        job.add_revision_hash(self.retrieve_revision_hash())

        # Add platform and build information
        job.add_machine(socket.getfqdn())
        platform = ("", "B2G Raptor %s %s" % (self.device, self.memory), "")
        job.add_machine_info(*platform)
        job.add_build_info(*platform)

        # TODO debug or others?
        job.add_option_collection({'opt': True})

        # TODO: Add e10s group once we run those tests
        job.add_group_name(
            self.settings['treeherder']['group_name'].format(**kwargs))
        job.add_group_symbol(
            self.settings['treeherder']['group_symbol'].format(**kwargs))

        # Bug 1174973 - for now we need unique job names even in different groups
        job.add_job_name(self.settings['treeherder']['job_name'].format(
            **kwargs) + ' (' + self.app_name + ')')
        if self.test_type == COLD_LAUNCH:
            job.add_job_symbol(self.settings['treeherder']['job_symbol'][
                self.app_name].format(**kwargs))
        else:
            job.add_job_symbol(
                self.settings['treeherder']['job_symbol'].format(**kwargs))

        # request time will be the jenkins TEST_TIME i.e. when jenkins job started
        job.add_submit_timestamp(int(os.environ['TEST_TIME']))

        # test start time for that paraticular app is set in jenkins job itself
        job.add_start_timestamp(int(self.start_time))

        # Bug 1175559 - Workaround for HTTP Error
        job.add_end_timestamp(0)

        return job
    def create_job(self, guid, **kwargs):
        job = TreeherderJob()

        job.add_job_guid(guid)

        job.add_product_name('raptor')

        job.add_project(self.repository)
        job.add_revision_hash(self.retrieve_revision_hash())

        # Add platform and build information
        job.add_machine(socket.getfqdn())
        platform = ("", "B2G Raptor %s %s" % (self.device, self.memory), "")
        job.add_machine_info(*platform)
        job.add_build_info(*platform)

        # TODO debug or others?
        job.add_option_collection({'opt': True})

        # TODO: Add e10s group once we run those tests
        job.add_group_name(self.settings['treeherder']['group_name'].format(**kwargs))
        job.add_group_symbol(self.settings['treeherder']['group_symbol'].format(**kwargs))

        # Bug 1174973 - for now we need unique job names even in different groups
        job.add_job_name(self.settings['treeherder']['job_name'].format(**kwargs) + ' (' + self.app_name + ')')
        if self.test_type == COLD_LAUNCH:
            job.add_job_symbol(self.settings['treeherder']['job_symbol'][self.app_name].format(**kwargs))
        else:
            job.add_job_symbol(self.settings['treeherder']['job_symbol'].format(**kwargs))

        # request time will be the jenkins TEST_TIME i.e. when jenkins job started
        job.add_submit_timestamp(int(os.environ['TEST_TIME']))

        # test start time for that paraticular app is set in jenkins job itself
        job.add_start_timestamp(int(self.start_time))

        # Bug 1175559 - Workaround for HTTP Error
        job.add_end_timestamp(0)

        return job
def create_treeherder_job(repo, revision, client, nodes):
    """
    Creates a treeherder job for the given set of data.

    :param repo: The repository this build came from.
    :param revision: The mercurial revision of the build.
    :param client: The TreeherderClient to use.
    :param nodes: The dataset for this build.
    """
    rev_hash = client.get_resultsets(repo, revision=revision)[0]['revision_hash']

    tj = TreeherderJob()
    tj.add_tier(2)
    tj.add_revision_hash(rev_hash)
    tj.add_project(repo)
    tj.add_job_guid(str(uuid.uuid4()))

    tj.add_job_name('awsy 1')
    tj.add_job_symbol('a1')
    tj.add_group_name('awsy')
    tj.add_group_symbol('AWSY')

    tj.add_product_name('firefox')
    tj.add_state('completed')
    tj.add_result('success')
    submit_timestamp = int(time.time())
    tj.add_submit_timestamp(submit_timestamp)
    tj.add_start_timestamp(submit_timestamp)
    tj.add_end_timestamp(submit_timestamp)
    tj.add_machine(socket.getfqdn())

    tj.add_build_info('linux', 'linux64', 'x86_64')
    tj.add_machine_info('linux', 'linux64', 'x86_64')
    tj.add_option_collection({'opt': True})

    perf_blob = create_perf_data(nodes)
    tj.add_artifact('performance_data', 'json', json.dumps({ 'performance_data': perf_blob }))

    return tj
    def create_job(self, guid, **kwargs):
        job = TreeherderJob()

        job.add_job_guid(guid)

        job.add_product_name('mozreview')

        job.add_project(self.repository)
        job.add_revision(self.revision)

        # Add platform and build information
        job.add_machine(socket.getfqdn())
        platform = self.get_treeherder_platform()

        job.add_machine_info(*platform)
        job.add_build_info(*platform)

        # TODO debug or others?
        job.add_option_collection({'opt': True})

        # TODO: Add e10s group once we run those tests
        job.add_group_name(self.settings['treeherder']['group_name'].format(**kwargs))
        job.add_group_symbol(self.settings['treeherder']['group_symbol'].format(**kwargs))

        # Bug 1174973 - for now we need unique job names even in different groups
        job.add_job_name(self.settings['treeherder'][self.test_suite]['job_name'].format(**kwargs))
        job.add_job_symbol(self.settings['treeherder'][self.test_suite]['job_symbol'].format(**kwargs))

        # request time and start time same is fine
        job.add_submit_timestamp(int(self.start_time))

        # test start time for that paraticular app is set in jenkins job itself
        job.add_start_timestamp(int(self.start_time))

        # Bug 1175559 - Workaround for HTTP Error
        job.add_end_timestamp(0)

        return job
def create_treeherder_job(repo, revision, client, nodes, s3=None):
    """
    Creates a treeherder job for the given set of data.

    :param repo: The repository this build came from.
    :param revision: The mercurial revision of the build.
    :param client: The TreeherderClient to use.
    :param nodes: The dataset for this build.
    :param s3: Optional Amazon S3 bucket to upload logs to.
    """
    tj = TreeherderJob()
    tj.add_tier(2)
    tj.add_revision(revision)
    tj.add_project(repo)

    tj.add_job_name('awsy 1')
    tj.add_job_symbol('a1')
    tj.add_group_name('awsy')
    tj.add_group_symbol('AWSY')

    tj.add_product_name('firefox')
    tj.add_state('completed')
    tj.add_result('success')
    submit_timestamp = int(time.time())
    tj.add_submit_timestamp(submit_timestamp)
    tj.add_start_timestamp(submit_timestamp)
    tj.add_end_timestamp(submit_timestamp)
    tj.add_machine(socket.getfqdn())

    tj.add_build_info('linux', 'linux64', 'x86_64')
    tj.add_machine_info('linux', 'linux64', 'x86_64')
    tj.add_option_collection({'opt': True})

    perf_blob = create_perf_data(nodes)
    perf_data = json.dumps({ 'performance_data': perf_blob })

    # Set the job guid to a combination of the revision and the job data. This
    # gives us a reasonably unique guid, but is also reproducible for the same
    # set of data.
    job_guid = hashlib.sha1(revision + perf_data)
    tj.add_job_guid(job_guid.hexdigest())

    # NB: The job guid must be set prior to adding artifacts.
    tj.add_artifact('performance_data', 'json', perf_data)

    # If an S3 connection is provided the logs for this revision are uploaded.
    # Addtionally a 'Job Info' blob is built up with links to the logs that
    # will be displayed in the 'Job details' pane in treeherder.
    if s3:
        job_details = []

        # To avoid overwriting existing data (perhaps if a job is retriggered)
        # the job guid is included in the key.
        log_prefix = "%s/%s/%s" % (repo, revision, job_guid.hexdigest())

        # Add the test log.
        log_id = '%s/%s' % (log_prefix, 'awsy_test_raw.log')
        job_detail = upload_artifact(s3, 'logs/%s.test.log' % revision,
                                     log_id, 'awsy_test.log')
        if 'url' in job_detail:
            # The test log is the main log and will be linked to the log
            # viewer and raw log icons in treeherder.
            tj.add_log_reference('test.log', job_detail['url'])
        job_details.append(job_detail)

        # Add the gecko log.
        log_id = '%s/%s' % (log_prefix, 'gecko.log')
        job_detail = upload_artifact(s3, "logs/%s.gecko.log" % revision,
                                     log_id, 'gecko.log')
        job_details.append(job_detail)

        tj.add_artifact('Job Info', 'json', { 'job_details': job_details })

    return tj