Beispiel #1
0
    def __init__(self,
                 configfile,
                 authfile,
                 treeherder_configfile,
                 debug,
                 log_folder,
                 logger,
                 message=None,
                 display_only=False):

        self.config = JSONFile(configfile).read()
        self.debug = debug
        self.log_folder = log_folder
        self.logger = logger
        self.display_only = display_only
        self.message = message
        self.treeherder_config = {}

        self.load_authentication_config(authfile)

        self.jenkins = jenkins.Jenkins(
            self.authentication['jenkins']['url'],
            self.authentication['jenkins']['user'],
            self.authentication['jenkins']['password'])

        # Setup Pulse listeners
        queue_name = 'queue/{user}/{host}/{type}'.format(
            user=self.authentication['pulse']['user'],
            host=socket.getfqdn(),
            type=self.config['pulse']['applabel'])

        # Load settings from the Treeherder config file
        with open(treeherder_configfile, 'r') as f:
            for line in f.readlines():
                if line.strip() and not line.startswith('#'):
                    key, value = line.strip().split('=')
                    self.treeherder_config.update({key: value})

        # Queue for build notifications
        queue_builds = NormalizedBuildQueue(name='{}_build'.format(queue_name),
                                            callback=self.process_build,
                                            pulse_config=self.config['pulse'])

        # Queue for release build notifications
        queue_release_builds = ReleaseTaskCompletedQueue(
            name='{}_build_release'.format(queue_name),
            callback=self.process_build,
            pulse_config=self.config['pulse'])

        # Queue for update notifications
        queue_updates = FunsizeTaskCompletedQueue(
            name='{}_update'.format(queue_name),
            callback=self.process_build,
            pulse_config=self.config['pulse'])

        # When a local message is used, process it and return immediately
        if self.message:
            data = JSONFile(self.message).read()

            # Check type of message and let it process by the correct queue
            if data.get('ACCEPTED_MAR_CHANNEL_IDS'):
                queue_updates.process_message(data, None)
            elif data.get('tags'):
                queue_builds.process_message(data, None)
            else:
                queue_release_builds.process_message(data, None)

            return

        with lib.PulseConnection(userid=self.authentication['pulse']['user'],
                                 password=self.authentication['pulse']
                                 ['password']) as connection:
            consumer = lib.PulseConsumer(connection)

            try:
                consumer.add_queue(queue_builds)
                consumer.add_queue(queue_release_builds)
                consumer.add_queue(queue_updates)

                consumer.run()
            except KeyboardInterrupt:
                self.logger.info('Shutting down Pulse listener')
Beispiel #2
0
    def process_build(self, **pulse_properties):
        """Check properties and trigger a Jenkins build.

        :param allowed_test: Type of tests which are allowed to be run.
        :param platform: Platform to run the tests on.
        :param product: Name of the product (application).
        :param branch: Name of the branch the build was created off.
        :param locale: Locale of the build.
        :param buildid: ID of the build.
        :param revision: Revision (changeset) of the build.
        :param tags: Build classification tags (e.g. nightly, l10n).
        :param test_packages_url: URL to the test_packages.json file.
        :param version: Version of the build.
        :param status: Build status from Buildbot (build notifications only).
        :param target_buildid: ID of the build after the upgrade (update notification only).
        :param target_version: Version of the build after the upgrade (update notification only).
        :param tree: Releng branch name the build was created off.
        :param raw_json: Raw pulse notification data

        """
        # Known failures from buildbot (http://mzl.la/1hlCYkw)
        buildbot_results = [
            'success', 'warnings', 'failure', 'skipped', 'exception', 'retry'
        ]

        # Bug 1176828 - Repack notifications for beta/release builds do not contain
        # a buildid. So use the timestamp if present as replacement
        if not pulse_properties['buildid'] and 'timestamp' in pulse_properties[
                'raw_json']:
            try:
                d = datetime.strptime(
                    pulse_properties['raw_json']['timestamp'],
                    '%Y-%m-%dT%H:%M:%SZ')
                pulse_properties['buildid'] = d.strftime('%Y%m%d%H%M')
            except:
                pass

        # Print build information to console
        if pulse_properties.get('target_buildid'):
            self.logger.info(
                '{product} {target_version} ({buildid} => {target_buildid},'
                ' {revision}, {locale}, {platform}) [{branch}]'.format(
                    **pulse_properties))
        else:
            self.logger.info(
                '{product} {version} ({buildid}, {revision}, {locale},'
                ' {platform}) [{branch}]'.format(**pulse_properties))

        # Store build information to disk
        basename = '{buildid}_{product}_{locale}_{platform}.log'.format(
            **pulse_properties)
        if pulse_properties.get('target_buildid'):
            basename = '{}_{}'.format(pulse_properties['target_buildid'],
                                      basename)
        filename = os.path.join(self.log_folder, pulse_properties['tree'],
                                basename)

        try:
            if not os.path.exists(filename):
                JSONFile(filename).write(pulse_properties['raw_json'])
        except Exception as e:
            self.logger.warning("Log file could not be written: {}.".format(
                str(e)))

        # Lets keep it after saving the log information so we might be able to
        # manually force-trigger those jobs in case of build failures.
        if pulse_properties.get(
                'status') and pulse_properties['status'] not in (0, 5):
            raise ValueError(
                'Cancel processing due to broken build: {}'.format(
                    buildbot_results[pulse_properties['status']]))

        tree_config = self.config['jenkins']['jobs'][pulse_properties['tree']]
        platform_id = self.get_platform_identifier(
            pulse_properties['platform'])

        # Get some properties now so it hasn't to be done for each individual platform version
        pulse_properties['build_url'] = self.get_installer_url(
            pulse_properties)
        pulse_properties['test_packages_url'] = self.get_test_packages_url(
            pulse_properties)
        pulse_properties['mozharness_url'] = self.get_mozharness_url(
            pulse_properties['test_packages_url'])

        # Generate job data and queue up in Jenkins
        for testrun in tree_config['testruns']:
            if testrun not in pulse_properties['allowed_testruns']:
                continue

            # Fire off a build for each supported platform version
            for node in tree_config['nodes'][platform_id]:
                job = '{}_{}'.format(pulse_properties['tree'], testrun)
                self.logger.info('Triggering job "{}" on "{}"'.format(
                    job, node))

                if node == 'taskcluster':
                    try:
                        th_url = self.treeherder_config['TREEHERDER_URL']

                        pulse_properties.update({
                            'revision_hash':
                            treeherder.get_revision_hash(
                                urlparse.urlparse(th_url).netloc,
                                pulse_properties['branch'],
                                pulse_properties['revision']),
                            'treeherder_instance':
                            self.treeherder_config['TREEHERDER_INSTANCE'],
                        })

                        # This includes a hard-coded channel name for now. Finally it has to be set
                        # via a web interface once we run tc tasks for mozilla-aurora, due to the
                        # 'auroratest' channel usage after branch merges.
                        extra_params = self.generate_job_parameters(
                            testrun, node, **pulse_properties)
                        pulse_properties.update(extra_params)

                        fxui_worker = tc.FirefoxUIWorker(
                            client_id=self.
                            treeherder_config['TASKCLUSTER_CLIENT_ID'],
                            authentication=self.
                            treeherder_config['TASKCLUSTER_SECRET'],
                        )

                        payload = fxui_worker.generate_task_payload(
                            testrun, pulse_properties)

                        if self.display_only:
                            self.logger.info('Payload: {}'.format(payload))
                            continue

                        task = fxui_worker.createTestTask(testrun, payload)
                        self.logger.info(
                            'Task has been created: {uri}{id}'.format(
                                uri=tc.URI_TASK_INSPECTOR,
                                id=task['status']['taskId'],
                            ))

                    except Exception:
                        # For now simply discard and continue.
                        # Later we might want to implement a queuing mechanism.
                        self.logger.exception(
                            'Cannot create task on Taskcluster')

                else:
                    try:
                        parameters = self.generate_job_parameters(
                            testrun, node, **pulse_properties)

                        if self.display_only:
                            self.logger.info(
                                'Parameters: {}'.format(parameters))
                            continue

                        self.logger.debug('Parameters: {}'.format(parameters))

                        self.jenkins.build_job(job, parameters)

                    except Exception as exc:
                        # For now simply discard and continue.
                        # Later we might want to implement a queuing mechanism.
                        self.logger.exception('Cannot create job: "{}"'.format(
                            exc.message))

            # Give Jenkins a bit of breath to process other threads
            time.sleep(2.5)
Beispiel #3
0
    def process_build(self, **pulse_properties):
        """Check properties and trigger a Jenkins build.

        :param allowed_test: Type of tests which are allowed to be run.
        :param platform: Platform to run the tests on.
        :param product: Name of the product (application).
        :param branch: Name of the branch the build was created off.
        :param locale: Locale of the build.
        :param buildid: ID of the build.
        :param revision: Revision (changeset) of the build.
        :param tags: Build classification tags (e.g. nightly, l10n).
        :param test_packages_url: URL to the test_packages.json file.
        :param version: Version of the build.
        :param status: Build status from Buildbot (build notifications only).
        :param target_buildid: ID of the build after the upgrade (update notification only).
        :param target_version: Version of the build after the upgrade (update notification only).
        :param tree: Releng branch name the build was created off.
        :param raw_json: Raw pulse notification data

        """
        # Known failures from buildbot (http://mzl.la/1hlCYkw)
        buildbot_results = [
            'success', 'warnings', 'failure', 'skipped', 'exception', 'retry'
        ]

        # Bug 1176828 - Repack notifications for beta/release builds do not contain
        # a buildid. So use the timestamp if present as replacement
        if not pulse_properties['buildid'] and 'timestamp' in pulse_properties[
                'raw_json']:
            try:
                d = datetime.strptime(
                    pulse_properties['raw_json']['timestamp'],
                    '%Y-%m-%dT%H:%M:%SZ')
                pulse_properties['buildid'] = d.strftime('%Y%m%d%H%M')
            except:
                pass

        # Print build information to console
        if pulse_properties.get('target_buildid'):
            self.logger.info(
                '{product} {target_version} ({buildid} => {target_buildid},'
                ' {revision}, {locale}, {platform}) [{branch}]'.format(
                    **pulse_properties))
        else:
            self.logger.info(
                '{product} {version} ({buildid}, {revision}, {locale},'
                ' {platform}) [{branch}]'.format(**pulse_properties))

        # Store build information to disk
        basename = '{buildid}_{product}_{locale}_{platform}.log'.format(
            **pulse_properties)
        if pulse_properties.get('target_buildid'):
            basename = '{}_{}'.format(pulse_properties['target_buildid'],
                                      basename)
        filename = os.path.join(self.log_folder, pulse_properties['tree'],
                                basename)

        try:
            if not os.path.exists(filename):
                JSONFile(filename).write(pulse_properties['raw_json'])
        except Exception as e:
            self.logger.warning("Log file could not be written: {}.".format(
                str(e)))

        # Lets keep it after saving the log information so we might be able to
        # manually force-trigger those jobs in case of build failures.
        if pulse_properties.get(
                'status') and pulse_properties['status'] not in (0, 5):
            raise ValueError(
                'Cancel processing due to broken build: {}'.format(
                    buildbot_results[pulse_properties['status']]))

        tree_config = self.config['jenkins']['jobs'][pulse_properties['tree']]
        platform_id = self.get_platform_identifier(
            pulse_properties['platform'])

        # Get some properties now so it hasn't to be done for each individual platform version
        pulse_properties['build_url'] = self.get_installer_url(
            pulse_properties)
        pulse_properties['test_packages_url'] = self.get_test_packages_url(
            pulse_properties)

        # Generate job data and queue up in Jenkins
        for testrun in tree_config['testruns']:
            if testrun not in pulse_properties['allowed_testruns']:
                continue

            # Fire off a build for each supported platform version
            for node in tree_config['nodes'][platform_id]:
                try:
                    job = '{}_{}'.format(pulse_properties['tree'], testrun)
                    parameters = self.generate_job_parameters(
                        testrun, node, **pulse_properties)

                    self.logger.info('Triggering job "{}" on "{}"'.format(
                        job, node))
                    if self.display_only:
                        self.logger.info('Parameters: {}'.format(parameters))
                        continue

                    self.logger.debug('Parameters: {}'.format(parameters))
                    self.jenkins.build_job(job, parameters)
                except Exception:
                    # For now simply discard and continue.
                    # Later we might want to implement a queuing mechanism.
                    self.logger.exception('Cannot create job at "{}"'.format(
                        self.config['jenkins']['url']))

            # Give Jenkins a bit of breath to process other threads
            time.sleep(2.5)