def test_retrigger_action(parameters, input, task_group_id, task_id, task):
    new_task_definition = copy.copy(task)

    # set new created, deadline, and expiry fields
    new_task_definition['created'] = current_json_time()
    new_task_definition['deadline'] = json_time_from_now('1d')
    new_task_definition['expires'] = json_time_from_now('30d')

    # reset artifact expiry
    for artifact in new_task_definition['payload'].get('artifacts', {}).values():
        artifact['expires'] = new_task_definition['expires']

    # don't want to run mozharness tests, want a custom mach command instead
    new_task_definition['payload']['command'] += ['--no-run-tests']

    custom_mach_command = [task['tags']['test-type']]

    # mochitests may specify a flavor
    if new_task_definition['payload']['env'].get('MOCHITEST_FLAVOR'):
        custom_mach_command += [
            '--keep-open=false',
            '-f',
            new_task_definition['payload']['env']['MOCHITEST_FLAVOR']
        ]

    enable_e10s = json.loads(new_task_definition['payload']['env'].get(
        'ENABLE_E10S', 'true'))
    if not enable_e10s:
        custom_mach_command += ['--disable-e10s']

    custom_mach_command += ['--log-tbpl=-',
                            '--log-tbpl-level={}'.format(input['logLevel'])]
    if input.get('runUntilFail'):
        custom_mach_command += ['--run-until-failure']
    if input.get('repeat'):
        custom_mach_command += ['--repeat', str(input['repeat'])]

    # add any custom gecko preferences
    for (key, val) in input.get('preferences', {}).iteritems():
        custom_mach_command += ['--setpref', '{}={}'.format(key, val)]

    custom_mach_command += [input['path']]
    new_task_definition['payload']['env']['CUSTOM_MACH_COMMAND'] = ' '.join(
        custom_mach_command)

    # update environment
    new_task_definition['payload']['env'].update(input.get('environment', {}))

    # tweak the treeherder symbol
    new_task_definition['extra']['treeherder']['symbol'] += '-custom'

    logging.info("New task definition: %s", new_task_definition)

    # actually create the new task
    new_task_id = slugid()
    logger.info("Creating new mochitest task with id %s", new_task_id)
    session = requests.Session()
    create_task(session, new_task_id, 'test-retrigger', new_task_definition)
Example #2
0
def get_action_yml(parameters):
    templates = Templates(os.path.join(GECKO, "taskcluster/taskgraph"))
    action_parameters = parameters.copy()
    action_parameters.update({
        "decision_task_id": "{{decision_task_id}}",
        "task_labels": "{{task_labels}}",
        "from_now": json_time_from_now,
        "now": current_json_time()
    })
    return templates.load('action.yml', action_parameters)
Example #3
0
def get_action_yml(parameters):
    templates = Templates(os.path.join(GECKO, "taskcluster/taskgraph"))
    action_parameters = parameters.copy()
    action_parameters.update(
        {
            "action": "{{action}}",
            "action_args": "{{action_args}}",
            "project": parameters["project"],
            "from_now": json_time_from_now,
            "now": current_json_time(),
        }
    )
    return templates.load("action.yml", action_parameters)
Example #4
0
def _create_task(session, task_id, label, task_def):
    # create the task using 'http://taskcluster/queue', which is proxied to the queue service
    # with credentials appropriate to this job.

    # Resolve timestamps
    now = current_json_time(datetime_format=True)
    task_def = resolve_timestamps(now, task_def)

    logger.debug("Creating task with taskId {} for {}".format(task_id, label))
    res = session.put('http://taskcluster/queue/v1/task/{}'.format(task_id),
                      data=json.dumps(task_def))
    if res.status_code != 200:
        try:
            logger.error(res.json()['message'])
        except:
            logger.error(res.text)
        res.raise_for_status()
Example #5
0
def create_task(session, task_id, label, task_def):
    # create the task using 'http://taskcluster/queue', which is proxied to the queue service
    # with credentials appropriate to this job.

    # Resolve timestamps
    now = current_json_time(datetime_format=True)
    task_def = resolve_timestamps(now, task_def)

    logger.debug("Creating task with taskId {} for {}".format(task_id, label))
    res = session.put('http://taskcluster/queue/v1/task/{}'.format(task_id),
                      data=json.dumps(task_def))
    if res.status_code != 200:
        try:
            logger.error(res.json()['message'])
        except:
            logger.error(res.text)
        res.raise_for_status()
Example #6
0
def get_action_yml(parameters):
    templates = Templates(os.path.join(GECKO, "taskcluster/taskgraph"))
    action_parameters = parameters.copy()

    match = re.match(r'https://(hg.mozilla.org)/(.*?)/?$',
                     action_parameters['head_repository'])
    if not match:
        raise Exception('Unrecognized head_repository')
    repo_scope = 'assume:repo:{}/{}:*'.format(match.group(1), match.group(2))

    action_parameters.update({
        "action": "{{action}}",
        "action_args": "{{action_args}}",
        "repo_scope": repo_scope,
        "from_now": json_time_from_now,
        "now": current_json_time()
    })
    return templates.load('action.yml', action_parameters)
Example #7
0
def get_action_yml(parameters):
    templates = Templates(os.path.join(GECKO, "taskcluster/taskgraph"))
    action_parameters = parameters.copy()

    match = re.match(r'https://(hg.mozilla.org)/(.*?)/?$', action_parameters['head_repository'])
    if not match:
        raise Exception('Unrecognized head_repository')
    repo_scope = 'assume:repo:{}/{}:*'.format(
        match.group(1), match.group(2))

    action_parameters.update({
        "action": "{{action}}",
        "action_args": "{{action_args}}",
        "repo_scope": repo_scope,
        "from_now": json_time_from_now,
        "now": current_json_time()
    })
    return templates.load('action.yml', action_parameters)
Example #8
0
def make_decision_task(params, root, symbol, arguments=[]):
    """Generate a basic decision task, based on the root .taskcluster.yml"""
    with open(os.path.join(root, '.taskcluster.yml'), 'rb') as f:
        taskcluster_yml = yaml.safe_load(f)

    push_info = find_hg_revision_push_info(params['repository_url'],
                                           params['head_rev'])

    # provide a similar JSON-e context to what mozilla-taskcluster provides:
    # https://docs.taskcluster.net/reference/integrations/mozilla-taskcluster/docs/taskcluster-yml
    # but with a different tasks_for and an extra `cron` section
    context = {
        'tasks_for': 'cron',
        'repository': {
            'url': params['repository_url'],
            'project': params['project'],
            'level': params['level'],
        },
        'push': {
            'revision': params['head_rev'],
            # remainder are fake values, but the decision task expects them anyway
            'pushlog_id': push_info['pushid'],
            'pushdate': push_info['pushdate'],
            'owner': 'cron',
        },
        'cron': {
            'task_id': os.environ.get('TASK_ID', '<cron task id>'),
            'job_name': params['job_name'],
            'job_symbol': symbol,
            # args are shell-quoted since they are given to `bash -c`
            'quoted_args': ' '.join(pipes.quote(a) for a in arguments),
        },
        'now': current_json_time(),
        'ownTaskId': slugid.nice(),
    }

    rendered = jsone.render(taskcluster_yml, context)
    if len(rendered['tasks']) != 1:
        raise Exception(
            "Expected .taskcluster.yml to only produce one cron task")
    task = rendered['tasks'][0]

    task_id = task.pop('taskId')
    return (task_id, task)
Example #9
0
def set_expiration(task, relative_datestamp):
    task_def = task['task']
    task_def['expires'] = {'relative-datestamp': relative_datestamp}
    if 'deadline' in task_def:
        now = current_json_time(datetime_format=True)
        timestamp = json_time_from_now(input_str=TRY_EXPIRATION,
                                       now=now,
                                       datetime_format=True)
        deadline = json_time_from_now(input_str=task_def['deadline']['relative-datestamp'],
                                      now=now,
                                      datetime_format=True)
        if deadline > timestamp:
            task_def['deadline']['relative-datestamp'] = relative_datestamp

    try:
        artifacts = task_def['payload']['artifacts']
    except KeyError:
        return

    # for docker-worker, artifacts is a dictionary
    # for generic-worker, artifacts is a list
    # for taskcluster-worker, it will depend on what we do in artifacts plugin
    for artifact in artifacts.values() if hasattr(artifacts, "values") else artifacts:
        artifact['expires']['relative-datestamp'] = relative_datestamp
Example #10
0
def set_expiration(task, relative_datestamp):
    task_def = task['task']
    task_def['expires'] = {'relative-datestamp': relative_datestamp}
    if 'deadline' in task_def:
        now = current_json_time(datetime_format=True)
        timestamp = json_time_from_now(input_str=TRY_EXPIRATION,
                                       now=now,
                                       datetime_format=True)
        deadline = json_time_from_now(input_str=task_def['deadline']['relative-datestamp'],
                                      now=now,
                                      datetime_format=True)
        if deadline > timestamp:
            task_def['deadline']['relative-datestamp'] = relative_datestamp

    try:
        artifacts = task_def['payload']['artifacts']
    except KeyError:
        return

    # for docker-worker, artifacts is a dictionary
    # for generic-worker, artifacts is a list
    # for taskcluster-worker, it will depend on what we do in artifacts plugin
    for artifact in artifacts.values() if hasattr(artifacts, "values") else artifacts:
        artifact['expires']['relative-datestamp'] = relative_datestamp
Example #11
0
def create_task(session, task_id, label, task_def):
    # create the task using 'http://taskcluster/queue', which is proxied to the queue service
    # with credentials appropriate to this job.

    # Resolve timestamps
    now = current_json_time(datetime_format=True)
    task_def = resolve_timestamps(now, task_def)

    if testing:
        json.dump([task_id, task_def], sys.stdout,
                  sort_keys=True, indent=4, separators=(',', ': '))
        # add a newline
        print("")
        return

    logger.debug("Creating task with taskId {} for {}".format(task_id, label))
    res = session.put('http://taskcluster/queue/v1/task/{}'.format(task_id),
                      data=json.dumps(task_def))
    if res.status_code != 200:
        try:
            logger.error(res.json()['message'])
        except Exception:
            logger.error(res.text)
        res.raise_for_status()
Example #12
0
    def load_tasks(self, params):
        root = os.path.abspath(
            os.path.join(self.path, self.config['legacy_path']))

        project = params['project']
        # NOTE: message is ignored here; we always use DEFAULT_TRY, then filter the
        # resulting task graph later
        message = DEFAULT_TRY

        templates = Templates(root)

        job_path = os.path.join(root, 'tasks', 'branches', project,
                                'job_flags.yml')
        job_path = job_path if os.path.exists(job_path) else \
            os.path.join(root, DEFAULT_JOB_PATH)

        jobs = templates.load(job_path, {})

        job_graph, trigger_tests = parse_commit(message, jobs)

        cmdline_interactive = params.get('interactive', False)

        # Default to current time if querying the head rev fails
        pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime())
        vcs_info = query_vcs_info(params['head_repository'],
                                  params['head_rev'])
        changed_files = set()
        if vcs_info:
            pushdate = time.strftime('%Y%m%d%H%M%S',
                                     time.gmtime(vcs_info.pushdate))

            logger.debug('{} commits influencing task scheduling:'.format(
                len(vcs_info.changesets)))
            for c in vcs_info.changesets:
                logger.debug("{cset} {desc}".format(
                    cset=c['node'][0:12],
                    desc=c['desc'].splitlines()[0].encode('ascii', 'ignore')))
                changed_files |= set(c['files'])

        # Template parameters used when expanding the graph
        parameters = dict(
            gaia_info().items() + {
                'index':
                'index',
                'project':
                project,
                'pushlog_id':
                params.get('pushlog_id', 0),
                'docker_image':
                docker_image,
                'base_repository':
                params['base_repository'] or params['head_repository'],
                'head_repository':
                params['head_repository'],
                'head_ref':
                params['head_ref'] or params['head_rev'],
                'head_rev':
                params['head_rev'],
                'pushdate':
                pushdate,
                'pushtime':
                pushdate[8:],
                'year':
                pushdate[0:4],
                'month':
                pushdate[4:6],
                'day':
                pushdate[6:8],
                'owner':
                params['owner'],
                'level':
                params['level'],
                'from_now':
                json_time_from_now,
                'now':
                current_json_time(),
            }.items())

        routes_file = os.path.join(root, 'routes.json')
        with open(routes_file) as f:
            contents = json.load(f)
            json_routes = contents['routes']
            # TODO: Nightly and/or l10n routes

        # Task graph we are generating for taskcluster...
        graph = {
            'tasks': [],
            'scopes': set(),
        }

        for env in TREEHERDER_ROUTES:
            route = format_treeherder_route(TREEHERDER_ROUTES[env],
                                            parameters['project'],
                                            parameters['head_rev'],
                                            parameters['pushlog_id'])
            graph['scopes'].add("queue:route:{}".format(route))

        graph['metadata'] = {
            'source':
            '{repo}file/{rev}/testing/taskcluster/mach_commands.py'.format(
                repo=params['head_repository'], rev=params['head_rev']),
            'owner':
            params['owner'],
            # TODO: Add full mach commands to this example?
            'description':
            'Task graph generated via ./mach taskcluster-graph',
            'name':
            'task graph local'
        }

        # Filter the job graph according to conditions met by this invocation run.
        def should_run(task):
            # Old style build or test task that doesn't define conditions. Always runs.
            if 'when' not in task:
                return True

            when = task['when']

            # If the task defines file patterns and we have a set of changed
            # files to compare against, only run if a file pattern matches one
            # of the changed files.
            file_patterns = when.get('file_patterns', None)
            if file_patterns and changed_files:
                # Always consider changes to the task definition itself
                file_patterns.append(
                    'testing/taskcluster/{task}'.format(task=task['task']))
                for pattern in file_patterns:
                    for path in changed_files:
                        if mozpackmatch(path, pattern):
                            logger.debug(
                                'scheduling {task} because pattern {pattern} '
                                'matches {path}'.format(
                                    task=task['task'],
                                    pattern=pattern,
                                    path=path,
                                ))
                            return True

                # No file patterns matched. Discard task.
                logger.debug(
                    'discarding {task} because no relevant files changed'.
                    format(task=task['task'], pattern=pattern, path=path))
                return False

            return True

        job_graph = filter(should_run, job_graph)

        all_routes = {}

        for build in job_graph:
            logging.debug("loading build task {}".format(build['task']))
            interactive = cmdline_interactive or build["interactive"]
            build_parameters = merge_dicts(parameters,
                                           build['additional-parameters'])
            build_parameters['build_slugid'] = mklabel()
            build_parameters[
                'source'] = '{repo}file/{rev}/testing/taskcluster/{file}'.format(
                    repo=params['head_repository'],
                    rev=params['head_rev'],
                    file=build['task'])
            build_task = templates.load(build['task'], build_parameters)

            # Copy build_* attributes to expose them to post-build tasks
            # as well as json routes and tests
            task_extra = build_task['task']['extra']
            build_parameters['build_name'] = task_extra['build_name']
            build_parameters['build_type'] = task_extra['build_type']
            build_parameters['build_product'] = task_extra['build_product']

            set_interactive_task(build_task, interactive)

            # try builds don't use cache
            if project == "try":
                remove_caches_from_task(build_task)
                set_expiration(build_task, json_time_from_now(TRY_EXPIRATION))

            decorate_task_treeherder_routes(build_task['task'],
                                            build_parameters['project'],
                                            build_parameters['head_rev'],
                                            build_parameters['pushlog_id'])
            decorate_task_json_routes(build_task['task'], json_routes,
                                      build_parameters)

            # Ensure each build graph is valid after construction.
            validate_build_task(build_task)
            attributes = build_task['attributes'] = {
                'kind': 'legacy',
                'legacy_kind': 'build'
            }
            if 'build_name' in build:
                attributes['build_platform'] = build['build_name']
            if 'build_type' in task_extra:
                attributes['build_type'] = {
                    'dbg': 'debug'
                }.get(task_extra['build_type'], task_extra['build_type'])
            if build.get('is_job'):
                attributes['job'] = build['build_name']
                attributes['legacy_kind'] = 'job'
            graph['tasks'].append(build_task)

            for location in build_task['task']['extra'].get('locations', {}):
                build_parameters['{}_location'.format(location)] = build_task[
                    'task']['extra']['locations'][location]

            for url in build_task['task']['extra'].get('url', {}):
                build_parameters['{}_url'.format(url)] = \
                    build_task['task']['extra']['url'][url]

            define_task = DEFINE_TASK.format(build_task['task']['workerType'])

            for route in build_task['task'].get('routes', []):
                if route.startswith('index.gecko.v2') and route in all_routes:
                    raise Exception(
                        "Error: route '%s' is in use by multiple tasks: '%s' and '%s'"
                        % (
                            route,
                            build_task['task']['metadata']['name'],
                            all_routes[route],
                        ))
                all_routes[route] = build_task['task']['metadata']['name']

            graph['scopes'].add(define_task)
            graph['scopes'] |= set(build_task['task'].get('scopes', []))
            route_scopes = map(lambda route: 'queue:route:' + route,
                               build_task['task'].get('routes', []))
            graph['scopes'] |= set(route_scopes)

            # Treeherder symbol configuration for the graph required for each
            # build so tests know which platform they belong to.
            build_treeherder_config = build_task['task']['extra']['treeherder']

            if 'machine' not in build_treeherder_config:
                message = '({}), extra.treeherder.machine required for all builds'
                raise ValueError(message.format(build['task']))

            if 'build' not in build_treeherder_config:
                build_treeherder_config['build'] = \
                    build_treeherder_config['machine']

            if 'collection' not in build_treeherder_config:
                build_treeherder_config['collection'] = {'opt': True}

            if len(build_treeherder_config['collection'].keys()) != 1:
                message = '({}), extra.treeherder.collection must contain one type'
                raise ValueError(message.fomrat(build['task']))

            for post_build in build['post-build']:
                # copy over the old parameters to update the template
                # TODO additional-parameters is currently not an option, only
                # enabled for build tasks
                post_parameters = merge_dicts(
                    build_parameters,
                    post_build.get('additional-parameters', {}))
                post_task = configure_dependent_task(post_build['task'],
                                                     post_parameters,
                                                     mklabel(), templates,
                                                     build_treeherder_config)
                set_interactive_task(post_task, interactive)

                if project == "try":
                    set_expiration(post_task,
                                   json_time_from_now(TRY_EXPIRATION))

                post_task['attributes'] = attributes.copy()
                post_task['attributes']['legacy_kind'] = 'post_build'
                post_task['attributes']['post_build'] = post_build['job_flag']
                graph['tasks'].append(post_task)

            for test in build['dependents']:
                test = test['allowed_build_tasks'][build['task']]
                # TODO additional-parameters is currently not an option, only
                # enabled for build tasks
                test_parameters = merge_dicts(
                    build_parameters, test.get('additional-parameters', {}))
                test_parameters = copy.copy(build_parameters)

                test_definition = templates.load(test['task'], {})['task']
                chunk_config = test_definition['extra'].get('chunks', {})

                # Allow branch configs to override task level chunking...
                if 'chunks' in test:
                    chunk_config['total'] = test['chunks']

                chunked = 'total' in chunk_config
                if chunked:
                    test_parameters['total_chunks'] = chunk_config['total']

                if 'suite' in test_definition['extra']:
                    suite_config = test_definition['extra']['suite']
                    test_parameters['suite'] = suite_config['name']
                    test_parameters['flavor'] = suite_config.get('flavor', '')

                for chunk in range(1, chunk_config.get('total', 1) + 1):
                    if 'only_chunks' in test and chunked and \
                            chunk not in test['only_chunks']:
                        continue

                    if chunked:
                        test_parameters['chunk'] = chunk
                    test_task = configure_dependent_task(
                        test['task'], test_parameters, mklabel(), templates,
                        build_treeherder_config)
                    set_interactive_task(test_task, interactive)

                    decorate_task_treeherder_routes(
                        test_task['task'], test_parameters['project'],
                        test_parameters['head_rev'],
                        test_parameters['pushlog_id'])

                    if project == "try":
                        set_expiration(test_task,
                                       json_time_from_now(TRY_EXPIRATION))

                    test_task['attributes'] = attributes.copy()
                    test_task['attributes']['legacy_kind'] = 'unittest'
                    test_task['attributes']['test_platform'] = attributes[
                        'build_platform']
                    test_task['attributes']['unittest_try_name'] = test[
                        'unittest_try_name']
                    for param, attr in [('suite', 'unittest_suite'),
                                        ('flavor', 'unittest_flavor'),
                                        ('chunk', 'test_chunk')]:
                        if param in test_parameters:
                            test_task['attributes'][attr] = str(
                                test_parameters[param])

                    # This will schedule test jobs N times
                    for i in range(0, trigger_tests):
                        graph['tasks'].append(test_task)
                        # If we're scheduling more tasks each have to be unique
                        test_task = copy.deepcopy(test_task)
                        test_task['taskId'] = mklabel()

                    define_task = DEFINE_TASK.format(
                        test_task['task']['workerType'])

                    graph['scopes'].add(define_task)
                    graph['scopes'] |= set(test_task['task'].get('scopes', []))

        graph['scopes'] = sorted(graph['scopes'])

        # save the graph for later, when taskgraph asks for additional information
        # such as dependencies
        self.graph = graph
        self.tasks_by_label = {t['taskId']: t for t in self.graph['tasks']}

        # Convert to a dictionary of tasks.  The process above has invented a
        # taskId for each task, and we use those as the *labels* for the tasks;
        # taskgraph will later assign them new taskIds.
        return [
            Task(self, t['taskId'], task=t['task'], attributes=t['attributes'])
            for t in self.graph['tasks']
        ]
Example #13
0
    def load_tasks(cls, kind, path, config, params, loaded_tasks):
        # TODO: make this match the pushdate (get it from a parameter rather than vcs)
        pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime())

        parameters = {
            'pushlog_id': params.get('pushlog_id', 0),
            'pushdate': pushdate,
            'pushtime': pushdate[8:],
            'year': pushdate[0:4],
            'month': pushdate[4:6],
            'day': pushdate[6:8],
            'project': params['project'],
            'docker_image': docker_image,
            'base_repository': params['base_repository'] or params['head_repository'],
            'head_repository': params['head_repository'],
            'head_ref': params['head_ref'] or params['head_rev'],
            'head_rev': params['head_rev'],
            'owner': params['owner'],
            'level': params['level'],
            'from_now': json_time_from_now,
            'now': current_json_time(),
            'source': '{repo}file/{rev}/testing/taskcluster/tasks/image.yml'
                      .format(repo=params['head_repository'], rev=params['head_rev']),
        }

        tasks = []
        templates = Templates(path)
        for image_name in config['images']:
            context_path = os.path.join('testing', 'docker', image_name)
            context_hash = generate_context_hash(context_path)

            image_parameters = dict(parameters)
            image_parameters['context_hash'] = context_hash
            image_parameters['context_path'] = context_path
            image_parameters['artifact_path'] = 'public/image.tar'
            image_parameters['image_name'] = image_name

            image_artifact_path = \
                "public/decision_task/image_contexts/{}/context.tar.gz".format(image_name)
            if os.environ.get('TASK_ID'):
                destination = os.path.join(
                    os.environ['HOME'],
                    "artifacts/decision_task/image_contexts/{}/context.tar.gz".format(image_name))
                image_parameters['context_url'] = ARTIFACT_URL.format(
                    os.environ['TASK_ID'], image_artifact_path)
                cls.create_context_tar(context_path, destination, image_name)
            else:
                # skip context generation since this isn't a decision task
                # TODO: generate context tarballs using subdirectory clones in
                # the image-building task so we don't have to worry about this.
                image_parameters['context_url'] = 'file:///tmp/' + image_artifact_path

            image_task = templates.load('image.yml', image_parameters)

            attributes = {'image_name': image_name}

            # As an optimization, if the context hash exists for mozilla-central, that image
            # task ID will be used.  The reasoning behind this is that eventually everything ends
            # up on mozilla-central at some point if most tasks use this as a common image
            # for a given context hash, a worker within Taskcluster does not need to contain
            # the same image per branch.
            index_paths = ['docker.images.v1.{}.{}.hash.{}'.format(
                                project, image_name, context_hash)
                           for project in ['mozilla-central', params['project']]]

            tasks.append(cls(kind, 'build-docker-image-' + image_name,
                             task=image_task['task'], attributes=attributes,
                             index_paths=index_paths))

        return tasks
Example #14
0
    def load_tasks(self, params):
        # TODO: make this match the pushdate (get it from a parameter rather than vcs)
        pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime())

        parameters = {
            'pushlog_id':
            params.get('pushlog_id', 0),
            'pushdate':
            pushdate,
            'pushtime':
            pushdate[8:],
            'year':
            pushdate[0:4],
            'month':
            pushdate[4:6],
            'day':
            pushdate[6:8],
            'project':
            params['project'],
            'docker_image':
            docker_image,
            'base_repository':
            params['base_repository'] or params['head_repository'],
            'head_repository':
            params['head_repository'],
            'head_ref':
            params['head_ref'] or params['head_rev'],
            'head_rev':
            params['head_rev'],
            'owner':
            params['owner'],
            'level':
            params['level'],
            'from_now':
            json_time_from_now,
            'now':
            current_json_time(),
            'source':
            '{repo}file/{rev}/testing/taskcluster/tasks/image.yml'.format(
                repo=params['head_repository'], rev=params['head_rev']),
        }

        tasks = []
        templates = Templates(self.path)
        for image_name in self.config['images']:
            context_path = os.path.join('testing', 'docker', image_name)
            context_hash = generate_context_hash(context_path)

            image_parameters = dict(parameters)
            image_parameters['context_hash'] = context_hash
            image_parameters['context_path'] = context_path
            image_parameters['artifact_path'] = 'public/image.tar'
            image_parameters['image_name'] = image_name

            image_artifact_path = \
                "public/decision_task/image_contexts/{}/context.tar.gz".format(image_name)
            if os.environ.get('TASK_ID'):
                destination = os.path.join(
                    os.environ['HOME'],
                    "artifacts/decision_task/image_contexts/{}/context.tar.gz".
                    format(image_name))
                image_parameters['context_url'] = ARTIFACT_URL.format(
                    os.environ['TASK_ID'], image_artifact_path)
                self.create_context_tar(context_path, destination, image_name)
            else:
                # skip context generation since this isn't a decision task
                # TODO: generate context tarballs using subdirectory clones in
                # the image-building task so we don't have to worry about this.
                image_parameters[
                    'context_url'] = 'file:///tmp/' + image_artifact_path

            image_task = templates.load('image.yml', image_parameters)

            attributes = {
                'kind': self.name,
                'image_name': image_name,
            }

            # As an optimization, if the context hash exists for mozilla-central, that image
            # task ID will be used.  The reasoning behind this is that eventually everything ends
            # up on mozilla-central at some point if most tasks use this as a common image
            # for a given context hash, a worker within Taskcluster does not need to contain
            # the same image per branch.
            index_paths = [
                'docker.images.v1.{}.{}.hash.{}'.format(
                    project, image_name, context_hash)
                for project in ['mozilla-central', params['project']]
            ]

            tasks.append(
                Task(self,
                     'build-docker-image-' + image_name,
                     task=image_task['task'],
                     attributes=attributes,
                     index_paths=index_paths))

        return tasks
Example #15
0
    def load_tasks(cls, kind, path, config, params, loaded_tasks):
        root = os.path.abspath(os.path.join(path, config['legacy_path']))

        project = params['project']
        # NOTE: message is ignored here; we always use DEFAULT_TRY, then filter the
        # resulting task graph later
        message = DEFAULT_TRY

        templates = Templates(root)

        job_path = os.path.join(root, 'tasks', 'branches', project, 'job_flags.yml')
        job_path = job_path if os.path.exists(job_path) else \
            os.path.join(root, DEFAULT_JOB_PATH)

        jobs = templates.load(job_path, {})

        job_graph, trigger_tests = parse_commit(message, jobs)

        cmdline_interactive = params.get('interactive', False)

        # Default to current time if querying the head rev fails
        push_epoch = int(time.time())
        vcs_info = query_vcs_info(params['head_repository'], params['head_rev'])
        changed_files = set()
        if vcs_info:
            push_epoch = vcs_info.pushdate

            logger.debug(
                '{} commits influencing task scheduling:'.format(len(vcs_info.changesets)))
            for c in vcs_info.changesets:
                logger.debug("{cset} {desc}".format(
                    cset=c['node'][0:12],
                    desc=c['desc'].splitlines()[0].encode('ascii', 'ignore')))
                changed_files |= set(c['files'])

        pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(push_epoch))

        # Template parameters used when expanding the graph
        parameters = dict(gaia_info().items() + {
            'index': 'index',
            'project': project,
            'pushlog_id': params.get('pushlog_id', 0),
            'docker_image': docker_image,
            'base_repository': params['base_repository'] or
            params['head_repository'],
            'head_repository': params['head_repository'],
            'head_ref': params['head_ref'] or params['head_rev'],
            'head_rev': params['head_rev'],
            'pushdate': pushdate,
            'pushtime': pushdate[8:],
            'year': pushdate[0:4],
            'month': pushdate[4:6],
            'day': pushdate[6:8],
            'rank': push_epoch,
            'owner': params['owner'],
            'level': params['level'],
            'from_now': json_time_from_now,
            'now': current_json_time(),
        }.items())

        routes_file = os.path.join(root, 'routes.json')
        with open(routes_file) as f:
            contents = json.load(f)
            json_routes = contents['routes']
            # TODO: Nightly and/or l10n routes

        # Task graph we are generating for taskcluster...
        graph = {
            'tasks': [],
            'scopes': set(),
        }

        for env in TREEHERDER_ROUTES:
            route = format_treeherder_route(TREEHERDER_ROUTES[env],
                                            parameters['project'],
                                            parameters['head_rev'],
                                            parameters['pushlog_id'])
            graph['scopes'].add("queue:route:{}".format(route))

        graph['metadata'] = {
            'source': '{repo}file/{rev}/testing/taskcluster/mach_commands.py'.format(
                repo=params['head_repository'], rev=params['head_rev']),
            'owner': params['owner'],
            # TODO: Add full mach commands to this example?
            'description': 'Task graph generated via ./mach taskcluster-graph',
            'name': 'task graph local'
        }

        # Filter the job graph according to conditions met by this invocation run.
        def should_run(task):
            # Old style build or test task that doesn't define conditions. Always runs.
            if 'when' not in task:
                return True

            when = task['when']

            # If the task defines file patterns and we have a set of changed
            # files to compare against, only run if a file pattern matches one
            # of the changed files.
            file_patterns = when.get('file_patterns', None)
            if file_patterns and changed_files:
                # Always consider changes to the task definition itself
                file_patterns.append('testing/taskcluster/{task}'.format(task=task['task']))
                for pattern in file_patterns:
                    for path in changed_files:
                        if mozpackmatch(path, pattern):
                            logger.debug('scheduling {task} because pattern {pattern} '
                                         'matches {path}'.format(
                                             task=task['task'],
                                             pattern=pattern,
                                             path=path,
                                         ))
                            return True

                # No file patterns matched. Discard task.
                logger.debug('discarding {task} because no relevant files changed'.format(
                    task=task['task'],
                    pattern=pattern,
                    path=path))
                return False

            return True

        job_graph = filter(should_run, job_graph)

        all_routes = {}

        for build in job_graph:
            logging.debug("loading build task {}".format(build['task']))
            interactive = cmdline_interactive or build["interactive"]
            build_parameters = merge_dicts(parameters, build['additional-parameters'])
            build_parameters['build_slugid'] = mklabel()
            build_parameters['source'] = '{repo}file/{rev}/testing/taskcluster/{file}'.format(
                repo=params['head_repository'], rev=params['head_rev'], file=build['task'])
            build_task = templates.load(build['task'], build_parameters)

            # Copy build_* attributes to expose them to post-build tasks
            # as well as json routes and tests
            task_extra = build_task['task']['extra']
            build_parameters['build_name'] = task_extra['build_name']
            build_parameters['build_type'] = task_extra['build_type']
            build_parameters['build_product'] = task_extra['build_product']

            if 'treeherder' in task_extra:
                tier = task_extra['treeherder'].get('tier', 1)
                if tier != 1:
                    # Only tier 1 jobs use the build time as rank. Everything
                    # else gets rank 0 until it is promoted to tier 1.
                    task_extra['index']['rank'] = 0

            set_interactive_task(build_task, interactive)

            # try builds don't use cache
            if project == "try":
                remove_caches_from_task(build_task)
                set_expiration(build_task, json_time_from_now(TRY_EXPIRATION))

            decorate_task_treeherder_routes(build_task['task'],
                                            build_parameters['project'],
                                            build_parameters['head_rev'],
                                            build_parameters['pushlog_id'])
            decorate_task_json_routes(build_task['task'],
                                      json_routes,
                                      build_parameters)

            # Ensure each build graph is valid after construction.
            validate_build_task(build_task)
            attributes = build_task['attributes'] = {'kind': 'legacy', 'legacy_kind': 'build'}
            if 'build_name' in build:
                attributes['build_platform'] = build['build_name']
            if 'build_type' in task_extra:
                attributes['build_type'] = {'dbg': 'debug'}.get(task_extra['build_type'],
                                                                task_extra['build_type'])
            if build.get('is_job'):
                attributes['job'] = build['build_name']
                attributes['legacy_kind'] = 'job'
            graph['tasks'].append(build_task)

            for location in build_task['task']['extra'].get('locations', {}):
                build_parameters['{}_location'.format(location)] = \
                    build_task['task']['extra']['locations'][location]

            for url in build_task['task']['extra'].get('url', {}):
                build_parameters['{}_url'.format(url)] = \
                    build_task['task']['extra']['url'][url]

            define_task = DEFINE_TASK.format(build_task['task']['workerType'])

            for route in build_task['task'].get('routes', []):
                if route.startswith('index.gecko.v2') and route in all_routes:
                    raise Exception(
                        "Error: route '%s' is in use by multiple tasks: '%s' and '%s'" % (
                            route,
                            build_task['task']['metadata']['name'],
                            all_routes[route],
                        ))
                all_routes[route] = build_task['task']['metadata']['name']

            graph['scopes'].add(define_task)
            graph['scopes'] |= set(build_task['task'].get('scopes', []))
            route_scopes = map(
                lambda route: 'queue:route:' + route, build_task['task'].get('routes', [])
            )
            graph['scopes'] |= set(route_scopes)

            # Treeherder symbol configuration for the graph required for each
            # build so tests know which platform they belong to.
            build_treeherder_config = build_task['task']['extra']['treeherder']

            if 'machine' not in build_treeherder_config:
                message = '({}), extra.treeherder.machine required for all builds'
                raise ValueError(message.format(build['task']))

            if 'build' not in build_treeherder_config:
                build_treeherder_config['build'] = \
                    build_treeherder_config['machine']

            if 'collection' not in build_treeherder_config:
                build_treeherder_config['collection'] = {'opt': True}

            if len(build_treeherder_config['collection'].keys()) != 1:
                message = '({}), extra.treeherder.collection must contain one type'
                raise ValueError(message.fomrat(build['task']))

            for post_build in build['post-build']:
                # copy over the old parameters to update the template
                # TODO additional-parameters is currently not an option, only
                # enabled for build tasks
                post_parameters = merge_dicts(build_parameters,
                                              post_build.get('additional-parameters', {}))
                post_task = configure_dependent_task(post_build['task'],
                                                     post_parameters,
                                                     mklabel(),
                                                     templates,
                                                     build_treeherder_config)
                set_interactive_task(post_task, interactive)

                if project == "try":
                    set_expiration(post_task, json_time_from_now(TRY_EXPIRATION))

                post_task['attributes'] = attributes.copy()
                post_task['attributes']['legacy_kind'] = 'post_build'
                post_task['attributes']['post_build'] = post_build['job_flag']
                graph['tasks'].append(post_task)

            for test in build['dependents']:
                test = test['allowed_build_tasks'][build['task']]
                # TODO additional-parameters is currently not an option, only
                # enabled for build tasks
                test_parameters = merge_dicts(build_parameters,
                                              test.get('additional-parameters', {}))
                test_parameters = copy.copy(build_parameters)

                test_definition = templates.load(test['task'], {})['task']
                chunk_config = test_definition['extra'].get('chunks', {})

                # Allow branch configs to override task level chunking...
                if 'chunks' in test:
                    chunk_config['total'] = test['chunks']

                chunked = 'total' in chunk_config
                if chunked:
                    test_parameters['total_chunks'] = chunk_config['total']

                if 'suite' in test_definition['extra']:
                    suite_config = test_definition['extra']['suite']
                    test_parameters['suite'] = suite_config['name']
                    test_parameters['flavor'] = suite_config.get('flavor', '')

                for chunk in range(1, chunk_config.get('total', 1) + 1):
                    if 'only_chunks' in test and chunked and \
                            chunk not in test['only_chunks']:
                        continue

                    if chunked:
                        test_parameters['chunk'] = chunk
                    test_task = configure_dependent_task(test['task'],
                                                         test_parameters,
                                                         mklabel(),
                                                         templates,
                                                         build_treeherder_config)
                    set_interactive_task(test_task, interactive)

                    decorate_task_treeherder_routes(test_task['task'],
                                                    test_parameters['project'],
                                                    test_parameters['head_rev'],
                                                    test_parameters['pushlog_id'])

                    if project == "try":
                        set_expiration(test_task, json_time_from_now(TRY_EXPIRATION))

                    test_task['attributes'] = attributes.copy()
                    test_task['attributes']['legacy_kind'] = 'unittest'
                    test_task['attributes']['test_platform'] = attributes['build_platform']
                    test_task['attributes']['unittest_try_name'] = test['unittest_try_name']
                    for param, attr in [
                            ('suite', 'unittest_suite'),
                            ('flavor', 'unittest_flavor'),
                            ('chunk', 'test_chunk')]:
                        if param in test_parameters:
                            test_task['attributes'][attr] = str(test_parameters[param])

                    # This will schedule test jobs N times
                    for i in range(0, trigger_tests):
                        graph['tasks'].append(test_task)
                        # If we're scheduling more tasks each have to be unique
                        test_task = copy.deepcopy(test_task)
                        test_task['taskId'] = mklabel()

                    define_task = DEFINE_TASK.format(
                        test_task['task']['workerType']
                    )

                    graph['scopes'].add(define_task)
                    graph['scopes'] |= set(test_task['task'].get('scopes', []))

        graph['scopes'] = sorted(graph['scopes'])

        # Convert to a dictionary of tasks.  The process above has invented a
        # taskId for each task, and we use those as the *labels* for the tasks;
        # taskgraph will later assign them new taskIds.
        return [
            cls(kind, t['taskId'], task=t['task'], attributes=t['attributes'], task_dict=t)
            for t in graph['tasks']
        ]