def RunSteps(api, properties): try: _validate_props(properties) except ValueError as exc: raise recipe_api.InfraFailure('Bad input properties: %s' % exc) # Checkout either the committed code or a pending CL, depending on the mode. # This also calculates metadata (labels, tags) to apply to images built from # this code. if properties.mode == PROPERTIES.MODE_CI: co, meta = _checkout_ci(api, properties.project) elif properties.mode == PROPERTIES.MODE_CL: co, meta = _checkout_cl(api, properties.project) else: raise recipe_api.InfraFailure( '%s is not implemented yet' % PROPERTIES.Mode.Name(properties.mode)) co.gclient_runhooks() # Discover what *.yaml manifests (full paths to them) we need to build. manifests = _discover_manifests(api, co.path, properties.manifests) if not manifests: # pragma: no cover raise recipe_api.InfraFailure('Found no manifests to build') with co.go_env(): # Use 'cloudbuildhelper' that comes with the infra checkout (it's in PATH), # to make sure builders use same version as developers. api.cloudbuildhelper.command = 'cloudbuildhelper' # Report the exact version we picked up from the infra checkout. api.cloudbuildhelper.report_version() # Build, tag and upload corresponding images. fails = [] for m in manifests: # TODO(vadimsh): Run this in parallel when it's possible. try: api.cloudbuildhelper.build( manifest=m, canonical_tag=meta.canonical_tag, build_id=api.buildbucket.build_url(), infra=properties.infra, labels=meta.labels, tags=meta.tags, ) except api.step.StepFailure: fails.append(api.path.basename(m)) if fails: raise recipe_api.StepFailure('Failed to build: %s' % ', '.join(fails))
def _isolated_script_collect_step(self, task, **kwargs): step_test_data = kwargs.pop('step_test_data', None) if not step_test_data: step_test_data = self.m.test_utils.test_api.canned_isolated_script_output( passing=True, is_win=self.m.platform.is_win, swarming=True) args = self.get_collect_cmd_args(task) args.extend(['--task-output-dir', self.m.raw_io.output_dir()]) try: self.m.python( name=self._get_step_name('', task), script=self.m.swarming_client.path.join('swarming.py'), args=args, step_test_data=lambda: step_test_data, **kwargs) finally: # Regardless of the outcome of the test (pass or fail), we try to parse # the results. If any error occurs while parsing results, then we set them # to None, which caller should treat as invalid results. # Note that try-except block below will not mask the # recipe_api.StepFailure exception from the collect step above. Instead # it is being allowed to propagate after the results have been parsed. try: step_result = self.m.step.active_result outdir_json = self.m.json.dumps(step_result.raw_io.output_dir, indent=2) step_result.presentation.logs[ 'outdir_json'] = outdir_json.splitlines() # Check if it's an internal failure. summary = self.m.json.loads( step_result.raw_io.output_dir['summary.json']) if any(shard['internal_failure'] for shard in summary['shards']): raise recipe_api.InfraFailure('Internal swarming failure.') # Always show the shards' links in the collect step. (It looks # like show_isolated_out_in_collect_step is false by default # in recipe runs.) links = step_result.presentation.links for index in xrange(task.shards): url = task.get_shard_view_url(index) if url: links['shard #%d' % index] = url step_result.isolated_script_results = \ self._merge_isolated_script_shards(task, step_result) self._display_pending(summary, step_result.presentation) except Exception as e: self.m.step.active_result.presentation.logs[ 'no_results_exc'] = [str(e)] self.m.step.active_result.isolated_script_results = None
def _isolated_script_collect_step(self, task, **kwargs): step_test_data = kwargs.pop('step_test_data', None) if not step_test_data: step_test_data = self.m.test_utils.test_api.canned_isolated_script_output( passing=True, is_win=self.m.platform.is_win, swarming=True) args = self.get_collect_cmd_args(task) args.extend(['--task-output-dir', self.m.raw_io.output_dir()]) try: self.m.python( name=self._get_step_name('', task), script=self.m.swarming_client.path.join('swarming.py'), args=args, step_test_data=lambda: step_test_data, **kwargs) finally: # Regardless of the outcome of the test (pass or fail), we try to parse # the results. If any error occurs while parsing results, then we set them # to None, which caller should treat as invalid results. # Note that try-except block below will not mask the # recipe_api.StepFailure exception from the collect step above. Instead # it is being allowed to propagate after the results have been parsed. try: step_result = self.m.step.active_result outdir_json = self.m.json.dumps(step_result.raw_io.output_dir, indent=2) step_result.presentation.logs[ 'outdir_json'] = outdir_json.splitlines() # Check if it's an internal failure. summary = self.m.json.loads( step_result.raw_io.output_dir['summary.json']) if any(shard['internal_failure'] for shard in summary['shards']): raise recipe_api.InfraFailure('Internal swarming failure.') # TODO(nednguyen, kbr): Combine isolated script results from multiple # shards rather than assuming that there is always just one shard. assert len(summary['shards']) == 1 results_raw = step_result.raw_io.output_dir[self.m.path.join( '0', 'output.json')] step_result.isolated_script_results = self.m.json.loads( results_raw) self._display_pending(summary, step_result.presentation) except Exception as e: self.m.step.active_result.presentation.logs[ 'no_results_exc'] = [str(e)] self.m.step.active_result.isolated_script_results = None
def _handle_summary_json(self, task, summary, step_result): # We store this now, and add links to all shards first, before failing the # build. Format is tuple of (error message, shard that failed) infra_failures = [] links = step_result.presentation.links for index, shard in enumerate(summary['shards']): url = task.get_shard_view_url(index) display_text = 'shard #%d' % index if not shard or shard.get('internal_failure'): # pragma: no cover display_text = ( 'shard #%d had an internal swarming failure' % index) infra_failures.append((index, 'Internal swarming failure')) elif self._is_expired(shard): display_text = ( 'shard #%d expired, not enough capacity' % index) infra_failures.append(( index, 'There isn\'t enough capacity to run your test')) elif self._is_timed_out(shard): display_text = ( 'shard #%d timed out, took too much time to complete' % index) elif self._get_exit_code(shard) != '0': # pragma: no cover display_text = 'shard #%d (failed)' % index if self.show_isolated_out_in_collect_step: isolated_out = shard.get('isolated_out') if isolated_out: link_name = 'shard #%d isolated out' % index links[link_name] = isolated_out['view_url'] if url and self.show_shards_in_collect_step: links[display_text] = url self._display_pending(summary, step_result.presentation) if infra_failures: template = 'Shard #%s failed: %s' # Done so that raising an InfraFailure doesn't cause an error. # TODO(martiniss): Remove this hack. Requires recipe engine change step_result._retcode = 2 step_result.presentation.status = self.m.step.EXCEPTION raise recipe_api.InfraFailure( '\n'.join(template % f for f in infra_failures), result=step_result)
def _checkout_ci(api, project): """Checks out some committed revision (based on Buildbucket properties). Args: api: recipes API. project: PROPERTIES.Project enum. Returns: (infra_checkout.Checkout, Metadata). """ conf, internal, repo_url = { PROPERTIES.PROJECT_INFRA: ( 'infra', False, 'https://chromium.googlesource.com/infra/infra', ), PROPERTIES.PROJECT_INFRA_INTERNAL: ( 'infra_internal', True, 'https://chrome-internal.googlesource.com/infra/infra_internal', ), }[project] co = api.infra_checkout.checkout(gclient_config_name=conf, internal=internal) rev = co.bot_update_step.presentation.properties['got_revision'] cp = co.bot_update_step.presentation.properties['got_revision_cp'] cp_ref, cp_num = api.commit_position.parse(cp) if cp_ref != 'refs/heads/master': # pragma: no cover raise recipe_api.InfraFailure( 'Only refs/heads/master commits are supporte for now, got %r' % cp_ref) return co, Metadata( canonical_tag='ci-%s-%d-%s' % ( _date(api), cp_num, rev[:7], ), labels={ 'org.opencontainers.image.source': repo_url, 'org.opencontainers.image.revision': rev, }, tags=['latest'])
def get_fuchsia_version(self, flutter_bin): """Get the Fuchsia SDK version from the given Flutter SDK. Args: flutter_bin: Path to Flutter bin with internal/fuchsia-linux.version. Returns: String of the Fuchsia SDK version to pull artifacts from GCP. """ # Flutter SDK only stores the CIPD version, so CIPD must be queried to # find the SDK version tag for this ref. version_path = flutter_bin.join('internal', 'fuchsia-linux.version') version = self.m.file.read_text('Read fuchsia cipd version', version_path) fuchsia_cipd = self.m.cipd.describe(FUCHSIA_SDK_CIPD, version=version) # There are multiple tags in a Fuchsia SDK CIPD description requiring # a search through the tags tuple for the version tag. for tag in fuchsia_cipd.tags: if 'version:' in tag.tag: return tag.tag.replace('version:', '') raise recipe_api.InfraFailure('No version tag on Fuchsia SDK CIPD ref')
def _merge_isolated_script_shards(self, task, step_result): # This code is unfortunately specialized to the "simplified" # JSON format that used to be the standard for recipes. The # isolated scripts should be changed to use the now-standard # Chromium JSON test results format: # https://www.chromium.org/developers/the-json-test-results-format # . Note that gtests, above, don't seem to conform to this # format yet, so it didn't seem like a good prerequisite to # switch the isolated tests over when adding sharding support. # # These are the only keys we pay attention to in the output JSON. merged_results = { 'successes': [], 'failures': [], 'valid': True, } for i in xrange(task.shards): path = self.m.path.join(str(i), 'output.json') if path not in step_result.raw_io.output_dir: raise Exception('no results from shard #%d' % i) results_raw = step_result.raw_io.output_dir[path] try: results_json = self.m.json.loads(results_raw) except Exception as e: raise Exception('error decoding JSON results from shard #%d' % i) for key in merged_results: if key in results_json: if isinstance(merged_results[key], list): merged_results[key].extend(results_json[key]) elif isinstance(merged_results[key], bool): merged_results[ key] = merged_results[key] and results_json[key] else: raise recipe_api.InfraFailure( 'Unknown key type ' + type(merged_results[key]) + ' when handling key ' + key + '.') # pragma: no cover return merged_results
def count_objects(self, previous_result=None, can_fail_build=False, **kwargs): """Returns `git count-objects` result as a dict. Args: previous_result (dict): the result of previous count_objects call. If passed, delta is reported in the log and step text. can_fail_build (bool): if True, may fail the build and/or raise an exception. Defaults to False. Returns: A dict of count-object values, or None if count-object run failed. """ if previous_result: assert isinstance(previous_result, dict) assert all(isinstance(v, long) for v in previous_result.itervalues()) assert 'size' in previous_result assert 'size-pack' in previous_result step_result = None try: step_result = self( 'count-objects', '-v', stdout=self.m.raw_io.output(), can_fail_build=can_fail_build, **kwargs) if not step_result.stdout: return None result = {} for line in step_result.stdout.splitlines(): name, value = line.split(':', 1) result[name] = long(value.strip()) def results_to_text(results): return [' %s: %s' % (k, v) for k, v in results.iteritems()] step_result.presentation.logs['result'] = results_to_text(result) if previous_result: delta = { key: value - previous_result[key] for key, value in result.iteritems() if key in previous_result} step_result.presentation.logs['delta'] = ( ['before:'] + results_to_text(previous_result) + ['', 'after:'] + results_to_text(result) + ['', 'delta:'] + results_to_text(delta) ) size_delta = ( result['size'] + result['size-pack'] - previous_result['size'] - previous_result['size-pack']) # size_delta is in KiB. step_result.presentation.step_text = ( 'size delta: %+.2f MiB' % (size_delta / 1024.0)) return result except Exception as ex: if step_result: step_result.presentation.logs['exception'] = ['%r' % ex] step_result.presentation.status = self.m.step.WARNING if can_fail_build: raise recipe_api.InfraFailure('count-objects failed: %s' % ex) return None
def build(self, manifest, canonical_tag=None, build_id=None, infra=None, labels=None, tags=None, step_test_image=None): """Calls `cloudbuildhelper build <manifest>` interpreting the result. Args: * manifest (Path) - path to YAML file with definition of what to build. * canonical_tag (str) - tag to push the image to if we built a new image. * build_id (str) - identifier of the CI build to put into metadata. * infra (str) - what section to pick from 'infra' field in the YAML. * labels ({str: str}) - labels to attach to the docker image. * tags ([str]) - tags to unconditionally push the image to. * step_test_image (Image) - image to produce in training mode. Returns: Image instance or NotUploadImage if the YAML doesn't specify a registry. Raises: StepFailure on failures. """ name, _ = self.m.path.splitext(self.m.path.basename(manifest)) cmd = [self.command, 'build', manifest] if canonical_tag: cmd += ['-canonical-tag', canonical_tag] if build_id: cmd += ['-build-id', build_id] if infra: cmd += ['-infra', infra] for k in sorted(labels or {}): cmd += ['-label', '%s=%s' % (k, labels[k])] for t in (tags or []): cmd += ['-tag', t] cmd += ['-json-output', self.m.json.output()] # Expected JSON output (may be produced even on failures). # # { # "error": "...", # error message on errors # "image": { # "image": "registry/name", # "digest": "sha256:...", # "tag": "its-canonical-tag", # }, # "view_image_url": "https://...", # for humans # "view_build_url": "https://...", # for humans # } try: res = self.m.step( name='cloudbuildhelper build %s' % name, cmd=cmd, step_test_data=lambda: self.test_api.output( step_test_image, name, canonical_tag, ), ) if not res.json.output: # pragma: no cover res.presentation.status = self.m.step.FAILURE raise recipe_api.InfraFailure( 'Call succeeded, but didn\'t produce -json-output') img = res.json.output.get('image') if not img: return self.NotUploadImage return self.Image( image=img['image'], digest=img['digest'], tag=img.get('tag'), ) finally: self._make_step_pretty(self.m.step.active_result, tags)
def _telemetry_gpu_collect_step(self, task, **kwargs): step_test_data = kwargs.pop('step_test_data', None) if not step_test_data: step_test_data = self.m.test_utils.test_api.canned_telemetry_gpu_output( passing=True, is_win=self.m.platform.is_win, swarming=True) args = self.get_collect_cmd_args(task) args.extend(['--task-output-dir', self.m.raw_io.output_dir()]) try: self.m.python( name=self._get_step_name('', task), script=self.m.swarming_client.path.join('swarming.py'), args=args, step_test_data=lambda: step_test_data, **kwargs) finally: # Regardless of the outcome of the test (pass or fail), we try to parse # the results. If any error occurs while parsing results, then we set them # to None, which will be treated as invalid test results by # SwarmingTelemetryGPUTest class in recipe_modules/chromium/steps.py. Note # that try-except block below will not mask the recipe_api.StepFailure # exception from the collect step above. Instead it is being allowed to # propagate after the results have been parsed. try: step_result = self.m.step.active_result outdir_json = self.m.json.dumps(step_result.raw_io.output_dir, indent=2) step_result.presentation.logs[ 'outdir_json'] = outdir_json.splitlines() # Check if it's an internal failure. summary = self.m.json.loads( step_result.raw_io.output_dir['summary.json']) if any(shard['internal_failure'] for shard in summary['shards']): raise recipe_api.InfraFailure('Internal swarming failure.') # TODO(sergiyb): Combine telemetry results from multiple shards rather # than assuming that there is always just one shard. assert len(summary['shards']) == 1 results_raw = step_result.raw_io.output_dir[self.m.path.join( '0', 'results.json')] # GPU test launcher may bail out early with return code 0 and empty # results file if there were no tests to run, e.g. when all tests are # disabled on current platform. # TODO(sergiyb): We should instead rewrite run_gpu_test.py to always # write valid results.json regardless of the return code. if step_result.retcode == 0 and results_raw == '': step_result.telemetry_results = { 'per_page_values': [], 'pages': [] } else: step_result.telemetry_results = self.m.json.loads( results_raw) self._display_pending(summary, step_result.presentation) except Exception as e: self.m.step.active_result.presentation.logs[ 'no_results_exc'] = [str(e)] self.m.step.active_result.telemetry_results = None