def get_snap_channel(config, task): if "channel" in task["payload"]: channel = task["payload"]["channel"] scope = SNAP_SCOPES_PREFIX + channel.split("/")[0] if config["push_to_store"] and scope not in task["scopes"]: raise TaskVerificationError( f"Channel {channel} not allowed, missing scope {scope}") else: scope = get_single_item_from_sequence( task["scopes"], lambda scope: scope.startswith(SNAP_SCOPES_PREFIX), ErrorClass=TaskVerificationError, no_item_error_message="No scope starts with {}".format( SNAP_SCOPES_PREFIX), too_many_item_error_message="Too many scopes start with {}".format( SNAP_SCOPES_PREFIX), ) channel = scope[len(SNAP_SCOPES_PREFIX):] channel = "esr/stable" if channel == "esr" else channel if channel not in ALLOWED_CHANNELS: raise TaskVerificationError( 'Channel "{}" is not allowed. Allowed ones are: {}'.format( channel, ALLOWED_CHANNELS)) return channel
def get_snap_channel(config, task): if 'channel' in task['payload']: channel = task['payload']['channel'] scope = SNAP_SCOPES_PREFIX + channel.split('/')[0] if config['push_to_store'] and scope not in task['scopes']: raise TaskVerificationError( f'Channel {channel} not allowed, missing scope {scope}' ) else: scope = get_single_item_from_sequence( task['scopes'], lambda scope: scope.startswith(SNAP_SCOPES_PREFIX), ErrorClass=TaskVerificationError, no_item_error_message='No scope starts with {}'.format(SNAP_SCOPES_PREFIX), too_many_item_error_message='Too many scopes start with {}'.format(SNAP_SCOPES_PREFIX), ) channel = scope[len(SNAP_SCOPES_PREFIX):] channel = 'esr/stable' if channel == 'esr' else channel if channel not in ALLOWED_CHANNELS: raise TaskVerificationError( 'Channel "{}" is not allowed. Allowed ones are: {}'.format(channel, ALLOWED_CHANNELS) ) return channel
def _get_play_config(context, android_product): try: accounts = context.config['google_play_accounts'] except KeyError: raise TaskVerificationError('"google_play_accounts" is not part of the configuration') try: return accounts[android_product] except KeyError: raise TaskVerificationError('Android "{}" does not exist in the configuration of this instance.\ Are you sure you allowed to push such APK?'.format(android_product))
def _check_locale_consistency(locale_in_payload, uniques_locales_in_upstream_artifacts): if len(uniques_locales_in_upstream_artifacts) > 1: raise TaskVerificationError( '`task.payload.locale` is defined ("{}") but too many locales set in \ `task.payload.upstreamArtifacts` ({})'.format(locale_in_payload, uniques_locales_in_upstream_artifacts) ) elif len(uniques_locales_in_upstream_artifacts) == 1: locale_in_upstream_artifacts = uniques_locales_in_upstream_artifacts[0] if locale_in_payload != locale_in_upstream_artifacts: raise TaskVerificationError( '`task.payload.locale` ("{}") does not match the one set in \ `task.payload.upstreamArtifacts` ("{}")'.format(locale_in_payload, locale_in_upstream_artifacts) )
def check_product_names_match_aliases(context): """Make sure we don't do any cross-product/channel alias update""" aliases = context.task["payload"]["aliases_entries"] validations = [] for alias, product_name in aliases.items(): if alias not in ALIASES_REGEXES.keys(): raise TaskVerificationError("Unrecognized alias:{}".format(alias)) validations.append(matches(product_name, ALIASES_REGEXES[alias])) if not all(validations): raise TaskVerificationError("The product/alias pairs are corrupt: {}".format(aliases))
def _check_archive_itself(zip_path, zip_max_size_in_mb): zip_size = os.path.getsize(zip_path) zip_size_in_mb = zip_size // (1024 * 1024) if zip_size_in_mb > zip_max_size_in_mb: raise TaskVerificationError( 'Archive "{}" is too big. Max accepted size (in MB): {}. File size (in MB): {}' .format(zip_path, zip_max_size_in_mb, zip_size_in_mb)) if not zipfile.is_zipfile(zip_path): raise TaskVerificationError('Archive "{}" is not a valid zip file.') log.info('Structure of archive "{}" is sane'.format(zip_path))
def _check_google_play_string_is_the_only_failed_task(failed_artifacts_per_task_id): if len(failed_artifacts_per_task_id) > 1: raise TaskVerificationError( 'Only 1 task is allowed to fail. Found: {}'.format(failed_artifacts_per_task_id.keys()) ) task_id = list(failed_artifacts_per_task_id.keys())[0] failed_artifacts = failed_artifacts_per_task_id[task_id] if _EXPECTED_L10N_STRINGS_FILE_NAME not in failed_artifacts: raise TaskVerificationError( 'Could not find "{}" in the only failed taskId "{}". Please note this is the only \ artifact allowed to be absent. Found: {}' .format(_EXPECTED_L10N_STRINGS_FILE_NAME, task_id, failed_artifacts) )
def build_filelist_dict(context): """Build a dictionary of cot-downloaded paths and formats. Scriptworker will pre-download and pre-verify the `upstreamArtifacts` in our `work_dir`. Let's build a dictionary of relative `path` to a dictionary of `full_path` and signing `formats`. Args: context (Context): the signing context Raises: TaskVerificationError: if the files don't exist on disk or if authenticode_comment is used without authenticode or on a non .msi Returns: dict of dicts: the dictionary of relative `path` to a dictionary with `full_path` and a list of signing `formats`. """ filelist_dict = {} messages = [] for artifact_dict in context.task["payload"]["upstreamArtifacts"]: authenticode_comment = artifact_dict.get("authenticode_comment") if authenticode_comment and not any( "authenticode" in fmt for fmt in artifact_dict["formats"]): raise TaskVerificationError( "Cannot use authenticode_comment without an authenticode format" ) if authenticode_comment and not any( path.endswith(".msi") for path in artifact_dict["paths"]): # Don't have to think about .zip and such unpacking for the comment raise TaskVerificationError( "There is no support for authenticode_comment outside of msi's at this time" ) for path in artifact_dict["paths"]: full_path = os.path.join(context.config["work_dir"], "cot", artifact_dict["taskId"], path) if not os.path.exists(full_path): messages.append("{} doesn't exist!".format(full_path)) filelist_dict[path] = { "full_path": full_path, "formats": _sort_formats(artifact_dict["formats"]) } if authenticode_comment: filelist_dict[path]["comment"] = authenticode_comment if messages: raise TaskVerificationError(messages) return filelist_dict
def _ensure_all_expected_files_are_deflated_on_disk(zip_path, expected_full_paths): for full_path in expected_full_paths: if not os.path.exists(full_path): raise TaskVerificationError( 'After extracting "{}", expected file "{}" does not exist'. format(zip_path, full_path)) if not os.path.isfile(full_path): raise TaskVerificationError( 'After extracting "{}", "{}" is not a file'.format( zip_path, full_path)) log.info( 'All files declared in archive "{}" exist and are regular files: {}'. format(zip_path, expected_full_paths))
def validate_task_schema(context, schema_key='schema_file'): """Validate the task definition. Args: context (scriptworker.context.Context): the scriptworker context. It must contain a task and the config pointing to the schema file schema_key: the key in `context.config` where the path to the schema file is. Key can contain dots (e.g.: 'schema_files.file_a'), in which case Raises: TaskVerificationError: if the task doesn't match the schema """ schema_path = context.config schema_keys = schema_key.split('.') for key in schema_keys: schema_path = schema_path[key] task_schema = load_json_or_yaml(schema_path, is_path=True) log.debug('Task is validated against this schema: {}'.format(task_schema)) try: validate_json_schema(context.task, task_schema) except ScriptWorkerTaskException as e: raise TaskVerificationError( 'Cannot validate task against schema. Task: {}.'.format( context.task)) from e
def build_filelist(context): """Build a list of cot-downloaded paths. Scriptworker will pre-download and pre-verify the `upstreamArtifacts` in our `work_dir`. Let's build a list of relative of full paths. Args: context (SigningContext): the signing context Raises: TaskVerificationError: if the files don't exist on disk Returns: list: `full_path` of all files. """ filelist = [] messages = [] for artifact_dict in context.task["payload"]["upstreamArtifacts"]: for path in artifact_dict["paths"]: full_path = os.path.join(context.config["work_dir"], "cot", artifact_dict["taskId"], path) if not os.path.exists(full_path): messages.append("{} doesn't exist!".format(full_path)) filelist.append(full_path) if messages: raise TaskVerificationError(messages) return filelist
def get_amo_instance_config_from_scope(context): """Get instance configuration from task scope. Args: context (Context): the scriptworker context Raises: TaskVerificationError: if the task doesn't have the necessary scopes or if the instance isn't configured to process it Returns: dict: configuration, formatted like: { 'amo_server': 'http://some-amo-it.url', 'jwt_user': '******', 'jwt_secret': 'some-secret' } """ scope = _get_scope(context.task) configured_instances = context.config["amo_instances"] try: return configured_instances[scope] except KeyError: raise TaskVerificationError( 'This worker is not configured to handle scope "{}"'.format(scope))
def _generate_beetmover_template_args_maven(task, release_props): tmpl_args = { "artifact_id": task["payload"]["artifact_id"], "template_key": "maven_{}".format(release_props["appName"]) } # Geckoview follows the FirefoxVersion pattern if release_props.get("appName") == "geckoview": payload_version = FirefoxVersion.parse(task["payload"]["version"]) # Change version number to major.minor.buildId because that's what the build task produces version = [ payload_version.major_number, payload_version.minor_number, release_props["buildid"] ] else: payload_version = MavenVersion.parse(task["payload"]["version"]) version = [ payload_version.major_number, payload_version.minor_number, payload_version.patch_number ] if any(number is None for number in version): raise TaskVerificationError( "At least one digit is undefined. Got: {}".format(version)) tmpl_args["version"] = ".".join(str(n) for n in version) # XXX: some appservices maven.zip files have a different structure, # encompassing only `pom` and `jar` files. We toggle that behavior in the # mapping by using this flag tmpl_args["is_jar"] = task["payload"].get("is_jar") return tmpl_args
def build_filelist_dict(context): """Build a dictionary of cot-downloaded paths and formats. Scriptworker will pre-download and pre-verify the `upstreamArtifacts` in our `work_dir`. Let's build a dictionary of relative `path` to a dictionary of `full_path` and signing `formats`. Args: context (Context): the signing context Raises: TaskVerificationError: if the files don't exist on disk Returns: dict of dicts: the dictionary of relative `path` to a dictionary with `full_path` and a list of signing `formats`. """ filelist_dict = {} messages = [] for artifact_dict in context.task['payload']['upstreamArtifacts']: for path in artifact_dict['paths']: full_path = os.path.join( context.config['work_dir'], 'cot', artifact_dict['taskId'], path ) if not os.path.exists(full_path): messages.append("{} doesn't exist!".format(full_path)) filelist_dict[path] = { "full_path": full_path, "formats": _sort_formats(artifact_dict['formats']), } if messages: raise TaskVerificationError(messages) return filelist_dict
def task_cert_type(context): """Extract task certificate type. Args: context (Context): the signing context. Raises: TaskVerificationError: if the number of cert scopes is not 1. Returns: str: the cert type. """ if not context.task or not context.task["scopes"]: raise TaskVerificationError("No scopes found") prefixes = _get_cert_prefixes(context) scopes = _extract_scopes_from_unique_prefix(scopes=context.task["scopes"], prefixes=prefixes) return get_single_item_from_sequence( scopes, condition=lambda _: True, # scopes must just contain 1 single item ErrorClass=TaskVerificationError, no_item_error_message="No scope starting with any of these prefixes {} found".format(prefixes), too_many_item_error_message="More than one scope found", )
def _generate_beetmover_template_args_maven(task, release_props): tmpl_args = { 'artifact_id': task['payload']['artifact_id'], 'template_key': 'maven_{}'.format(release_props['appName']), } # FIXME: this is a temporarily solution while we sanitize the payload # under https://github.com/mozilla-releng/beetmoverscript/issues/196 if 'SNAPSHOT' in task['payload']['version']: payload_version = MavenVersion.parse(task['payload']['version']) else: payload_version = FirefoxVersion.parse(task['payload']['version']) # Change version number to major.minor.buildId because that's what the build task produces version = [ payload_version.major_number, payload_version.minor_number, release_props.get('buildid', payload_version.patch_number) ] if any(number is None for number in version): raise TaskVerificationError( 'At least one digit is undefined. Got: {}'.format(version)) tmpl_args['version'] = '.'.join(str(n) for n in version) if isinstance(payload_version, MavenVersion) and payload_version.is_snapshot: tmpl_args['snapshot_version'] = payload_version tmpl_args['date_timestamp'] = "{{date_timestamp}}" tmpl_args['clock_timestamp'] = "{{clock_timestamp}}" tmpl_args['build_number'] = "{{build_number}}" return tmpl_args
def check_product_names_match_nightly_locations(context): """Double check that nightly products are as expected""" products = context.task["payload"]["bouncer_products"] valid_sets = [] for product_set in BOUNCER_PATH_REGEXES_PER_PRODUCT: valid_sets.append(sorted(product_set.keys())) if sorted(products) not in valid_sets: raise TaskVerificationError("Products {} don't correspond to nightly ones".format(products))
def _check_tarball_size(tarball_path): tar_size = os.path.getsize(tarball_path) tar_size_in_mb = tar_size // (1024 * 1024) if tar_size_in_mb > TAR_MAX_SIZE_IN_MB: raise TaskVerificationError( f"Tar {tarball_path} is too big. Max accepted size is {TAR_MAX_SIZE_IN_MB}" )
def _check_scopes_exist_and_all_have_the_same_prefix(scopes, prefixes): for prefix in prefixes: if all(scope.startswith(prefix) for scope in scopes): break else: raise TaskVerificationError( 'Scopes must exist and all have the same prefix. ' 'Given scopes: {}. Allowed prefixes: {}'.format(scopes, prefixes))
def _get_product_config(context, android_product): try: products = context.config['products'] except KeyError: raise ConfigValidationError('"products" is not part of the configuration') matching_products = [product for product in products if android_product in product['product_names']] if len(matching_products) == 0: raise TaskVerificationError('Android "{}" does not exist in the configuration of this ' 'instance. Are you sure you allowed to push such an ' 'APK?'.format(android_product)) if len(matching_products) > 1: raise TaskVerificationError('The configuration is invalid: multiple product configs match ' 'the product "{}"'.format(android_product)) return matching_products[0]
def get_ship_it_instance_config_from_scope(context): scope = _get_scope(context, "server") configured_instances = context.config['ship_it_instances'] try: return configured_instances[scope] except KeyError: raise TaskVerificationError( 'This worker is not configured to handle scope "{}"'.format(scope))
def get_ship_it_instance_config_from_scope(context): scope = _get_scope(context, "server") configured_instance = context.config["shipit_instance"] if configured_instance.get("scope") == scope: return configured_instance raise TaskVerificationError( 'This worker is not configured to handle scope "{}"'.format(scope))
def extract_file_config_from_artifact_map(artifact_map, path, task_id, locale): """Return matching artifact map config.""" for entry in artifact_map: if entry["taskId"] != task_id or entry["locale"] != locale: continue if not entry["paths"].get(path): continue return entry["paths"][path] raise TaskVerificationError("No artifact map entry for {}/{} {}".format(task_id, locale, path))
def _extract_and_check_timestamps(archive_filename, regex): match = re.search(regex, archive_filename) try: identifier = match.group() except AttributeError: raise TaskVerificationError( 'File "{}" present in archive has invalid identifier. ' 'Expected YYYYMMDD.HHMMSS-BUILDNUMBER within in'.format( archive_filename)) timestamp, build_number = identifier.split('-') try: datetime.datetime.strptime(timestamp, '%Y%m%d.%H%M%S') except ValueError: raise TaskVerificationError( 'File "{}" present in archive has invalid timestamp. ' 'Expected YYYYMMDD.HHMMSS within in'.format(archive_filename)) date_timestamp, clock_timestamp = timestamp.split('.') return date_timestamp, clock_timestamp, build_number
def get_flatpak_channel(config, task): payload = task["payload"] if "channel" not in payload: raise TaskVerificationError( f"channel must be defined in the task payload. Given payload: {payload}" ) channel = payload["channel"] scope = FLATPAK_SCOPES_PREFIX + channel if config["push_to_flathub"] and scope not in task["scopes"]: raise TaskVerificationError( f"Channel {channel} not allowed, missing scope {scope}") if channel not in ALLOWED_CHANNELS: raise TaskVerificationError( 'Channel "{}" is not allowed. Allowed ones are: {}'.format( channel, ALLOWED_CHANNELS)) return channel
def _ensure_all_expected_files_are_present_in_archive(zip_path, files_in_archive, expected_files): files_in_archive = set(files_in_archive) unique_expected_files = set(expected_files) if len(expected_files) != len(unique_expected_files): duplicated_files = [ file for file in unique_expected_files if expected_files.count(file) > 1 ] raise TaskVerificationError( 'Found duplicated expected files in archive "{}": {}'.format( zip_path, duplicated_files)) for file_ in files_in_archive: if os.path.isabs(file_): raise TaskVerificationError( 'File "{}" in archive "{}" cannot be an absolute one.'.format( file_, zip_path)) if os.path.normpath(file_) != file_: raise TaskVerificationError( 'File "{}" in archive "{}" cannot contain up-level reference nor redundant separators' .format(file_, zip_path)) if file_ not in unique_expected_files: raise TaskVerificationError( 'File "{}" present in archive "{}" is not expected. Expected: {}' .format(file_, zip_path, unique_expected_files)) if len(files_in_archive) != len(unique_expected_files): missing_expected_files = [ file for file in unique_expected_files if file not in files_in_archive ] raise TaskVerificationError( 'Expected files are missing in archive "{}": {}'.format( zip_path, missing_expected_files)) log.info('Archive "{}" contains all expected files: {}'.format( zip_path, unique_expected_files))
def _ensure_no_file_got_overwritten(task_id, extracted_files): unique_paths = set(extracted_files) if len(unique_paths) != len(extracted_files): duplicated_paths = [ path for path in unique_paths if extracted_files.count(path) > 1 ] raise TaskVerificationError( 'Archives from task "{}" overwrote files: {}'.format( task_id, duplicated_paths)) log.info('All archives from task "{}" outputed different files.'.format( task_id))
def _ensure_files_in_archive_have_decent_sizes(zip_path, zip_metadata, zip_max_size_in_mb): for file_name, file_metadata in zip_metadata.items(): compressed_size = file_metadata['compress_size'] real_size = file_metadata['file_size'] compressed_size_size_in_mb = compressed_size // (1024 * 1024) if compressed_size_size_in_mb > zip_max_size_in_mb: raise TaskVerificationError( 'In archive "{}", compressed file "{}" is too big. Max accepted size (in MB): {}. File size (in MB): {}' .format(zip_path, file_name, zip_max_size_in_mb, compressed_size_size_in_mb)) compression_ratio = real_size / compressed_size if compression_ratio > ZIP_MAX_COMPRESSION_RATIO: raise TaskVerificationError( 'In archive "{}", file "{}" has a suspicious compression ratio. Max accepted: {}. Found: {}' .format(zip_path, file_name, ZIP_MAX_COMPRESSION_RATIO, compression_ratio)) log.info( 'Archive "{}" contains files with legitimate sizes.'.format(zip_path))
async def async_main(context): # perform schema validation for the corresponding type of task validate_task_schema(context) # determine the task server and action context.server = get_task_server(context.task, context.config) context.action = get_task_action(context.task, context.config) # perform the appropriate behavior if action_map.get(context.action): await action_map[context.action](context) else: raise TaskVerificationError("Unknown action: {}!".format(context.action))
def _check_current_snap_is_not_released(current_revision, current_version, latest_released_revision, latest_released_version): if latest_released_version == current_version: if latest_released_revision == current_revision: raise AlreadyLatestError(latest_released_version, latest_released_revision) else: raise TaskVerificationError( 'Versions "{0}" are the same but revisions differ. This may mean someone shipped a rogue "{0}" before automation! ' "Latest on store: {1}. Revision of current Snap: {2}".format( latest_released_version, latest_released_revision, current_revision)) elif latest_released_version > current_version: # We don't check if the revision is higher because raise TaskVerificationError( 'Current version "{}" is lower than the latest one released on the store "{}". Downgrades are not allowed.' .format(current_version, latest_released_version)) log.debug( 'Current version "{}" is higher than the latest released one "{}". Okay to release the current one' .format(current_version, latest_released_version))