async def manage_partial( partial_def, artifacts_dir, tools_dir, downloads, semaphore, arch=None ): from_url = partial_def["from_mar"] to_url = partial_def["to_mar"] from_path = downloads[from_url]["extracted_path"] to_path = downloads[to_url]["extracted_path"] mar_data = { "MAR_CHANNEL_ID": os.environ["MAR_CHANNEL_ID"], "version": get_option( to_path, filename="application.ini", section="App", option="Version" ), "appName": get_option( from_path, filename="application.ini", section="App", option="Name" ), # Use Gecko repo and rev from platform.ini, not application.ini "repo": get_option( to_path, filename="platform.ini", section="Build", option="SourceRepository" ), "revision": get_option( to_path, filename="platform.ini", section="Build", option="SourceStamp" ), "locale": partial_def["locale"], "from_mar": partial_def["from_mar"], "from_size": os.path.getsize(downloads[from_url]["download_path"]), "from_hash": get_hash(downloads[from_url]["download_path"], hash_alg="sha512"), "from_buildid": get_option( from_path, filename="application.ini", section="App", option="BuildID" ), "to_mar": partial_def["to_mar"], "to_size": os.path.getsize(downloads[to_url]["download_path"]), "to_hash": get_hash(downloads[to_url]["download_path"], hash_alg="sha512"), "to_buildid": get_option( to_path, filename="application.ini", section="App", option="BuildID" ), "mar": partial_def["dest_mar"], } # if branch not set explicitly use repo-name mar_data["branch"] = partial_def.get("branch", Path(mar_data["repo"]).name) for field in ( "update_number", "previousVersion", "previousBuildNumber", "toVersion", "toBuildNumber", ): if field in partial_def: mar_data[field] = partial_def[field] dest_mar = Path(artifacts_dir) / mar_data["mar"] async with semaphore: await generate_partial(from_path, to_path, dest_mar, mar_data, tools_dir, arch) mar_data["size"] = os.path.getsize(dest_mar) mar_data["hash"] = get_hash(dest_mar, hash_alg="sha512") return mar_data
def _release_if_needed(store, channel, snap_file_path): # We can't easily know what's the revision and the version of the current and the latest snap. # That's why this function fetches all availables revisions, transforms the data, and then # finds what's current and latest. all_revisions = _list_all_revisions(store) metadata_per_revision = _pluck_metadata(all_revisions) metadata_per_revision = _filter_versions_that_are_not_the_same_type(metadata_per_revision, channel) metadata_per_revision = _populate_sha3_384(store, metadata_per_revision) current_sha3_384 = get_hash(snap_file_path, hash_alg='sha3_384') current_snap_revision, current_snap_version = _find_revision_and_version_of_current_snap( metadata_per_revision, current_sha3_384 ) latest_released_revision, latest_released_version = _pick_revision_and_version_of_latest_released_snap( channel, metadata_per_revision ) try: _check_current_snap_is_not_released( current_snap_revision, current_snap_version, latest_released_revision, latest_released_version ) except AlreadyLatestError as e: # Not raising when Snap is already the latest allows the task to be idempotent. # Other errors must raise. log.warning(e) return release_kwargs = { 'snap_name': _SNAP_NAME_ON_STORE, 'revision': current_snap_revision, 'channels': [channel] } log.debug('Calling store.release() with these kwargs: {}'.format(snap_file_path)) store.release(**release_kwargs)
async def download_cot(chain): """Download the signed chain of trust artifacts. Args: chain (ChainOfTrust): the chain of trust to add to. Raises: DownloadError: on failure. """ async_tasks = [] # only deal with chain.links, which are previously finished tasks with # signed chain of trust artifacts. ``chain.task`` is the current running # task, and will not have a signed chain of trust artifact yet. for link in chain.links: task_id = link.task_id url = get_artifact_url(chain.context, task_id, 'public/chainOfTrust.json.asc') parent_dir = link.cot_dir async_tasks.append( asyncio.ensure_future( download_artifacts(chain.context, [url], parent_dir=parent_dir, valid_artifact_task_ids=[task_id]))) paths = await raise_future_exceptions(async_tasks) for path in paths: sha = get_hash(path[0]) log.debug("{} downloaded; hash is {}".format(path[0], sha))
def get_cot_artifacts(context): """Generate the artifact relative paths and shas for the chain of trust Args: context (scriptworker.context.Context): the scriptworker context. Returns: dict: a dictionary of {"path/to/artifact": {"hash_alg": "..."}, ...} """ artifacts = {} filepaths = filepaths_in_dir(context.config['artifact_dir']) hash_alg = context.config['chain_of_trust_hash_algorithm'] for filepath in sorted(filepaths): path = os.path.join(context.config['artifact_dir'], filepath) sha = get_hash(path, hash_alg=hash_alg) artifacts[filepath] = {hash_alg: sha} return artifacts
def get_cot_artifacts(context): """Generate the artifact relative paths and shas for the chain of trust. Args: context (scriptworker.context.Context): the scriptworker context. Returns: dict: a dictionary of {"path/to/artifact": {"hash_alg": "..."}, ...} """ artifacts = {} filepaths = filepaths_in_dir(context.config['artifact_dir']) hash_alg = context.config['chain_of_trust_hash_algorithm'] for filepath in sorted(filepaths): path = os.path.join(context.config['artifact_dir'], filepath) sha = get_hash(path, hash_alg=hash_alg) artifacts[filepath] = {hash_alg: sha} return artifacts
async def download(url, dest, mode=None): # noqa: E999 log.info("Downloading %s to %s", url, dest) chunk_size = 4096 bytes_downloaded = 0 async with aiohttp.ClientSession(raise_for_status=True) as session: start = time.time() async with session.get(url, timeout=120) as resp: # Additional early logging for download timeouts. log.debug("Fetching from url %s", resp.url) for history in resp.history: log.debug("Redirection history: %s", history.url) log.debug("Headers for %s: %s", resp.url, resp.headers) if "Content-Length" in resp.headers: log.debug( "Content-Length expected for %s: %s", url, resp.headers["Content-Length"], ) log_interval = chunk_size * 1024 with open(dest, "wb") as fd: while True: chunk = await resp.content.read(chunk_size) if not chunk: break fd.write(chunk) bytes_downloaded += len(chunk) log_interval -= len(chunk) if log_interval <= 0: log.debug("Bytes downloaded for %s: %d", url, bytes_downloaded) log_interval = chunk_size * 1024 end = time.time() log.info( "Downloaded %s, %s bytes in %s seconds: sha256:%s", url, bytes_downloaded, int(end - start), get_hash(dest, hash_alg="sha256"), ) if mode: log.info("chmod %o %s", mode, dest) os.chmod(dest, mode)
async def download_cot_artifact(chain, task_id, path): """Download an artifact and verify its SHA against the chain of trust. Args: chain (ChainOfTrust): the chain of trust object task_id (str): the task ID to download from path (str): the relative path to the artifact to download Returns: str: the full path of the downloaded artifact Raises: CoTError: on failure. """ link = chain.get_link(task_id) log.debug("Verifying {} is in {} cot artifacts...".format(path, task_id)) if path not in link.cot['artifacts']: raise CoTError("path {} not in {} {} chain of trust artifacts!".format( path, link.name, link.task_id)) url = get_artifact_url(chain.context, task_id, path) log.info("Downloading Chain of Trust artifact:\n{}".format(url)) await download_artifacts(chain.context, [url], parent_dir=link.cot_dir, valid_artifact_task_ids=[task_id]) full_path = link.get_artifact_full_path(path) for alg, expected_sha in link.cot['artifacts'][path].items(): if alg not in chain.context.config['valid_hash_algorithms']: raise CoTError("BAD HASH ALGORITHM: {}: {} {}!".format( link.name, alg, full_path)) real_sha = get_hash(full_path, hash_alg=alg) if expected_sha != real_sha: raise CoTError("BAD HASH: {}: Expected {} {}; got {}!".format( link.name, alg, expected_sha, real_sha)) log.debug("{} matches the expected {} {}".format( full_path, alg, expected_sha)) return full_path
def test_get_hash(): path = os.path.join(os.path.dirname(__file__), "data", "azure.xml") sha = utils.get_hash(path, hash_alg="sha256") assert sha == "584818280d7908da33c810a25ffb838b1e7cec1547abd50c859521229942c5a5"