async def manage_partial(
partial_def, artifacts_dir, tools_dir, downloads, semaphore, arch=None
):
from_url = partial_def["from_mar"]
to_url = partial_def["to_mar"]
from_path = downloads[from_url]["extracted_path"]
to_path = downloads[to_url]["extracted_path"]
mar_data = {
"MAR_CHANNEL_ID": os.environ["MAR_CHANNEL_ID"],
"version": get_option(
to_path, filename="application.ini", section="App", option="Version"
),
"appName": get_option(
from_path, filename="application.ini", section="App", option="Name"
),
# Use Gecko repo and rev from platform.ini, not application.ini
"repo": get_option(
to_path, filename="platform.ini", section="Build", option="SourceRepository"
),
"revision": get_option(
to_path, filename="platform.ini", section="Build", option="SourceStamp"
),
"locale": partial_def["locale"],
"from_mar": partial_def["from_mar"],
"from_size": os.path.getsize(downloads[from_url]["download_path"]),
"from_hash": get_hash(downloads[from_url]["download_path"], hash_alg="sha512"),
"from_buildid": get_option(
from_path, filename="application.ini", section="App", option="BuildID"
),
"to_mar": partial_def["to_mar"],
"to_size": os.path.getsize(downloads[to_url]["download_path"]),
"to_hash": get_hash(downloads[to_url]["download_path"], hash_alg="sha512"),
"to_buildid": get_option(
to_path, filename="application.ini", section="App", option="BuildID"
),
"mar": partial_def["dest_mar"],
}
# if branch not set explicitly use repo-name
mar_data["branch"] = partial_def.get("branch", Path(mar_data["repo"]).name)
for field in (
"update_number",
"previousVersion",
"previousBuildNumber",
"toVersion",
"toBuildNumber",
):
if field in partial_def:
mar_data[field] = partial_def[field]
dest_mar = Path(artifacts_dir) / mar_data["mar"]
async with semaphore:
await generate_partial(from_path, to_path, dest_mar, mar_data, tools_dir, arch)
mar_data["size"] = os.path.getsize(dest_mar)
mar_data["hash"] = get_hash(dest_mar, hash_alg="sha512")
return mar_data
def _release_if_needed(store, channel, snap_file_path):
# We can't easily know what's the revision and the version of the current and the latest snap.
# That's why this function fetches all availables revisions, transforms the data, and then
# finds what's current and latest.
all_revisions = _list_all_revisions(store)
metadata_per_revision = _pluck_metadata(all_revisions)
metadata_per_revision = _filter_versions_that_are_not_the_same_type(metadata_per_revision, channel)
metadata_per_revision = _populate_sha3_384(store, metadata_per_revision)
current_sha3_384 = get_hash(snap_file_path, hash_alg='sha3_384')
current_snap_revision, current_snap_version = _find_revision_and_version_of_current_snap(
metadata_per_revision, current_sha3_384
)
latest_released_revision, latest_released_version = _pick_revision_and_version_of_latest_released_snap(
channel, metadata_per_revision
)
try:
_check_current_snap_is_not_released(
current_snap_revision, current_snap_version,
latest_released_revision, latest_released_version
)
except AlreadyLatestError as e:
# Not raising when Snap is already the latest allows the task to be idempotent.
# Other errors must raise.
log.warning(e)
return
release_kwargs = {
'snap_name': _SNAP_NAME_ON_STORE,
'revision': current_snap_revision,
'channels': [channel]
}
log.debug('Calling store.release() with these kwargs: {}'.format(snap_file_path))
store.release(**release_kwargs)
async def download_cot(chain):
"""Download the signed chain of trust artifacts.
Args:
chain (ChainOfTrust): the chain of trust to add to.
Raises:
DownloadError: on failure.
"""
async_tasks = []
# only deal with chain.links, which are previously finished tasks with
# signed chain of trust artifacts. ``chain.task`` is the current running
# task, and will not have a signed chain of trust artifact yet.
for link in chain.links:
task_id = link.task_id
url = get_artifact_url(chain.context, task_id,
'public/chainOfTrust.json.asc')
parent_dir = link.cot_dir
async_tasks.append(
asyncio.ensure_future(
download_artifacts(chain.context, [url],
parent_dir=parent_dir,
valid_artifact_task_ids=[task_id])))
paths = await raise_future_exceptions(async_tasks)
for path in paths:
sha = get_hash(path[0])
log.debug("{} downloaded; hash is {}".format(path[0], sha))
def get_cot_artifacts(context):
"""Generate the artifact relative paths and shas for the chain of trust
Args:
context (scriptworker.context.Context): the scriptworker context.
Returns:
dict: a dictionary of {"path/to/artifact": {"hash_alg": "..."}, ...}
"""
artifacts = {}
filepaths = filepaths_in_dir(context.config['artifact_dir'])
hash_alg = context.config['chain_of_trust_hash_algorithm']
for filepath in sorted(filepaths):
path = os.path.join(context.config['artifact_dir'], filepath)
sha = get_hash(path, hash_alg=hash_alg)
artifacts[filepath] = {hash_alg: sha}
return artifacts
def get_cot_artifacts(context):
"""Generate the artifact relative paths and shas for the chain of trust.
Args:
context (scriptworker.context.Context): the scriptworker context.
Returns:
dict: a dictionary of {"path/to/artifact": {"hash_alg": "..."}, ...}
"""
artifacts = {}
filepaths = filepaths_in_dir(context.config['artifact_dir'])
hash_alg = context.config['chain_of_trust_hash_algorithm']
for filepath in sorted(filepaths):
path = os.path.join(context.config['artifact_dir'], filepath)
sha = get_hash(path, hash_alg=hash_alg)
artifacts[filepath] = {hash_alg: sha}
return artifacts
async def download(url, dest, mode=None): # noqa: E999
log.info("Downloading %s to %s", url, dest)
chunk_size = 4096
bytes_downloaded = 0
async with aiohttp.ClientSession(raise_for_status=True) as session:
start = time.time()
async with session.get(url, timeout=120) as resp:
# Additional early logging for download timeouts.
log.debug("Fetching from url %s", resp.url)
for history in resp.history:
log.debug("Redirection history: %s", history.url)
log.debug("Headers for %s: %s", resp.url, resp.headers)
if "Content-Length" in resp.headers:
log.debug(
"Content-Length expected for %s: %s",
url,
resp.headers["Content-Length"],
)
log_interval = chunk_size * 1024
with open(dest, "wb") as fd:
while True:
chunk = await resp.content.read(chunk_size)
if not chunk:
break
fd.write(chunk)
bytes_downloaded += len(chunk)
log_interval -= len(chunk)
if log_interval <= 0:
log.debug("Bytes downloaded for %s: %d", url,
bytes_downloaded)
log_interval = chunk_size * 1024
end = time.time()
log.info(
"Downloaded %s, %s bytes in %s seconds: sha256:%s",
url,
bytes_downloaded,
int(end - start),
get_hash(dest, hash_alg="sha256"),
)
if mode:
log.info("chmod %o %s", mode, dest)
os.chmod(dest, mode)
async def download_cot_artifact(chain, task_id, path):
"""Download an artifact and verify its SHA against the chain of trust.
Args:
chain (ChainOfTrust): the chain of trust object
task_id (str): the task ID to download from
path (str): the relative path to the artifact to download
Returns:
str: the full path of the downloaded artifact
Raises:
CoTError: on failure.
"""
link = chain.get_link(task_id)
log.debug("Verifying {} is in {} cot artifacts...".format(path, task_id))
if path not in link.cot['artifacts']:
raise CoTError("path {} not in {} {} chain of trust artifacts!".format(
path, link.name, link.task_id))
url = get_artifact_url(chain.context, task_id, path)
log.info("Downloading Chain of Trust artifact:\n{}".format(url))
await download_artifacts(chain.context, [url],
parent_dir=link.cot_dir,
valid_artifact_task_ids=[task_id])
full_path = link.get_artifact_full_path(path)
for alg, expected_sha in link.cot['artifacts'][path].items():
if alg not in chain.context.config['valid_hash_algorithms']:
raise CoTError("BAD HASH ALGORITHM: {}: {} {}!".format(
link.name, alg, full_path))
real_sha = get_hash(full_path, hash_alg=alg)
if expected_sha != real_sha:
raise CoTError("BAD HASH: {}: Expected {} {}; got {}!".format(
link.name, alg, expected_sha, real_sha))
log.debug("{} matches the expected {} {}".format(
full_path, alg, expected_sha))
return full_path
def test_get_hash():
path = os.path.join(os.path.dirname(__file__), "data", "azure.xml")
sha = utils.get_hash(path, hash_alg="sha256")
assert sha == "584818280d7908da33c810a25ffb838b1e7cec1547abd50c859521229942c5a5"
def test_get_hash():
path = os.path.join(os.path.dirname(__file__), "data", "azure.xml")
sha = utils.get_hash(path, hash_alg="sha256")
assert sha == "584818280d7908da33c810a25ffb838b1e7cec1547abd50c859521229942c5a5"
