def test_write_json(): sample_data = get_fake_balrog_props() with tempfile.NamedTemporaryFile(delete=True) as fp: write_json(fp.name, sample_data) with open(fp.name, "r") as fread: retrieved_data = json.load(fread) assert sample_data == retrieved_data
def test_write_json(): sample_data = get_fake_valid_task()['payload']['releaseProperties'] with tempfile.NamedTemporaryFile(delete=True) as fp: write_json(fp.name, sample_data) with open(fp.name, "r") as fread: retrieved_data = json.load(fread) assert sample_data == retrieved_data
async def move_beets(context, artifacts_to_beetmove, manifest=None, artifact_map=None): beets = [] for locale in artifacts_to_beetmove: installer_artifact = "" buildhub_artifact_exists = False # get path of installer beet for artifact in artifacts_to_beetmove[locale]: if exists_or_endswith(artifact, INSTALLER_ARTIFACTS): installer_artifact = artifact if exists_or_endswith(artifact, BUILDHUB_ARTIFACT): buildhub_artifact_exists = True # throws error if buildhub.json is present and installer isn't if not installer_artifact and buildhub_artifact_exists: raise ScriptWorkerTaskException( "could not determine installer path from task payload") # move beets for artifact in artifacts_to_beetmove[locale]: source = artifacts_to_beetmove[locale][artifact] # update buildhub.json file # if there is no installer then there will be no buildhub.json artifact # in logical coding terms, this means that if installer_path is an empty # string, then this if-block is never reached if exists_or_endswith(artifact, BUILDHUB_ARTIFACT): write_json( source, get_updated_buildhub_artifact( path=source, installer_artifact=installer_artifact, installer_path=artifacts_to_beetmove[locale] [installer_artifact], context=context, locale=locale, manifest=manifest, artifact_map=artifact_map, ), ) if artifact_map: task_id = get_taskId_from_full_path(source) # Should only ever be one (taskId, locale) match. map_entry = extract_file_config_from_artifact_map( artifact_map, artifact, task_id, locale) artifact_pretty_name = map_entry["checksums_path"] destinations = map_entry["destinations"] update_balrog_manifest = map_entry.get( "update_balrog_manifest", False) balrog_format = map_entry.get("balrog_format", "") from_buildid = map_entry.get("from_buildid") else: artifact_pretty_name = manifest["mapping"][locale][artifact][ "s3_key"] destinations = [ os.path.join(manifest["s3_bucket_path"], dest) for dest in manifest["mapping"][locale][artifact]["destinations"] ] update_balrog_manifest = manifest["mapping"][locale][ artifact].get("update_balrog_manifest") # Adjust old template format. # artifact map specifies these separately. # templates say "update_balrog_manifest": true # or "update_balrog_manifest": {"format": mozinfo} if isinstance(update_balrog_manifest, dict): balrog_format = update_balrog_manifest.get("format") update_balrog_manifest = True else: balrog_format = "" from_buildid = manifest["mapping"][locale][artifact].get( "from_buildid") beets.append( asyncio.ensure_future( move_beet( context, source, destinations, locale=locale, update_balrog_manifest=update_balrog_manifest, balrog_format=balrog_format, from_buildid=from_buildid, artifact_pretty_name=artifact_pretty_name, ))) await raise_future_exceptions(beets) # Fix up balrog manifest. We need an entry with both completes and # partials, which is why we store up the data from each moved beet # and collate it now. for locale, info in context.raw_balrog_manifest.items(): for format in info["completeInfo"]: balrog_entry = enrich_balrog_manifest(context, locale) balrog_entry["completeInfo"] = [info["completeInfo"][format]] if "partialInfo" in info: balrog_entry["partialInfo"] = info["partialInfo"] if format: balrog_entry["blob_suffix"] = "-{}".format(format) context.balrog_manifest.append(balrog_entry)
def add_balrog_manifest_to_artifacts(context): abs_file_path = os.path.join(context.config['artifact_dir'], 'public/manifest.json') utils.write_json(abs_file_path, context.balrog_manifest)
async def move_beets(context, artifacts_to_beetmove, manifest): beets = [] for locale in artifacts_to_beetmove: installer_path = '' buildhub_artifact_exists = False # get path of installer beet for artifact in artifacts_to_beetmove[locale]: if artifact in INSTALLER_ARTIFACTS: installer_path = artifacts_to_beetmove[locale][artifact] if artifact == BUILDHUB_ARTIFACT: buildhub_artifact_exists = True # throws error if buildhub.json is present and installer isn't if not installer_path and buildhub_artifact_exists: raise ScriptWorkerTaskException( "could not determine installer path from task payload") # move beets for artifact in artifacts_to_beetmove[locale]: source = artifacts_to_beetmove[locale][artifact] artifact_pretty_name = manifest['mapping'][locale][artifact][ 's3_key'] # update buildhub.json file # if there is no installer then there will be no buildhub.json artifact # in logical coding terms, this means that if installer_path is an empty # string, then this if-block is never reached if artifact == BUILDHUB_ARTIFACT: write_json( source, get_updated_buildhub_artifact(source, installer_path, context, manifest, locale)) destinations = [ os.path.join(manifest["s3_bucket_path"], dest) for dest in manifest['mapping'][locale][artifact]['destinations'] ] balrog_manifest = manifest['mapping'][locale][artifact].get( 'update_balrog_manifest') # For partials from_buildid = manifest['mapping'][locale][artifact].get( 'from_buildid') beets.append( asyncio.ensure_future( move_beet(context, source, destinations, locale=locale, update_balrog_manifest=balrog_manifest, from_buildid=from_buildid, artifact_pretty_name=artifact_pretty_name))) await raise_future_exceptions(beets) # Fix up balrog manifest. We need an entry with both completes and # partials, which is why we store up the data from each moved beet # and collate it now. for locale in context.raw_balrog_manifest: balrog_entry = enrich_balrog_manifest(context, locale) balrog_entry['completeInfo'] = context.raw_balrog_manifest[locale][ 'completeInfo'] if 'partialInfo' in context.raw_balrog_manifest[locale]: balrog_entry['partialInfo'] = context.raw_balrog_manifest[locale][ 'partialInfo'] context.balrog_manifest.append(balrog_entry)