def _build_asset(
genome,
asset_key,
tag,
build_pkg,
genome_outfolder,
specific_args,
specific_params,
alias,
**kwargs,
):
"""
Builds assets with pypiper and updates a genome config file.
This function actually runs the build commands in a given build package,
and then update the refgenie config file.
:param str genome: The assembly key; e.g. 'mm10'.
:param str asset_key: The unique asset identifier; e.g. 'bowtie2_index'
:param dict build_pkg: A dict (see examples) specifying lists
of required input_assets, commands to run, and outputs to register as
assets.
"""
if args.map:
# Performing a build map step.
# The reduce step will need to be performed to get the built
# asset metadata to the master config file
genome_alias = rgc.get_genome_alias(digest=genome)
# create an empty config file in the genome directory
_LOGGER.info(f"Using new map genome config: {locked_map_gencfg}")
make_sure_path_exists(os.path.dirname(locked_map_gencfg))
open(locked_map_gencfg, "a").close()
# initialize a new RefGenConf.
# Use the master location for data storage,
# but change path to the in asset dir location
rgc_map = RefGenConf(
entries={"genome_folder": rgc.genome_folder},
filepath=locked_map_gencfg,
)
# set the alias first (if available), based on the master file
rgc_map.set_genome_alias(
digest=genome,
genome=genome_alias,
create_genome=True,
)
# copy the genome of interest section to the new RefGenConf,
# so that possible dependancies can be satisfied
rgc_map.update_genomes(
genome=genome_alias,
data=rgc[CFG_GENOMES_KEY][genome],
)
else:
rgc_map = rgc
_LOGGER.info(
f"Saving outputs to:{block_iter_repr(['content: ' + genome_outfolder, 'logs: ' + build_stats_dir])}"
)
if args.docker:
# Set up some docker stuff
if args.volumes:
# TODO: is volumes list defined here?
volumes = volumes.append(genome_outfolder)
else:
volumes = genome_outfolder
if not _writeable(genome_outfolder):
_LOGGER.error(
f"Insufficient permissions to write to output folder: {genome_outfolder}"
)
return False, rgc_map
pm = pypiper.PipelineManager(name=PKG_NAME,
outfolder=build_stats_dir,
args=args)
tk = pypiper.NGSTk(pm=pm)
if args.docker:
pm.get_container(build_pkg[CONT], volumes)
_LOGGER.debug("Asset build package: " + str(build_pkg))
# create a bundle list to simplify calls below
gat = [genome, asset_key, tag]
# collect variables required to populate the command templates
asset_vars = get_asset_vars(
genome,
asset_key,
tag,
genome_outfolder,
specific_args,
specific_params,
**kwargs,
)
# populate command templates
# prior to populating, remove any seek_key parts from the keys, since these are not supported by format method
command_list_populated = [
x.format(**{k.split(".")[0]: v
for k, v in asset_vars.items()})
for x in build_pkg[CMD_LST]
]
# create output directory
tk.make_dir(asset_vars["asset_outfolder"])
target = os.path.join(build_stats_dir,
TEMPLATE_TARGET.format(genome, asset_key, tag))
# add target command
command_list_populated.append("touch {target}".format(target=target))
_LOGGER.debug("Command populated: '{}'".format(
" ".join(command_list_populated)))
try:
# run build command
signal.signal(signal.SIGINT, _handle_sigint(gat))
pm.run(command_list_populated, target, container=pm.container)
except pypiper.exceptions.SubprocessError:
_LOGGER.error("asset '{}' build failed".format(asset_key))
return False, rgc_map
else:
# save build recipe to the JSON-formatted file
recipe_file_name = TEMPLATE_RECIPE_JSON.format(asset_key, tag)
with open(os.path.join(build_stats_dir, recipe_file_name),
"w") as outfile:
json.dump(build_pkg, outfile)
# since the assets are always built to a standard dir structure, we
# can just stitch a path together for asset digest calculation
asset_dir = os.path.join(rgc_map.data_dir, *gat)
if not os.path.exists(asset_dir):
raise OSError("Could not compute asset digest. Path does not "
"exist: {}".format(asset_dir))
digest = get_dir_digest(asset_dir)
_LOGGER.info(f"Asset digest: {digest}")
# add a 'dir' seek_key that points to the asset directory
build_pkg[ASSETS].update({"dir": "."})
# add updates to config file
with rgc_map as r:
if asset_key == "fasta":
r.update_genomes(genome,
data={CFG_ALIASES_KEY: [alias]},
force_digest=genome)
r.update_assets(
*gat[0:2],
data={CFG_ASSET_DESC_KEY: build_pkg[DESC]},
force_digest=genome,
)
r.update_tags(
*gat,
force_digest=genome,
data={
CFG_ASSET_PATH_KEY: asset_key,
CFG_ASSET_CHECKSUM_KEY: digest,
},
)
r.update_seek_keys(
*gat,
force_digest=genome,
keys={
k: v.format(**asset_vars)
for k, v in build_pkg[ASSETS].items()
},
)
r.set_default_pointer(*gat, force_digest=genome)
pm.stop_pipeline()
return True, rgc_map
def archive(rgc, registry_paths, force, remove, cfg_path, genomes_desc):
"""
Takes the RefGenConf object and builds individual tar archives
that can be then served with 'refgenieserver serve'. Additionally determines their md5 checksums, file sizes and
updates the original refgenie config with these data. If the --asset and/or --genome options are used (specific
build is requested) the archiver will check for the existence of config file saved in the path provided in
`genome_server` in the original config and update it so that no archive metadata is lost
:param RefGenConf rgc: configuration object with the data to build the servable archives for
:param list[dict] registry_paths: a collection of mappings that identifies the assets to update
:param bool force: whether to force the build of archive, regardless of its existence
:param bool remove: whether remove specified genome/asset:tag from the archive
:param str cfg_path: config file path
"""
if float(rgc[CFG_VERSION_KEY]) < float(REQ_CFG_VERSION):
raise ConfigNotCompliantError(
"You need to update the genome config to v{} in order to use the archiver. "
"The required version can be generated with refgenie >= {}".format(
REQ_CFG_VERSION, REFGENIE_BY_CFG[REQ_CFG_VERSION]))
if CFG_ARCHIVE_CONFIG_KEY in rgc:
srp = rgc[CFG_ARCHIVE_CONFIG_KEY]
server_rgc_path = srp if os.path.isabs(srp) \
else os.path.join(os.path.dirname(rgc.file_path), srp)
else:
try:
server_rgc_path = os.path.join(rgc[CFG_ARCHIVE_KEY],
os.path.basename(cfg_path))
except KeyError:
raise GenomeConfigFormatError(
"The config '{}' is missing a {} entry. Can't determine the desired archive."
.format(cfg_path,
" or ".join([CFG_ARCHIVE_KEY, CFG_ARCHIVE_KEY_OLD])))
if os.path.isfile(server_rgc_path) and not os.access(
server_rgc_path, os.W_OK):
raise OSError(
"The determined archive config path is not writable: {}".format(
server_rgc_path))
if force:
_LOGGER.info("Build forced; file existence will be ignored")
if os.path.exists(server_rgc_path):
_LOGGER.debug("'{}' file was found and will be updated".format(
server_rgc_path))
_LOGGER.debug("Registry_paths: {}".format(registry_paths))
# original RefGenConf has been created in read-only mode,
# make it RW compatible and point to new target path for server use or initialize a new object
if os.path.exists(server_rgc_path):
rgc_server = RefGenConf(filepath=server_rgc_path)
if remove:
if not registry_paths:
_LOGGER.error(
"To remove archives you have to specify them. Use 'asset_registry_path' argument."
)
exit(1)
with rgc_server as r:
_remove_archive(r, registry_paths, CFG_ARCHIVE_KEY)
exit(0)
else:
if remove:
_LOGGER.error(
"You can't remove archives since the genome_archive path does not exist yet."
)
exit(1)
rgc_server = RefGenConf(filepath=rgc.file_path)
rgc_server.make_writable(filepath=server_rgc_path)
rgc_server.make_readonly()
if registry_paths:
genomes = _get_paths_element(registry_paths, "namespace")
asset_list = _get_paths_element(registry_paths, "item")
tag_list = _get_paths_element(registry_paths, "tag")
else:
genomes = rgc.genomes_list()
asset_list, tag_list = None, None
if not genomes:
_LOGGER.error("No genomes found")
exit(1)
else:
_LOGGER.debug("Genomes to be processed: {}".format(str(genomes)))
if genomes_desc is not None:
if os.path.exists(genomes_desc):
import csv
_LOGGER.info("Found a genomes descriptions CSV file: {}".format(
genomes_desc))
with open(genomes_desc, mode='r') as infile:
reader = csv.reader(infile)
descs = {rows[0]: rows[1] for rows in reader}
else:
_LOGGER.error(
"Genomes descriptions CSV file does not exist: {}".format(
genomes_desc))
sys.exit(1)
counter = 0
for genome in genomes:
genome_dir = os.path.join(rgc[CFG_FOLDER_KEY], genome)
target_dir = os.path.join(rgc[CFG_ARCHIVE_KEY], genome)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
genome_desc = rgc[CFG_GENOMES_KEY][genome].setdefault(CFG_GENOME_DESC_KEY, DESC_PLACEHOLDER) \
if genomes_desc is None or genome not in descs else descs[genome]
genome_checksum = rgc[CFG_GENOMES_KEY][genome].\
setdefault(CFG_CHECKSUM_KEY, CHECKSUM_PLACEHOLDER)
genome_attrs = {
CFG_GENOME_DESC_KEY: genome_desc,
CFG_CHECKSUM_KEY: genome_checksum
}
rgc_server.update_genomes(genome, genome_attrs)
_LOGGER.debug("Updating '{}' genome attributes...".format(genome))
asset = asset_list[counter] if asset_list is not None else None
assets = asset or rgc[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY].keys()
if not assets:
_LOGGER.error("No assets found")
continue
else:
_LOGGER.debug("Assets to be processed: {}".format(str(assets)))
for asset_name in assets if isinstance(assets, list) else [assets]:
asset_desc = rgc[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY][asset_name]\
.setdefault(CFG_ASSET_DESC_KEY, DESC_PLACEHOLDER)
default_tag = rgc[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY][asset_name]\
.setdefault(CFG_ASSET_DEFAULT_TAG_KEY, DEFAULT_TAG)
asset_attrs = {
CFG_ASSET_DESC_KEY: asset_desc,
CFG_ASSET_DEFAULT_TAG_KEY: default_tag
}
_LOGGER.debug("Updating '{}/{}' asset attributes...".format(
genome, asset_name))
with rgc_server as r:
r.update_assets(genome, asset_name, asset_attrs)
tag = tag_list[counter] if tag_list is not None else None
tags = tag or rgc[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY][
asset_name][CFG_ASSET_TAGS_KEY].keys()
for tag_name in tags if isinstance(tags, list) else [tags]:
if not rgc.is_asset_complete(genome, asset_name, tag_name):
raise MissingConfigDataError(
"Asset '{}/{}:{}' is incomplete. This probably means an"
" attempt to archive a partially pulled parent. "
"refgenieserver archive requires all assets to be built"
" prior to archiving.".format(genome, asset_name,
tag_name))
file_name = rgc[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY][
asset_name][CFG_ASSET_TAGS_KEY][tag_name][
CFG_ASSET_PATH_KEY]
target_file = os.path.join(
target_dir, "{}__{}".format(asset_name, tag_name) + ".tgz")
input_file = os.path.join(genome_dir, file_name, tag_name)
# these attributes have to be read from the original RefGenConf in case the archiver just increments
# an existing server RefGenConf
parents = rgc[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY][asset_name][CFG_ASSET_TAGS_KEY][tag_name]. \
setdefault(CFG_ASSET_PARENTS_KEY, [])
children = rgc[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY][asset_name][CFG_ASSET_TAGS_KEY][tag_name]. \
setdefault(CFG_ASSET_CHILDREN_KEY, [])
seek_keys = rgc[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY][asset_name][CFG_ASSET_TAGS_KEY][tag_name]. \
setdefault(CFG_SEEK_KEYS_KEY, {})
asset_digest = rgc[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY][asset_name][CFG_ASSET_TAGS_KEY][tag_name]. \
setdefault(CFG_ASSET_CHECKSUM_KEY, None)
if not os.path.exists(target_file) or force:
_LOGGER.info(
"Creating archive '{}' from '{}' asset".format(
target_file, input_file))
try:
_check_tgz(input_file, target_file, asset_name)
_copy_recipe(input_file, target_dir, asset_name,
tag_name)
_copy_log(input_file, target_dir, asset_name, tag_name)
except OSError as e:
_LOGGER.warning(e)
continue
else:
_LOGGER.info(
"Updating '{}/{}:{}' tag attributes...".format(
genome, asset_name, tag_name))
tag_attrs = {
CFG_ASSET_PATH_KEY: file_name,
CFG_SEEK_KEYS_KEY: seek_keys,
CFG_ARCHIVE_CHECKSUM_KEY: checksum(target_file),
CFG_ARCHIVE_SIZE_KEY: size(target_file),
CFG_ASSET_SIZE_KEY: size(input_file),
CFG_ASSET_PARENTS_KEY: parents,
CFG_ASSET_CHILDREN_KEY: children,
CFG_ASSET_CHECKSUM_KEY: asset_digest
}
_LOGGER.debug("attr dict: {}".format(tag_attrs))
with rgc_server as r:
for parent in parents:
# here we update any pre-existing parents' children attr with the newly added asset
_LOGGER.debug(
"Updating {} children list with {}".format(
parent, "{}/{}:{}".format(
genome, asset_name, tag_name)))
rp = parse_registry_path(parent)
parent_genome = rp["namespace"]
parent_asset = rp["item"]
parent_tag = rp["tag"]
try:
r.get_asset(parent_genome, parent_asset,
parent_tag)
except RefgenconfError:
_LOGGER.warning(
"'{}/{}:{}'s parent '{}' does not exist, "
"skipping relationship updates".format(
genome, asset_name, tag_name,
parent))
continue
r.update_relatives_assets(
parent_genome,
parent_asset,
parent_tag, [
"{}/{}:{}".format(
genome, asset_name, tag_name)
],
children=True)
r.update_tags(genome, asset_name, tag_name,
tag_attrs)
else:
_LOGGER.debug("'{}' exists".format(target_file))
counter += 1
with rgc_server as r:
_purge_nonservable(r)
_LOGGER.info(
"Builder finished; server config file saved to: '{}'".format(
r.write(server_rgc_path)))