def test_pull_asset_works_with_nonwritable_and_writable_rgc(
cfg_file, gname, aname, tname, state
):
rgc = RefGenConf(filepath=cfg_file, writable=state)
remove_asset_and_file(rgc, gname, aname, tname)
print("\nPulling; genome: {}, asset: {}, tag: {}\n".format(gname, aname, tname))
with mock.patch("refgenconf.refgenconf.query_yes_no", return_value=True):
rgc.pull(gname, aname, tname)
if state:
rgc.make_readonly()
def test_pull_asset_updates_genome_config(cfg_file, gname, aname, tname):
"""
Test that the object that was identical prior to the asset pull differs afterwards
and the pulled asset metadata has been written to the config file
"""
ori_rgc = RefGenConf(filepath=cfg_file, writable=False)
rgc = RefGenConf(filepath=cfg_file, writable=False)
remove_asset_and_file(rgc, gname, aname, tname)
remove_asset_and_file(ori_rgc, gname, aname, tname)
# ori_rgc.remove_assets(gname, aname, tname)
assert ori_rgc.to_dict() == rgc.to_dict()
with mock.patch("refgenconf.refgenconf.query_yes_no", return_value=True):
print("\nPulling; genome: {}, asset: {}, tag: {}\n".format(gname, aname, tname))
rgc.pull(gname, aname, tname)
assert not ori_rgc.to_dict() == rgc.to_dict()
post_rgc = RefGenConf(filepath=cfg_file, writable=False)
assert isinstance(post_rgc.seek(gname, aname, tname), str)
def main():
""" Primary workflow """
parser = logmuse.add_logging_options(build_argparser())
args, remaining_args = parser.parse_known_args()
global _LOGGER
_LOGGER = logmuse.logger_via_cli(args, make_root=True)
_LOGGER.debug(f"versions: refgenie {__version__} | refgenconf {rgc_version}")
_LOGGER.debug(f"Args: {args}")
if not args.command:
parser.print_help()
_LOGGER.error("No command given")
sys.exit(1)
if args.command == ALIAS_CMD and not args.subcommand:
parser.print_help()
_LOGGER.error("No alias subcommand command given")
sys.exit(1)
gencfg = select_genome_config(
filename=args.genome_config,
check_exist=not args.command == INIT_CMD,
on_missing=lambda fp: fp,
strict_env=True,
)
if gencfg is None:
raise MissingGenomeConfigError(args.genome_config)
_LOGGER.debug("Determined genome config: {}".format(gencfg))
skip_read_lock = _skip_lock(args.skip_read_lock, gencfg)
# From user input we want to construct a list of asset dicts, where each
# asset has a genome name, asset name, and tag
if "asset_registry_paths" in args and args.asset_registry_paths:
_LOGGER.debug("Found registry_path: {}".format(args.asset_registry_paths))
asset_list = [parse_registry_path(x) for x in args.asset_registry_paths]
for a in asset_list:
# every asset must have a genome, either provided via registry path
# or the args.genome arg.
if not a["genome"]:
if args.genome:
a["genome"] = args.genome
else:
_LOGGER.error(
"Provided asset registry path ({}/{}:{}) is invalid. See help for usage reference.".format(
a["genome"], a["asset"], a["tag"]
)
)
sys.exit(1)
else:
if args.genome and args.genome != a["genome"]:
_LOGGER.warn(
"Two different genomes specified for asset '{}'.".format(
a["asset"]
)
)
else:
if args.command in GENOME_ONLY_REQUIRED and not args.genome:
parser.error("You must provide either a genome or a registry path")
sys.exit(1)
if args.command in ASSET_REQUIRED:
parser.error("You must provide an asset registry path")
sys.exit(1)
if args.command == INIT_CMD:
_LOGGER.debug("Initializing refgenie genome configuration")
entries = OrderedDict(
{
CFG_VERSION_KEY: REQ_CFG_VERSION,
CFG_FOLDER_KEY: os.path.dirname(os.path.abspath(gencfg)),
CFG_SERVERS_KEY: args.genome_server or [DEFAULT_SERVER],
CFG_GENOMES_KEY: None,
}
)
if args.settings_json:
if os.path.isfile(args.settings_json):
with open(args.settings_json, "r") as json_file:
data = json.load(json_file)
entries.update(data)
else:
raise FileNotFoundError(
"JSON file with config init settings does not exist: {}".format(
args.settings_json
)
)
if args.genome_folder:
entries.update({CFG_FOLDER_KEY: args.genome_folder})
if args.remote_url_base:
entries.update({CFG_REMOTE_URL_BASE_KEY: args.remote_url_base})
if args.genome_archive_folder:
entries.update({CFG_ARCHIVE_KEY: args.genome_archive_folder})
if args.genome_archive_config:
entries.update({CFG_ARCHIVE_CONFIG_KEY: args.genome_archive_config})
_LOGGER.debug("initializing with entries: {}".format(entries))
rgc = RefGenConf(entries=entries, skip_read_lock=skip_read_lock)
rgc.initialize_config_file(os.path.abspath(gencfg))
elif args.command == BUILD_CMD:
if not all([x["genome"] == asset_list[0]["genome"] for x in asset_list]):
_LOGGER.error("Build can only build assets for one genome")
sys.exit(1)
recipe_name = None
if args.recipe:
if len(asset_list) > 1:
_LOGGER.error("Recipes cannot be specified for multi-asset builds")
sys.exit(1)
recipe_name = args.recipe
if args.requirements:
for a in asset_list:
recipe = recipe_name or a["asset"]
if recipe not in asset_build_packages.keys():
_raise_missing_recipe_error(recipe)
_LOGGER.info("'{}' recipe requirements: ".format(recipe))
_make_asset_build_reqs(recipe)
sys.exit(0)
refgenie_build(gencfg, asset_list[0]["genome"], asset_list, recipe_name, args)
elif args.command == GET_ASSET_CMD:
rgc = RefGenConf(filepath=gencfg, writable=False, skip_read_lock=skip_read_lock)
check = args.check_exists if args.check_exists else None
for a in asset_list:
_LOGGER.debug(
"getting asset: '{}/{}.{}:{}'".format(
a["genome"], a["asset"], a["seek_key"], a["tag"]
)
)
print(
rgc.seek(
a["genome"],
a["asset"],
a["tag"],
a["seek_key"],
strict_exists=check,
)
)
return
elif args.command == INSERT_CMD:
rgc = RefGenConf(filepath=gencfg, writable=False, skip_read_lock=skip_read_lock)
if len(asset_list) > 1:
raise NotImplementedError("Can only add 1 asset at a time")
else:
sk = args.seek_keys
if sk:
sk = json.loads(args.seek_keys)
rgc.add(
path=args.path,
genome=asset_list[0]["genome"],
asset=asset_list[0]["asset"],
tag=asset_list[0]["tag"],
seek_keys=sk,
force=args.force,
)
elif args.command == PULL_CMD:
rgc = RefGenConf(filepath=gencfg, writable=False, skip_read_lock=skip_read_lock)
# existing assets overwriting
if args.no_overwrite:
force = False
elif args.force_overwrite:
force = True
else:
force = None
# large archive pulling
if args.no_large:
force_large = False
elif args.pull_large:
force_large = True
else:
force_large = None
# batch mode takes precedence over other choices
if args.batch:
force_large = True
force = False
outdir = rgc.data_dir
if not os.path.exists(outdir):
raise MissingFolderError(outdir)
if not perm_check_x(outdir):
return
if not _single_folder_writeable(outdir):
_LOGGER.error("Insufficient permissions to write to: {}".format(outdir))
return
for a in asset_list:
rgc.pull(
a["genome"],
a["asset"],
a["tag"],
force=force,
force_large=force_large,
size_cutoff=args.size_cutoff,
)
elif args.command in [LIST_LOCAL_CMD, LIST_REMOTE_CMD]:
rgc = RefGenConf(filepath=gencfg, writable=False, skip_read_lock=skip_read_lock)
console = Console()
if args.command == LIST_REMOTE_CMD:
num_servers = 0
bad_servers = []
for server_url in rgc[CFG_SERVERS_KEY]:
num_servers += 1
try:
table = rgc.get_asset_table(
genomes=args.genome, server_url=server_url
)
except (DownloadJsonError, ConnectionError, MissingSchema):
bad_servers.append(server_url)
continue
else:
console.print(table)
if num_servers >= len(rgc[CFG_SERVERS_KEY]) and bad_servers:
_LOGGER.error(
"Could not list assets from the following servers: {}".format(
bad_servers
)
)
else:
if args.recipes:
print(", ".join(sorted(list(asset_build_packages.keys()))))
else:
console.print(rgc.get_asset_table(genomes=args.genome))
elif args.command == GETSEQ_CMD:
rgc = RefGenConf(filepath=gencfg, writable=False, skip_read_lock=skip_read_lock)
print(rgc.getseq(args.genome, args.locus))
elif args.command == REMOVE_CMD:
force = args.force
rgc = RefGenConf(filepath=gencfg, skip_read_lock=skip_read_lock)
for a in asset_list:
a["tag"] = a["tag"] or rgc.get_default_tag(
a["genome"], a["asset"], use_existing=False
)
_LOGGER.debug("Determined tag for removal: {}".format(a["tag"]))
if a["seek_key"] is not None:
raise NotImplementedError("You can't remove a specific seek_key.")
gat = {"genome": a["genome"], "asset": a["asset"], "tag": a["tag"]}
try:
if not rgc.is_asset_complete(**gat):
with rgc as r:
r.cfg_remove_assets(**gat)
_LOGGER.info(
"Removed an incomplete asset "
"'{genome}/{asset}:{tag}'".format(*gat)
)
return
except (KeyError, MissingAssetError, MissingGenomeError):
_LOGGER.info(
"Asset '{genome}/{asset}:{tag}' does not exist".format(**gat)
)
return
if len(asset_list) > 1:
if not query_yes_no(
"Are you sure you want to remove {} assets?".format(len(asset_list))
):
_LOGGER.info("Action aborted by the user")
return
force = True
for a in asset_list:
rgc.remove(genome=a["genome"], asset=a["asset"], tag=a["tag"], force=force)
elif args.command == TAG_CMD:
rgc = RefGenConf(filepath=gencfg, skip_read_lock=skip_read_lock)
if len(asset_list) > 1:
raise NotImplementedError("Can only tag 1 asset at a time")
if args.default:
# set the default tag and exit
with rgc as r:
r.set_default_pointer(a["genome"], a["asset"], a["tag"], True)
sys.exit(0)
rgc.tag(a["genome"], a["asset"], a["tag"], args.tag, force=args.force)
elif args.command == ID_CMD:
rgc = RefGenConf(filepath=gencfg, writable=False, skip_read_lock=skip_read_lock)
if len(asset_list) == 1:
g, a = asset_list[0]["genome"], asset_list[0]["asset"]
t = asset_list[0]["tag"] or rgc.get_default_tag(g, a)
print(rgc.id(g, a, t))
return
for asset in asset_list:
g, a = asset["genome"], asset["asset"]
t = asset["tag"] or rgc.get_default_tag(g, a)
print("{}/{}:{},".format(g, a, t) + rgc.id(g, a, t))
return
elif args.command == SUBSCRIBE_CMD:
rgc = RefGenConf(filepath=gencfg, writable=False, skip_read_lock=skip_read_lock)
rgc.subscribe(urls=args.genome_server, reset=args.reset)
return
elif args.command == UNSUBSCRIBE_CMD:
rgc = RefGenConf(filepath=gencfg, writable=False, skip_read_lock=skip_read_lock)
rgc.unsubscribe(urls=args.genome_server)
return
elif args.command == ALIAS_CMD:
rgc = RefGenConf(filepath=gencfg, skip_read_lock=skip_read_lock)
if args.subcommand == ALIAS_GET_CMD:
if args.aliases is not None:
for a in args.aliases:
print(rgc.get_genome_alias_digest(alias=a))
return
console = Console()
console.print(rgc.genome_aliases_table)
if args.subcommand == ALIAS_SET_CMD:
rgc.set_genome_alias(
digest=args.digest,
genome=args.aliases,
reset_digest=args.reset,
create_genome=args.force,
)
return
elif args.subcommand == ALIAS_REMOVE_CMD:
rgc.remove_genome_aliases(digest=args.digest, aliases=args.aliases)
return
elif args.command == COMPARE_CMD:
rgc = RefGenConf(filepath=gencfg, writable=False, skip_read_lock=skip_read_lock)
res = rgc.compare(
args.genome1[0], args.genome2[0], explain=not args.no_explanation
)
if args.no_explanation:
print(res)
elif args.command == UPGRADE_CMD:
upgrade_config(
target_version=args.target_version, filepath=gencfg, force=args.force
)
def main():
""" Primary workflow """
parser = logmuse.add_logging_options(build_argparser())
args, remaining_args = parser.parse_known_args()
global _LOGGER
_LOGGER = logmuse.logger_via_cli(args, make_root=True)
_LOGGER.debug("refgenie {}".format(__version__))
_LOGGER.debug("Args: {}".format(args))
if not args.command:
parser.print_help()
_LOGGER.error("No command given")
sys.exit(1)
gencfg = refgenconf.select_genome_config(
filename=args.genome_config,
check_exist=not args.command == INIT_CMD,
on_missing=lambda fp: fp,
strict_env=True)
if gencfg is None:
raise MissingGenomeConfigError(args.genome_config)
_LOGGER.debug("Determined genome config: {}".format(gencfg))
# From user input we want to construct a list of asset dicts, where each
# asset has a genome name, asset name, and tag
if "asset_registry_paths" in args and args.asset_registry_paths:
_LOGGER.debug("Found registry_path: {}".format(
args.asset_registry_paths))
asset_list = [
parse_registry_path(x) for x in args.asset_registry_paths
]
for a in asset_list:
# every asset must have a genome, either provided via registry path
# or the args.genome arg.
if not a["genome"]:
if args.genome:
a["genome"] = args.genome
else:
_LOGGER.error(
"Provided asset registry path ({}/{}:{}) is invalid. See help for usage reference."
.format(a["genome"], a["asset"], a["tag"]))
sys.exit(1)
else:
if args.genome and args.genome != a["genome"]:
_LOGGER.warn(
"Two different genomes specified for asset '{}'.".
format(a["asset"]))
else:
if args.command in GENOME_ONLY_REQUIRED and not args.genome:
parser.error("You must provide either a genome or a registry path")
sys.exit(1)
if args.command in ASSET_REQUIRED:
parser.error("You must provide an asset registry path")
sys.exit(1)
if args.command == INIT_CMD:
_LOGGER.debug("Initializing refgenie genome configuration")
rgc = RefGenConf(entries=OrderedDict(
{
CFG_VERSION_KEY: REQ_CFG_VERSION,
CFG_FOLDER_KEY: os.path.dirname(os.path.abspath(gencfg)),
CFG_SERVERS_KEY: args.genome_server or [DEFAULT_SERVER],
CFG_GENOMES_KEY: None
}))
rgc.initialize_config_file(os.path.abspath(gencfg))
elif args.command == BUILD_CMD:
if not all(
[x["genome"] == asset_list[0]["genome"] for x in asset_list]):
_LOGGER.error("Build can only build assets for one genome")
sys.exit(1)
recipe_name = None
if args.recipe:
if len(asset_list) > 1:
_LOGGER.error(
"Recipes cannot be specified for multi-asset builds")
sys.exit(1)
recipe_name = args.recipe
if args.requirements:
for a in asset_list:
recipe = recipe_name or a["asset"]
if recipe not in asset_build_packages.keys():
_raise_missing_recipe_error(recipe)
_LOGGER.info("'{}' recipe requirements: ".format(recipe))
_make_asset_build_reqs(recipe)
sys.exit(0)
refgenie_build(gencfg, asset_list[0]["genome"], asset_list,
recipe_name, args)
elif args.command == GET_ASSET_CMD:
rgc = RefGenConf(filepath=gencfg, writable=False)
check = args.check_exists if args.check_exists else None
for a in asset_list:
_LOGGER.debug("getting asset: '{}/{}.{}:{}'".format(
a["genome"], a["asset"], a["seek_key"], a["tag"]))
print(
rgc.seek(a["genome"],
a["asset"],
a["tag"],
a["seek_key"],
strict_exists=check))
return
elif args.command == INSERT_CMD:
rgc = RefGenConf(filepath=gencfg, writable=False)
if len(asset_list) > 1:
raise NotImplementedError("Can only add 1 asset at a time")
else:
refgenie_add(rgc, asset_list[0], args.path, args.force)
elif args.command == PULL_CMD:
rgc = RefGenConf(filepath=gencfg, writable=False)
force = None if not args.force else True
outdir = rgc[CFG_FOLDER_KEY]
if not os.path.exists(outdir):
raise MissingFolderError(outdir)
target = _key_to_name(CFG_FOLDER_KEY)
if not perm_check_x(outdir, target):
return
if not _single_folder_writeable(outdir):
_LOGGER.error("Insufficient permissions to write to {}: {}".format(
target, outdir))
return
for a in asset_list:
rgc.pull(a["genome"],
a["asset"],
a["tag"],
unpack=not args.no_untar,
force=force)
elif args.command in [LIST_LOCAL_CMD, LIST_REMOTE_CMD]:
rgc = RefGenConf(filepath=gencfg, writable=False)
if args.command == LIST_REMOTE_CMD:
num_servers = 0
# Keep all servers so that child updates maintain server list
server_list = rgc[CFG_SERVERS_KEY]
bad_servers = []
for server_url in rgc[CFG_SERVERS_KEY]:
num_servers += 1
try:
rgc[CFG_SERVERS_KEY] = server_url
pfx, genomes, assets, recipes = _exec_list(
rgc, args.command == LIST_REMOTE_CMD, args.genome)
if assets is None and genomes is None:
continue
_LOGGER.info("{} genomes: {}".format(pfx, genomes))
if args.command != LIST_REMOTE_CMD: # Not implemented yet
_LOGGER.info("{} recipes: {}".format(pfx, recipes))
_LOGGER.info("{} assets:\n{}\n".format(pfx, assets))
except (DownloadJsonError, ConnectionError):
bad_servers.append(server_url)
continue
if num_servers >= len(server_list) and bad_servers:
_LOGGER.error(
"Could not list assets from the following server(s): {}".
format(bad_servers))
# Restore original server list, even when we couldn't find assets on a server
rgc[CFG_SERVERS_KEY] = server_list
else: # Only check local assets once
_LOGGER.info("Server subscriptions: {}".format(", ".join(
rgc[CFG_SERVERS_KEY])))
pfx, genomes, assets, recipes = _exec_list(
rgc, args.command == LIST_REMOTE_CMD, args.genome)
_LOGGER.info("{} genomes: {}".format(pfx, genomes))
if args.command != LIST_REMOTE_CMD: # Not implemented yet
_LOGGER.info("{} recipes: {}".format(pfx, recipes))
_LOGGER.info("{} assets:\n{}".format(pfx, assets))
elif args.command == GETSEQ_CMD:
rgc = RefGenConf(filepath=gencfg, writable=False)
rgc.getseq(rgc, args.genome, args.locus)
elif args.command == REMOVE_CMD:
force = args.force
rgc = RefGenConf(filepath=gencfg)
for a in asset_list:
a["tag"] = a["tag"] or rgc.get_default_tag(
a["genome"], a["asset"], use_existing=False)
_LOGGER.debug("Determined tag for removal: {}".format(a["tag"]))
if a["seek_key"] is not None:
raise NotImplementedError(
"You can't remove a specific seek_key.")
bundle = [a["genome"], a["asset"], a["tag"]]
try:
if not rgc.is_asset_complete(*bundle):
with rgc as r:
r.cfg_remove_assets(*bundle)
_LOGGER.info(
"Removed an incomplete asset '{}/{}:{}'".format(
*bundle))
return
except (KeyError, MissingAssetError, MissingGenomeError):
_LOGGER.info("Asset '{}/{}:{}' does not exist".format(*bundle))
return
if len(asset_list) > 1:
if not query_yes_no(
"Are you sure you want to remove {} assets?".format(
len(asset_list))):
_LOGGER.info("Action aborted by the user")
return
force = True
for a in asset_list:
rgc.remove(genome=a["genome"],
asset=a["asset"],
tag=a["tag"],
force=force)
elif args.command == TAG_CMD:
rgc = RefGenConf(filepath=gencfg)
if len(asset_list) > 1:
raise NotImplementedError("Can only tag 1 asset at a time")
if args.default:
# set the default tag and exit
with rgc as r:
r.set_default_pointer(a["genome"], a["asset"], a["tag"], True)
sys.exit(0)
rgc.tag(a["genome"], a["asset"], a["tag"], args.tag)
elif args.command == ID_CMD:
rgc = RefGenConf(filepath=gencfg, writable=False)
if len(asset_list) == 1:
g, a = asset_list[0]["genome"], asset_list[0]["asset"]
t = asset_list[0]["tag"] or rgc.get_default_tag(g, a)
print(rgc.id(g, a, t))
return
for asset in asset_list:
g, a = asset["genome"], asset["asset"]
t = asset["tag"] or rgc.get_default_tag(g, a)
print("{}/{}:{},".format(g, a, t) + rgc.id(g, a, t))
return
elif args.command == SUBSCRIBE_CMD:
rgc = RefGenConf(filepath=gencfg, writable=False)
rgc.subscribe(urls=args.genome_server, reset=args.reset)
return
elif args.command == UNSUBSCRIBE_CMD:
rgc = RefGenConf(filepath=gencfg, writable=False)
rgc.unsubscribe(urls=args.genome_server)
return
def refgenie_build(gencfg, genome, asset_list, recipe_name, args):
"""
Runs the refgenie build recipe.
:param str gencfg: path to the genome configuration file
:param str genome:
:param list asset_list:
:param str recipe_name:
:param argparse.Namespace args: parsed command-line options/arguments
"""
rgc = RefGenConf(
filepath=gencfg,
writable=False,
skip_read_lock=_skip_lock(args.skip_read_lock, gencfg),
)
specified_args = _parse_user_build_input(args.files)
specified_params = _parse_user_build_input(args.params)
def _read_json_file(filepath):
"""
Read a JSON file
:param str filepath: path to the file to read
:return dict: read data
"""
with open(filepath, "r") as f:
data = json.load(f)
return data
if recipe_name and os.path.isfile(recipe_name) and recipe_name.endswith(
".json"):
recipe_name = _read_json_file(filepath=recipe_name)
def _build_asset(
genome,
asset_key,
tag,
build_pkg,
genome_outfolder,
specific_args,
specific_params,
alias,
**kwargs,
):
"""
Builds assets with pypiper and updates a genome config file.
This function actually runs the build commands in a given build package,
and then update the refgenie config file.
:param str genome: The assembly key; e.g. 'mm10'.
:param str asset_key: The unique asset identifier; e.g. 'bowtie2_index'
:param dict build_pkg: A dict (see examples) specifying lists
of required input_assets, commands to run, and outputs to register as
assets.
"""
if args.map:
# Performing a build map step.
# The reduce step will need to be performed to get the built
# asset metadata to the master config file
genome_alias = rgc.get_genome_alias(digest=genome)
# create an empty config file in the genome directory
_LOGGER.info(f"Using new map genome config: {locked_map_gencfg}")
make_sure_path_exists(os.path.dirname(locked_map_gencfg))
open(locked_map_gencfg, "a").close()
# initialize a new RefGenConf.
# Use the master location for data storage,
# but change path to the in asset dir location
rgc_map = RefGenConf(
entries={"genome_folder": rgc.genome_folder},
filepath=locked_map_gencfg,
)
# set the alias first (if available), based on the master file
rgc_map.set_genome_alias(
digest=genome,
genome=genome_alias,
create_genome=True,
)
# copy the genome of interest section to the new RefGenConf,
# so that possible dependancies can be satisfied
rgc_map.update_genomes(
genome=genome_alias,
data=rgc[CFG_GENOMES_KEY][genome],
)
else:
rgc_map = rgc
_LOGGER.info(
f"Saving outputs to:{block_iter_repr(['content: ' + genome_outfolder, 'logs: ' + build_stats_dir])}"
)
if args.docker:
# Set up some docker stuff
if args.volumes:
# TODO: is volumes list defined here?
volumes = volumes.append(genome_outfolder)
else:
volumes = genome_outfolder
if not _writeable(genome_outfolder):
_LOGGER.error(
f"Insufficient permissions to write to output folder: {genome_outfolder}"
)
return False, rgc_map
pm = pypiper.PipelineManager(name=PKG_NAME,
outfolder=build_stats_dir,
args=args)
tk = pypiper.NGSTk(pm=pm)
if args.docker:
pm.get_container(build_pkg[CONT], volumes)
_LOGGER.debug("Asset build package: " + str(build_pkg))
# create a bundle list to simplify calls below
gat = [genome, asset_key, tag]
# collect variables required to populate the command templates
asset_vars = get_asset_vars(
genome,
asset_key,
tag,
genome_outfolder,
specific_args,
specific_params,
**kwargs,
)
# populate command templates
# prior to populating, remove any seek_key parts from the keys, since these are not supported by format method
command_list_populated = [
x.format(**{k.split(".")[0]: v
for k, v in asset_vars.items()})
for x in build_pkg[CMD_LST]
]
# create output directory
tk.make_dir(asset_vars["asset_outfolder"])
target = os.path.join(build_stats_dir,
TEMPLATE_TARGET.format(genome, asset_key, tag))
# add target command
command_list_populated.append("touch {target}".format(target=target))
_LOGGER.debug("Command populated: '{}'".format(
" ".join(command_list_populated)))
try:
# run build command
signal.signal(signal.SIGINT, _handle_sigint(gat))
pm.run(command_list_populated, target, container=pm.container)
except pypiper.exceptions.SubprocessError:
_LOGGER.error("asset '{}' build failed".format(asset_key))
return False, rgc_map
else:
# save build recipe to the JSON-formatted file
recipe_file_name = TEMPLATE_RECIPE_JSON.format(asset_key, tag)
with open(os.path.join(build_stats_dir, recipe_file_name),
"w") as outfile:
json.dump(build_pkg, outfile)
# since the assets are always built to a standard dir structure, we
# can just stitch a path together for asset digest calculation
asset_dir = os.path.join(rgc_map.data_dir, *gat)
if not os.path.exists(asset_dir):
raise OSError("Could not compute asset digest. Path does not "
"exist: {}".format(asset_dir))
digest = get_dir_digest(asset_dir)
_LOGGER.info(f"Asset digest: {digest}")
# add a 'dir' seek_key that points to the asset directory
build_pkg[ASSETS].update({"dir": "."})
# add updates to config file
with rgc_map as r:
if asset_key == "fasta":
r.update_genomes(genome,
data={CFG_ALIASES_KEY: [alias]},
force_digest=genome)
r.update_assets(
*gat[0:2],
data={CFG_ASSET_DESC_KEY: build_pkg[DESC]},
force_digest=genome,
)
r.update_tags(
*gat,
force_digest=genome,
data={
CFG_ASSET_PATH_KEY: asset_key,
CFG_ASSET_CHECKSUM_KEY: digest,
},
)
r.update_seek_keys(
*gat,
force_digest=genome,
keys={
k: v.format(**asset_vars)
for k, v in build_pkg[ASSETS].items()
},
)
r.set_default_pointer(*gat, force_digest=genome)
pm.stop_pipeline()
return True, rgc_map
for a in asset_list:
asset_key = a["asset"]
asset_tag = a["tag"] or rgc.get_default_tag(
genome, a["asset"], use_existing=False)
recipe_name = recipe_name or asset_key
if isinstance(
recipe_name,
dict) or (isinstance(recipe_name, str)
and recipe_name in asset_build_packages.keys()):
if isinstance(recipe_name, dict):
_LOGGER.info("Using custom recipe: \n{}".format(recipe_name))
asset_build_package = _check_recipe(recipe_name)
recipe_name = asset_build_package["name"]
else:
asset_build_package = _check_recipe(
asset_build_packages[recipe_name])
# handle user-requested parents for the required assets
input_assets = {}
parent_assets = []
specified_asset_keys, specified_assets = None, None
if args.assets is not None:
parsed_parents_input = _parse_user_build_input(args.assets)
specified_asset_keys = list(parsed_parents_input.keys())
specified_assets = list(parsed_parents_input.values())
_LOGGER.debug(f"Custom assets requested: {args.assets}")
if not specified_asset_keys and isinstance(args.assets, list):
_LOGGER.warning(
"Specified parent assets format is invalid. Using defaults."
)
for req_asset in asset_build_package[REQ_ASSETS]:
req_asset_data = parse_registry_path(req_asset[KEY])
# for each req asset see if non-default parents were requested
if (specified_asset_keys is not None
and req_asset_data["asset"] in specified_asset_keys):
parent_data = parse_registry_path(
specified_assets[specified_asset_keys.index(
req_asset_data["asset"])])
g, a, t, s = (
parent_data["genome"],
parent_data["asset"],
parent_data["tag"]
or rgc.get_default_tag(genome, parent_data["asset"]),
parent_data["seek_key"],
)
else: # if no custom parents requested for the req asset, use default one
default = parse_registry_path(req_asset[DEFAULT])
g, a, t, s = (
genome,
default["asset"],
rgc.get_default_tag(genome, default["asset"]),
req_asset_data["seek_key"],
)
try:
parent_assets.append("{}/{}:{}".format(
rgc.get_genome_alias_digest(g, fallback=True), a, t))
except UndefinedAliasError as e:
_LOGGER.warning(
f"'{g}' namespace has not been initialized yet")
if args.pull_parents:
_LOGGER.info(f"Pulling missing parent: {g}/{a}:{t}")
ret = rgc.pull(genome=g, asset=a, tag=t)
if ret is None or not all(ret):
_LOGGER.info(
f"Missing parent asset pull requested, but failed: {g}/{a}:{t}. "
f"Reason: {str(e)}")
return False
else:
raise
try:
input_assets[req_asset[KEY]] = _seek(rgc, g, a, t, s)
except (
MissingAssetError,
MissingGenomeError,
MissingTagError,
MissingSeekKeyError,
) as e:
if args.pull_parents:
_LOGGER.info(f"Pulling missing parent: {g}/{a}:{t}")
ret = rgc.pull(genome=g, asset=a, tag=t)
if ret is None or not all(ret):
_LOGGER.info(
f"Missing parent asset pull requested, but failed: {g}/{a}:{t}. "
f"Reason: {str(e)}")
return False
else:
raise
_LOGGER.debug("Using parents: {}".format(", ".join(parent_assets)))
_LOGGER.debug("Provided files: {}".format(specified_args))
_LOGGER.debug("Provided parameters: {}".format(specified_params))
for required_file in asset_build_package[REQ_FILES]:
if (specified_args is None
or required_file[KEY] not in specified_args.keys()):
raise ValueError(
"Path to the '{x}' input ({desc}) is required, but not provided. "
"Specify it with: --files {x}=/path/to/{x}_file".
format(x=required_file[KEY], desc=required_file[DESC]))
for required_param in asset_build_package[REQ_PARAMS]:
if specified_params is None:
specified_params = {}
if required_param[KEY] not in specified_params.keys():
if required_param[DEFAULT] is None:
raise ValueError(
"Value for the parameter '{x}' ({desc}) is required, but not provided. "
"Specify it with: --params {x}=value".format(
x=required_param[KEY],
desc=required_param[DESC]))
else:
specified_params.update(
{required_param[KEY]: required_param[DEFAULT]})
_LOGGER.info("Building '{}/{}:{}' using '{}' recipe".format(
genome, asset_key, asset_tag, recipe_name))
ori_genome = genome
if recipe_name == "fasta":
if (genome in rgc.genomes_list()
and "fasta" in rgc.list_assets_by_genome(genome)):
pretag = rgc.get_default_tag(genome, "fasta")
_LOGGER.warning(
"'{g}' genome is already initialized with other fasta asset ({g}/{a}:{t})"
.format(g=genome, a=asset_key, t=pretag))
genome = rgc.get_genome_alias_digest(alias=genome,
fallback=True)
else:
# if the recipe is "fasta" we first initialiaze the genome, based on the provided path to the input FASTA file
genome, _ = rgc.initialize_genome(
fasta_path=specified_args["fasta"],
alias=ori_genome,
skip_alias_write=True,
)
else:
try:
genome = rgc.get_genome_alias_digest(genome, fallback=True)
except UndefinedAliasError:
_LOGGER.error("Genome '{}' has not been initialized yet; "
"no key found for this alias".format(genome))
return False
recipe_name = None
genome_outfolder = os.path.join(rgc.data_dir, genome)
build_stats_dir = os.path.abspath(
os.path.join(genome_outfolder, asset_key, asset_tag,
BUILD_STATS_DIR))
locked_map_gencfg = os.path.join(build_stats_dir,
LOCKED_BUILD_MAP_CFG)
map_gencfg = os.path.join(build_stats_dir, BUILD_MAP_CFG)
is_built, rgc_map = _build_asset(
genome,
asset_key,
asset_tag,
asset_build_package,
genome_outfolder,
specified_args,
specified_params,
ori_genome,
**input_assets,
)
if not is_built:
log_path = os.path.abspath(
os.path.join(
genome_outfolder,
asset_key,
asset_tag,
BUILD_STATS_DIR,
ORI_LOG_NAME,
))
_LOGGER.info(
"'{}/{}:{}' was not added to the config, but directory has been left in place. "
"See the log file for details: {}".format(
genome, asset_key, asset_tag, log_path))
return False
_LOGGER.info("Finished building '{}' asset".format(asset_key))
with rgc_map as r:
# update asset relationships
r.update_relatives_assets(genome, asset_key, asset_tag,
parent_assets) # adds parents
for i in parent_assets:
parsed_parent = parse_registry_path(i)
# adds child (currently built asset) to the parent
r.update_relatives_assets(
parsed_parent["genome"],
parsed_parent["asset"],
parsed_parent["tag"],
["{}/{}:{}".format(genome, asset_key, asset_tag)],
True,
)
if args.genome_description is not None:
_LOGGER.debug(
"adding genome ({}) description: '{}'".format(
genome, args.genome_description))
r.update_genomes(
genome, {CFG_GENOME_DESC_KEY: args.genome_description})
if args.tag_description is not None:
_LOGGER.debug(
"adding tag ({}/{}:{}) description: '{}'".format(
genome, asset_key, asset_tag,
args.tag_description))
r.update_tags(
genome,
asset_key,
asset_tag,
{CFG_TAG_DESC_KEY: args.tag_description},
)
rgc_map._symlink_alias(genome, asset_key, asset_tag)
if args.map:
# move the contents of the locked map config to a map config,
# which is discoverable by the reduce step
os.rename(locked_map_gencfg, map_gencfg)
_LOGGER.info(
f"Asset metadata saved in '{map_gencfg}'. "
f"To make the asset accessible globally run 'refgenie build --reduce'"
)
else:
_raise_missing_recipe_error(recipe_name)
return True