def check_user_library_import_dir(self, user):
if getattr(self.app.config, "user_library_import_dir_auto_creation", False):
# try to create a user library import directory
try:
safe_makedirs(os.path.join(self.app.config.user_library_import_dir, user.email))
except ConfigurationError as e:
self.log_event(str(e))
def dataset_wrapper_to_file_json(inputs_dir, dataset_wrapper):
if dataset_wrapper.ext == "expression.json":
with open(dataset_wrapper.file_name, "r") as f:
return json.load(f)
if dataset_wrapper.ext == "directory":
return dataset_wrapper_to_directory_json(inputs_dir, dataset_wrapper)
extra_files_path = dataset_wrapper.extra_files_path
secondary_files_path = os.path.join(extra_files_path, "__secondary_files__")
path = str(dataset_wrapper)
raw_file_object = {"class": "File"}
if os.path.exists(secondary_files_path):
safe_makedirs(inputs_dir)
name = os.path.basename(path)
new_input_path = os.path.join(inputs_dir, name)
os.symlink(path, new_input_path)
secondary_files = []
for secondary_file_name in os.listdir(secondary_files_path):
secondary_file_path = os.path.join(secondary_files_path, secondary_file_name)
target = os.path.join(inputs_dir, secondary_file_name)
log.info("linking [%s] to [%s]" % (secondary_file_path, target))
os.symlink(secondary_file_path, target)
is_dir = os.path.isdir(os.path.realpath(secondary_file_path))
secondary_files.append({"class": "File" if not is_dir else "Directory", "location": target})
raw_file_object["secondaryFiles"] = secondary_files
path = new_input_path
raw_file_object["location"] = path
raw_file_object["size"] = int(dataset_wrapper.get_size())
set_basename_and_derived_properties(raw_file_object, str(dataset_wrapper.cwl_filename or dataset_wrapper.name))
return raw_file_object
def output_secondary_files_dir(self, output_name, create=False):
extra_files_dir = self._output_extra_files_dir(output_name)
secondary_files_dir = os.path.join(extra_files_dir,
SECONDARY_FILES_EXTRA_PREFIX)
if create and not os.path.exists(secondary_files_dir):
safe_makedirs(secondary_files_dir)
return secondary_files_dir
def check_user_library_import_dir(self, user):
if getattr(self.app.config, "user_library_import_dir_auto_creation", False):
# try to create a user library import directory
try:
safe_makedirs(os.path.join(self.app.config.user_library_import_dir, user.email))
except ConfigurationError as e:
self.log_event(unicodify(e))
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.cache_directory = kwargs.get(
"cache_directory",
os.path.join(kwargs['app_info'].container_image_cache_path,
"singularity", "mulled"))
safe_makedirs(self.cache_directory)
def test_from_directory():
target_dir = tempfile.mkdtemp()
dir_1 = os.path.join(target_dir, "dir1")
dir_2 = os.path.join(target_dir, "dir2")
safe_makedirs(dir_1)
safe_makedirs(dir_2)
with open(os.path.join(dir_1, "foo.json"), "w") as f:
json.dump(EXAMPLE_1, f)
with open(os.path.join(dir_2, "seqanalysis.yml"), "w") as f:
f.write("""
name: Sequence Analysis
type: generic
items:
- type: workflow
id: 12345abcd
""")
config_param = f"{os.path.abspath(dir_1)},{os.path.abspath(dir_2)}"
try:
sources = StaticToolBoxViewSources(view_directories=config_param)
defs = sources.get_definitions()
assert len(defs) == 2
assert defs[0].id == "rna"
assert defs[1].id == "seqanalysis"
finally:
shutil.rmtree(target_dir)
def main(argv=None):
mod_docstring = sys.modules[__name__].__doc__
arg_parser = ArgumentParser(description=mod_docstring)
arg_parser.add_argument("--conda_prefix", required=True)
args = arg_parser.parse_args(argv)
conda_prefix = args.conda_prefix
safe_makedirs(os.path.dirname(conda_prefix))
conda_context = CondaContext(conda_prefix=conda_prefix, )
install_conda(conda_context)
def setup_external_metadata(self, datasets_dict, sa_session, exec_dir=None,
tmp_dir=None, dataset_files_path=None,
output_fnames=None, config_root=None,
config_file=None, datatypes_config=None,
job_metadata=None, compute_tmp_dir=None,
include_command=True, max_metadata_value_size=0,
kwds=None):
assert job_metadata, "setup_external_metadata must be supplied with job_metadata path"
kwds = kwds or {}
tmp_dir = _init_tmp_dir(tmp_dir)
metadata_dir = os.path.join(tmp_dir, "metadata")
# may already exist (i.e. metadata collection in the job handler)
safe_makedirs(metadata_dir)
def job_relative_path(path):
path_relative = os.path.relpath(path, tmp_dir)
return path_relative
outputs = {}
for name, dataset in datasets_dict.items():
assert name is not None
assert name not in outputs
key = name
def _metadata_path(what):
return os.path.join(metadata_dir, "metadata_%s_%s" % (what, key))
_initialize_metadata_inputs(dataset, _metadata_path, tmp_dir, kwds)
outputs[name] = {
"filename_override": _get_filename_override(output_fnames, dataset.file_name)
}
metadata_params_path = os.path.join(metadata_dir, "params.json")
metadata_params = {
"job_metadata": job_relative_path(job_metadata),
"datatypes_config": datatypes_config,
"max_metadata_value_size": max_metadata_value_size,
"outputs": outputs,
"object_store_store_by": galaxy.model.Dataset.object_store.store_by,
}
with open(metadata_params_path, "w") as f:
json.dump(metadata_params, f)
if include_command:
# return command required to build
script_path = os.path.join(metadata_dir, "set.py")
with open(script_path, "w") as f:
f.write(SET_METADATA_SCRIPT)
return 'python "metadata/set.py"'
else:
# return args to galaxy_ext.metadata.set_metadata required to build
return ''
def create(self, obj, **kwargs):
"""Override `ObjectStore`'s stub by creating any files and folders on disk."""
if not self.exists(obj, **kwargs):
path = self._construct_path(obj, **kwargs)
dir_only = kwargs.get('dir_only', False)
# Create directory if it does not exist
dir = path if dir_only else os.path.dirname(path)
safe_makedirs(dir)
# Create the file if it does not exist
if not dir_only:
open(path, 'w').close() # Should be rb?
umask_fix_perms(path, self.config.umask, 0o666)
def create(self, trans, job_id, payload, **kwargs):
"""
create( self, trans, job_id, payload, **kwargs )
* POST /api/jobs/{job_id}/files
Populate an output file (formal dataset, task split part, working
directory file (such as those related to metadata)). This should be
a multipart post with a 'file' parameter containing the contents of
the actual file to create.
:type job_id: str
:param job_id: encoded id string of the job
:type payload: dict
:param payload: dictionary structure containing::
'job_key' = Key authenticating
'path' = Path to file to create.
..note:
This API method is intended only for consumption by job runners,
not end users.
:rtype: dict
:returns: an okay message
"""
job = self.__authorize_job_access(trans, job_id, **payload)
path = payload.get("path")
self.__check_job_can_write_to_path(trans, job, path)
# Is this writing an unneeded file? Should this just copy in Python?
if '__file_path' in payload:
file_path = payload.get('__file_path')
upload_store = trans.app.config.nginx_upload_job_files_store
assert upload_store, ("Request appears to have been processed by"
" nginx_upload_module but Galaxy is not"
" configured to recognize it")
assert file_path.startswith(upload_store), \
("Filename provided by nginx (%s) is not in correct"
" directory (%s)" % (file_path, upload_store))
input_file = open(file_path)
else:
input_file = payload.get("file",
payload.get("__file", None)).file
target_dir = os.path.dirname(path)
util.safe_makedirs(target_dir)
try:
shutil.move(input_file.name, path)
finally:
try:
input_file.close()
except OSError:
# Fails to close file if not using nginx upload because the
# tempfile has moved and Python wants to delete it.
pass
return {"message": "ok"}
def __write_workdir_file(self, config_filename, content, context, is_template=True, strip=False):
parent_dir = os.path.dirname(config_filename)
if not os.path.exists(parent_dir):
safe_makedirs(parent_dir)
if is_template:
value = fill_template(content, context=context, python_template_version=self.tool.python_template_version)
else:
value = unicodify(content)
if strip:
value = value.strip()
with open(config_filename, "w", encoding='utf-8') as f:
f.write(value)
# For running jobs as the actual user, ensure the config file is globally readable
os.chmod(config_filename, RW_R__R__)
def _write_to_file(args, f, path):
if hasattr(f, "getvalue"):
contents = f.getvalue()
else:
contents = f
if args.dry_run:
contents_indented = "\n".join(f" |{line}" for line in contents.splitlines())
print(f"Overwriting {path} with the following contents:\n{contents_indented}")
print("... skipping because --dry-run is enabled.")
else:
print(f"Overwriting {path}")
safe_makedirs(os.path.dirname(path))
with open(path, "w") as to_f:
to_f.write(contents)
def move_directory(output, target_path, output_name=None):
assert output["class"] == "Directory"
output_path = _possible_uri_to_path(output["location"])
if output_path.startswith("_:"):
# No a real path, just copy listing to target path.
safe_makedirs(target_path)
for listed_file in output["listing"]:
# TODO: handle directories
assert listed_file["class"] == "File"
file_description = file_dict_to_description(listed_file)
file_description.write_to(os.path.join(target_path, listed_file["basename"]))
else:
shutil.move(output_path, target_path)
return {"cwl_filename": output["basename"]}
def _write_to_file(args, f, path):
dry_run = args.dry_run
if hasattr(f, "getvalue"):
contents = f.getvalue()
else:
contents = f
contents_indented = "\n".join([" |%s" % l for l in contents.splitlines()])
print("Writing the file contents:\n%s\nto %s" % (contents_indented, path))
if dry_run:
print("... skipping because --dry-run is enabled.")
else:
safe_makedirs(os.path.dirname(path))
with open(path, "w") as to_f:
to_f.write(contents)
def _write_to_file(args, f, path):
dry_run = args.dry_run
if hasattr(f, "getvalue"):
contents = f.getvalue()
else:
contents = f
contents_indented = "\n".join([" |%s" % l for l in contents.splitlines()])
print("Writing the file contents:\n%s\nto %s" % (contents_indented, path))
if dry_run:
print("... skipping because --dry-run is enabled.")
else:
safe_makedirs(os.path.dirname(path))
with open(path, "w") as to_f:
to_f.write(contents)
def move_directory(output, target_path, output_name=None):
assert output["class"] == "Directory"
output_path = _possible_uri_to_path(output["location"])
if output_path.startswith("_:"):
# No a real path, just copy listing to target path.
safe_makedirs(target_path)
for listed_file in output["listing"]:
# TODO: handle directories
assert listed_file["class"] == "File"
file_description = file_dict_to_description(listed_file)
file_description.write_to(os.path.join(target_path, listed_file["basename"]))
else:
shutil.move(output_path, target_path)
return {"created_from_basename": output["basename"]}
def simple_value(input, param_dict_value, cwl_type=None):
# Hmm... cwl_type isn't really the cwl type in every case,
# like in the case of json for instance.
if cwl_type is None:
input_type = input.type
cwl_type = GALAXY_TO_CWL_TYPES[input_type]
if cwl_type == "null":
assert param_dict_value is None
return None
if cwl_type == "File":
dataset_wrapper = param_dict_value
extra_files_path = dataset_wrapper.extra_files_path
secondary_files_path = os.path.join(extra_files_path,
"__secondary_files__")
path = str(dataset_wrapper)
if os.path.exists(secondary_files_path):
safe_makedirs(inputs_dir)
name = os.path.basename(path)
new_input_path = os.path.join(inputs_dir, name)
os.symlink(path, new_input_path)
for secondary_file_name in os.listdir(secondary_files_path):
secondary_file_path = os.path.join(secondary_files_path,
secondary_file_name)
os.symlink(secondary_file_path,
new_input_path + secondary_file_name)
path = new_input_path
return {"location": path, "class": "File"}
elif cwl_type == "integer":
return int(str(param_dict_value))
elif cwl_type == "long":
return int(str(param_dict_value))
elif cwl_type == "float":
return float(str(param_dict_value))
elif cwl_type == "double":
return float(str(param_dict_value))
elif cwl_type == "boolean":
return string_as_bool(param_dict_value)
elif cwl_type == "text":
return str(param_dict_value)
elif cwl_type == "json":
raw_value = param_dict_value.value
log.info("raw_value is %s (%s)" % (raw_value, type(raw_value)))
return json.loads(raw_value)
else:
return str(param_dict_value)
def get_dataset(dataset_details, filename=None):
parent_basename = dataset_details.get("cwl_file_name") or dataset_details.get("name")
file_ext = dataset_details["file_ext"]
if file_ext == "directory":
# TODO: rename output_directory to outputs_directory because we can have output directories
# and this is confusing...
the_output_directory = os.path.join(output_directory, parent_basename)
safe_makedirs(the_output_directory)
destination = self.download_output_to(ctx, dataset_details, the_output_directory, filename=filename)
else:
destination = self.download_output_to(ctx, dataset_details, output_directory, filename=filename)
if filename is None:
basename = parent_basename
else:
basename = os.path.basename(filename)
return {"path": destination, "basename": basename}
def dataset_wrapper_to_file_json(inputs_dir, dataset_wrapper):
if dataset_wrapper.ext == "expression.json":
with open(dataset_wrapper.file_name) as f:
return json.load(f)
if dataset_wrapper.ext == "directory":
return dataset_wrapper_to_directory_json(inputs_dir, dataset_wrapper)
extra_files_path = dataset_wrapper.extra_files_path
secondary_files_path = os.path.join(extra_files_path,
"__secondary_files__")
path = str(dataset_wrapper)
raw_file_object = {"class": "File"}
if os.path.exists(secondary_files_path):
safe_makedirs(inputs_dir)
name = os.path.basename(path)
new_input_path = os.path.join(inputs_dir, name)
os.symlink(path, new_input_path)
secondary_files = []
for secondary_file_name in os.listdir(secondary_files_path):
secondary_file_path = os.path.join(secondary_files_path,
secondary_file_name)
target = os.path.join(inputs_dir, secondary_file_name)
log.info("linking [{}] to [{}]".format(secondary_file_path,
target))
os.symlink(secondary_file_path, target)
is_dir = os.path.isdir(os.path.realpath(secondary_file_path))
secondary_files.append({
"class": "File" if not is_dir else "Directory",
"location": target
})
raw_file_object["secondaryFiles"] = secondary_files
path = new_input_path
raw_file_object["location"] = path
# Verify it isn't a NoneDataset
if dataset_wrapper.unsanitized:
raw_file_object["size"] = int(dataset_wrapper.get_size())
set_basename_and_derived_properties(
raw_file_object,
str(dataset_wrapper.created_from_basename or dataset_wrapper.name))
return raw_file_object
def simple_value(input, param_dict_value, cwl_type=None):
# Hmm... cwl_type isn't really the cwl type in every case,
# like in the case of json for instance.
if cwl_type is None:
input_type = input.type
cwl_type = GALAXY_TO_CWL_TYPES[input_type]
if cwl_type == "null":
assert param_dict_value is None
return None
if cwl_type == "File":
dataset_wrapper = param_dict_value
extra_files_path = dataset_wrapper.extra_files_path
secondary_files_path = os.path.join(extra_files_path, "__secondary_files__")
path = str(dataset_wrapper)
if os.path.exists(secondary_files_path):
safe_makedirs(inputs_dir)
name = os.path.basename(path)
new_input_path = os.path.join(inputs_dir, name)
os.symlink(path, new_input_path)
for secondary_file_name in os.listdir(secondary_files_path):
secondary_file_path = os.path.join(secondary_files_path, secondary_file_name)
os.symlink(secondary_file_path, new_input_path + secondary_file_name)
path = new_input_path
return {"location": path,
"class": "File"}
elif cwl_type == "integer":
return int(str(param_dict_value))
elif cwl_type == "long":
return int(str(param_dict_value))
elif cwl_type == "float":
return float(str(param_dict_value))
elif cwl_type == "double":
return float(str(param_dict_value))
elif cwl_type == "boolean":
return string_as_bool(param_dict_value)
elif cwl_type == "text":
return str(param_dict_value)
elif cwl_type == "json":
raw_value = param_dict_value.value
log.info("raw_value is %s (%s)" % (raw_value, type(raw_value)))
return json.loads(raw_value)
else:
return str(param_dict_value)
def get_dataset(dataset_details, filename=None):
parent_basename = dataset_details.get("cwl_file_name")
if not parent_basename:
parent_basename = dataset_details.get("name")
file_ext = dataset_details["file_ext"]
if file_ext == "directory":
# TODO: rename output_directory to outputs_directory because we can have output directories
# and this is confusing...
the_output_directory = os.path.join(output_directory, parent_basename)
safe_makedirs(the_output_directory)
destination = self.download_output_to(dataset_details, the_output_directory, filename=filename)
else:
destination = self.download_output_to(dataset_details, output_directory, filename=filename)
if filename is None:
basename = parent_basename
else:
basename = os.path.basename(filename)
return {"path": destination, "basename": basename}
def handle_galaxy_config_kwds(cls, config):
amqp_url = os.environ.get("GALAXY_TEST_AMQP_URL", None)
if amqp_url is None:
pytest.skip("External AMQP URL not configured for test")
jobs_directory = os.path.join(cls._test_driver.mkdtemp(),
"pulsar_staging")
safe_makedirs(jobs_directory)
job_conf_template = string.Template(JOB_CONF_TEMPLATE)
job_conf_str = job_conf_template.substitute(
amqp_url=AMQP_URL,
jobs_directory=jobs_directory,
galaxy_home=os.path.join(SCRIPT_DIRECTORY, os.pardir))
with tempfile.NamedTemporaryFile(suffix="_mq_job_conf.yml",
mode="w",
delete=False) as job_conf:
job_conf.write(job_conf_str)
config["job_config_file"] = job_conf.name
infrastructure_url = "http://localhost:$GALAXY_WEB_PORT"
config["galaxy_infrastructure_url"] = infrastructure_url
def rewrite_inputs_for_staging(self):
if hasattr(self._cwl_job, "pathmapper"):
pass
# DO SOMETHING LIKE THE FOLLOWING?
# path_rewrites = {}
# for f, p in self._cwl_job.pathmapper.items():
# if not p.staged:
# continue
# if p.type in ("File", "Directory"):
# path_rewrites[p.resolved] = p.target
# for key, value in self._input_dict.items():
# if key in path_rewrites:
# self._input_dict[key]["location"] = path_rewrites[value]
else:
stagedir = os.path.join(self._job_directory, "cwlstagedir")
safe_makedirs(stagedir)
def stage_recursive(value):
is_list = isinstance(value, list)
is_dict = isinstance(value, dict)
log.info(
f"handling value {value}, is_list {is_list}, is_dict {is_dict}"
)
if is_list:
for val in value:
stage_recursive(val)
elif is_dict:
if "location" in value and "basename" in value:
location = value["location"]
basename = value["basename"]
if not location.endswith(basename): # TODO: sep()[-1]
staged_loc = os.path.join(stagedir, basename)
if not os.path.exists(staged_loc):
os.symlink(location, staged_loc)
value["location"] = staged_loc
for dict_value in value.values():
stage_recursive(dict_value)
else:
log.info("skipping simple value...")
stage_recursive(self._input_dict)
def walk_extra_files(items, prefix=""):
for item in items:
if "elements" in item:
name = item.get("name")
if not prefix:
item_prefix = name
else:
item_prefix = os.path.join(prefix, name)
walk_extra_files(item.get("elements"), prefix=item_prefix)
else:
name, src_path = _has_src_to_path(upload_config, item)
if prefix:
rel_path = os.path.join(prefix, name)
else:
rel_path = name
file_output_path = os.path.join(staged_extra_files, rel_path)
parent_dir = os.path.dirname(file_output_path)
if not os.path.exists(parent_dir):
safe_makedirs(parent_dir)
shutil.move(src_path, file_output_path)
def dataset_wrapper_to_file_json(inputs_dir, dataset_wrapper):
if dataset_wrapper.ext == "expression.json":
with open(dataset_wrapper.file_name, "r") as f:
return json.load(f)
extra_files_path = dataset_wrapper.extra_files_path
secondary_files_path = os.path.join(extra_files_path,
"__secondary_files__")
path = str(dataset_wrapper)
if os.path.exists(secondary_files_path):
safe_makedirs(inputs_dir)
name = os.path.basename(path)
new_input_path = os.path.join(inputs_dir, name)
os.symlink(path, new_input_path)
for secondary_file_name in os.listdir(secondary_files_path):
secondary_file_path = os.path.join(secondary_files_path,
secondary_file_name)
os.symlink(secondary_file_path,
new_input_path + secondary_file_name)
path = new_input_path
return {"location": path, "class": "File"}
def metadata_command(self, output_datasets, output_collections=None):
output_collections = output_collections or {}
metadata_compute_strategy = get_metadata_compute_strategy(
self.app.config, self.job.id)
self.metadata_compute_strategy = metadata_compute_strategy
exec_dir = None
dataset_files_path = self.app.model.Dataset.file_path
config_root = self.app.config.root
config_file = None
datatypes_config = os.path.join(self.job_working_directory, 'metadata',
'registry.xml')
safe_makedirs(os.path.join(self.job_working_directory, 'metadata'))
self.app.datatypes_registry.to_xml_file(path=datatypes_config)
job_metadata = os.path.join(self.tool_working_directory,
self.tool.provided_metadata_file)
output_fnames = [
DatasetPath(o.dataset.id, o.dataset.file_name, None)
for o in output_datasets.values()
]
command = metadata_compute_strategy.setup_external_metadata(
output_datasets,
output_collections,
self.app.model.session,
exec_dir=exec_dir,
tmp_dir=self.
job_working_directory, # set in jobs/runners.py - better if was default.
dataset_files_path=dataset_files_path,
config_root=config_root,
config_file=config_file,
datatypes_config=datatypes_config,
job_metadata=job_metadata,
output_fnames=output_fnames,
tool=self.tool,
job=self.job,
object_store_conf=self.app.object_store.to_dict(),
max_metadata_value_size=10000)
return command
def rewrite_inputs_for_staging(self):
if hasattr(self._cwl_job, "pathmapper"):
pass
# DO SOMETHING LIKE THE FOLLOWING?
# path_rewrites = {}
# for f, p in self._cwl_job.pathmapper.items():
# if not p.staged:
# continue
# if p.type in ("File", "Directory"):
# path_rewrites[p.resolved] = p.target
# for key, value in self._input_dict.items():
# if key in path_rewrites:
# self._input_dict[key]["location"] = path_rewrites[value]
else:
stagedir = os.path.join(self._job_directory, "cwlstagedir")
safe_makedirs(stagedir)
def stage_recursive(value):
is_list = isinstance(value, list)
is_dict = isinstance(value, dict)
log.info("handling value %s, is_list %s, is_dict %s" % (value, is_list, is_dict))
if is_list:
for val in value:
stage_recursive(val)
elif is_dict:
if "location" in value and "basename" in value:
location = value["location"]
basename = value["basename"]
if not location.endswith(basename): # TODO: sep()[-1]
staged_loc = os.path.join(stagedir, basename)
if not os.path.exists(staged_loc):
os.symlink(location, staged_loc)
value["location"] = staged_loc
for key, dict_value in value.items():
stage_recursive(dict_value)
else:
log.info("skipping simple value...")
stage_recursive(self._input_dict)
def mull_targets(
targets,
involucro_context=None,
command="build",
channels=DEFAULT_CHANNELS,
namespace="biocontainers",
test='true',
test_files=None,
image_build=None,
name_override=None,
repository_template=DEFAULT_REPOSITORY_TEMPLATE,
dry_run=False,
conda_version=None,
verbose=False,
binds=DEFAULT_BINDS,
rebuild=True,
oauth_token=None,
hash_func="v2",
singularity=False,
singularity_image_dir="singularity_import",
base_image=None,
determine_base_image=True,
):
targets = list(targets)
if involucro_context is None:
involucro_context = InvolucroContext()
image_function = v1_image_name if hash_func == "v1" else v2_image_name
if len(targets) > 1 and image_build is None:
# Force an image build in this case - this seems hacky probably
# shouldn't work this way but single case broken else wise.
image_build = "0"
repo_template_kwds = {
"namespace":
namespace,
"image":
image_function(targets,
image_build=image_build,
name_override=name_override)
}
repo = string.Template(repository_template).safe_substitute(
repo_template_kwds)
if not rebuild or "push" in command:
repo_name = repo_template_kwds["image"].split(":", 1)[0]
repo_data = quay_repository(repo_template_kwds["namespace"], repo_name)
if not rebuild:
tags = repo_data.get("tags", [])
target_tag = None
if ":" in repo_template_kwds["image"]:
image_name_parts = repo_template_kwds["image"].split(":")
assert len(
image_name_parts
) == 2, f": not allowed in image name [{repo_template_kwds['image']}]"
target_tag = image_name_parts[1]
if tags and (target_tag is None or target_tag in tags):
raise BuildExistsException()
if "push" in command and "error_type" in repo_data and oauth_token:
# Explicitly create the repository so it can be built as public.
create_repository(repo_template_kwds["namespace"], repo_name,
oauth_token)
for channel in channels:
if channel.startswith('file://'):
bind_path = channel[7:]
binds.append(f'/{bind_path}:/{bind_path}')
channels = ",".join(channels)
target_str = ",".join(map(conda_build_target_str, targets))
bind_str = ",".join(binds)
involucro_args = [
'-f',
f'{DIRNAME}/invfile.lua',
'-set',
f"CHANNELS={channels}",
'-set',
f"TARGETS={target_str}",
'-set',
f"REPO={repo}",
'-set',
f"BINDS={bind_str}",
]
dest_base_image = None
if base_image:
dest_base_image = base_image
elif DEST_BASE_IMAGE:
dest_base_image = DEST_BASE_IMAGE
elif determine_base_image:
dest_base_image = base_image_for_targets(targets)
if dest_base_image:
involucro_args.extend(["-set", f"DEST_BASE_IMAGE={dest_base_image}"])
if CONDA_IMAGE:
involucro_args.extend(["-set", f"CONDA_IMAGE={CONDA_IMAGE}"])
if verbose:
involucro_args.extend(["-set", "VERBOSE=1"])
if singularity:
singularity_image_name = repo_template_kwds['image']
involucro_args.extend(["-set", "SINGULARITY=1"])
involucro_args.extend(
["-set", f"SINGULARITY_IMAGE_NAME={singularity_image_name}"])
involucro_args.extend(
["-set", f"SINGULARITY_IMAGE_DIR={singularity_image_dir}"])
involucro_args.extend(["-set", f"USER_ID={os.getuid()}:{os.getgid()}"])
if test:
involucro_args.extend(["-set", f"TEST={test}"])
if conda_version is not None:
verbose = "--verbose" if verbose else "--quiet"
involucro_args.extend([
"-set",
f"PREINSTALL=conda install {verbose} --yes conda={conda_version}"
])
involucro_args.append(command)
if test_files:
test_bind = []
for test_file in test_files:
if ':' not in test_file:
if os.path.exists(test_file):
test_bind.append(
f"{test_file}:{DEFAULT_WORKING_DIR}/{test_file}")
else:
if os.path.exists(test_file.split(':')[0]):
test_bind.append(test_file)
if test_bind:
involucro_args.insert(6, '-set')
involucro_args.insert(7, f"TEST_BINDS={','.join(test_bind)}")
cmd = involucro_context.build_command(involucro_args)
print(f"Executing: {' '.join(shlex.quote(_) for _ in cmd)}")
if dry_run:
return 0
ensure_installed(involucro_context, True)
if singularity:
if not os.path.exists(singularity_image_dir):
safe_makedirs(singularity_image_dir)
with open(os.path.join(singularity_image_dir, 'Singularity.def'),
'w+') as sin_def:
fill_template = SINGULARITY_TEMPLATE % {
'container_test': test,
'base_image': dest_base_image or DEFAULT_BASE_IMAGE
}
sin_def.write(fill_template)
with PrintProgress():
ret = involucro_context.exec_command(involucro_args)
if singularity:
# we can not remove this folder as it contains the image wich is owned by root
pass
# shutil.rmtree('./singularity_import')
return ret
def make_files_path():
safe_makedirs(files_path)
def __init__(self, results_path):
safe_makedirs(results_path)
self.lifecycle = AllureLifecycle()
self.logger = AllureFileLogger(results_path)
self.listener = AllureListener(self.lifecycle)
def _init_cache_directory(self):
cacher_class = get_cache_directory_cacher(
self.cache_directory_cacher_type)
self.cache_directory = cacher_class(self.cache_directory_path,
hash_func=self.hash_func)
safe_makedirs(self.cache_directory.path)
def setup_external_metadata(self,
datasets_dict,
out_collections,
sa_session,
exec_dir=None,
tmp_dir=None,
dataset_files_path=None,
output_fnames=None,
config_root=None,
use_bin=False,
config_file=None,
datatypes_config=None,
job_metadata=None,
provided_metadata_style=None,
compute_tmp_dir=None,
include_command=True,
max_metadata_value_size=0,
validate_outputs=False,
object_store_conf=None,
tool=None,
job=None,
kwds=None):
assert job_metadata, "setup_external_metadata must be supplied with job_metadata path"
kwds = kwds or {}
if not job:
job = sa_session.query(galaxy.model.Job).get(self.job_id)
tmp_dir = _init_tmp_dir(tmp_dir)
metadata_dir = os.path.join(tmp_dir, "metadata")
# may already exist (i.e. metadata collection in the job handler)
safe_makedirs(metadata_dir)
def job_relative_path(path):
path_relative = os.path.relpath(path, tmp_dir)
return path_relative
outputs = {}
output_collections = {}
for name, dataset in datasets_dict.items():
assert name is not None
assert name not in outputs
key = name
def _metadata_path(what):
return os.path.join(metadata_dir, f"metadata_{what}_{key}")
_initialize_metadata_inputs(
dataset,
_metadata_path,
tmp_dir,
kwds,
real_metadata_object=self.write_object_store_conf)
outputs[name] = {
"filename_override":
_get_filename_override(output_fnames, dataset.file_name),
"validate":
validate_outputs,
"object_store_store_by":
dataset.dataset.store_by,
'id':
dataset.id,
'model_class':
'LibraryDatasetDatasetAssociation' if isinstance(
dataset, galaxy.model.LibraryDatasetDatasetAssociation)
else 'HistoryDatasetAssociation'
}
metadata_params_path = os.path.join(metadata_dir, "params.json")
datatypes_config = os.path.relpath(datatypes_config, tmp_dir)
metadata_params = {
"job_metadata": job_relative_path(job_metadata),
"provided_metadata_style": provided_metadata_style,
"datatypes_config": datatypes_config,
"max_metadata_value_size": max_metadata_value_size,
"outputs": outputs,
}
# export model objects and object store configuration for extended metadata also.
export_directory = os.path.join(metadata_dir, "outputs_new")
with DirectoryModelExportStore(export_directory,
for_edit=True,
serialize_dataset_objects=True,
serialize_jobs=False) as export_store:
for dataset in datasets_dict.values():
export_store.add_dataset(dataset)
for name, dataset_collection in out_collections.items():
export_store.add_dataset_collection(dataset_collection)
output_collections[name] = {
'id': dataset_collection.id,
}
if self.write_object_store_conf:
with open(os.path.join(metadata_dir, "object_store_conf.json"),
"w") as f:
json.dump(object_store_conf, f)
# setup tool
tool_as_dict = {}
tool_as_dict["stdio_exit_codes"] = [
e.to_dict() for e in tool.stdio_exit_codes
]
tool_as_dict["stdio_regexes"] = [
r.to_dict() for r in tool.stdio_regexes
]
tool_as_dict["outputs"] = {
name: output.to_dict()
for name, output in tool.outputs.items()
}
tool_as_dict["output_collections"] = {
name: output.to_dict()
for name, output in tool.output_collections.items()
}
# setup the rest
metadata_params["tool"] = tool_as_dict
metadata_params["tool_path"] = tool.config_file
metadata_params["job_id_tag"] = job.get_id_tag()
metadata_params["job_params"] = job.raw_param_dict()
metadata_params["output_collections"] = output_collections
with open(metadata_params_path, "w") as f:
json.dump(metadata_params, f)
if include_command:
# return command required to build
if use_bin:
return "galaxy-set-metadata"
else:
script_path = os.path.join(metadata_dir, "set.py")
with open(script_path, "w") as f:
f.write(SET_METADATA_SCRIPT)
return 'python "metadata/set.py"'
else:
# return args to galaxy_ext.metadata.set_metadata required to build
return ''
def _init_tmp_dir(tmp_dir):
assert tmp_dir is not None
safe_makedirs(tmp_dir)
return tmp_dir
def ensure_configs_directory(work_dir):
configs_dir = os.path.join(work_dir, "configs")
if not os.path.exists(configs_dir):
safe_makedirs(configs_dir)
return configs_dir
def mull_targets(
targets,
involucro_context=None,
command="build",
channels=DEFAULT_CHANNELS,
namespace="biocontainers",
test='true',
test_files=None,
image_build=None,
name_override=None,
repository_template=DEFAULT_REPOSITORY_TEMPLATE,
dry_run=False,
conda_version=None,
verbose=False,
binds=DEFAULT_BINDS,
rebuild=True,
oauth_token=None,
hash_func="v2",
singularity=False,
singularity_image_dir="singularity_import",
):
targets = list(targets)
if involucro_context is None:
involucro_context = InvolucroContext()
image_function = v1_image_name if hash_func == "v1" else v2_image_name
if len(targets) > 1 and image_build is None:
# Force an image build in this case - this seems hacky probably
# shouldn't work this way but single case broken else wise.
image_build = "0"
repo_template_kwds = {
"namespace":
namespace,
"image":
image_function(targets,
image_build=image_build,
name_override=name_override)
}
repo = string.Template(repository_template).safe_substitute(
repo_template_kwds)
if not rebuild or "push" in command:
repo_name = repo_template_kwds["image"].split(":", 1)[0]
repo_data = quay_repository(repo_template_kwds["namespace"], repo_name)
if not rebuild:
tags = repo_data.get("tags", [])
target_tag = None
if ":" in repo_template_kwds["image"]:
image_name_parts = repo_template_kwds["image"].split(":")
assert len(
image_name_parts
) == 2, ": not allowed in image name [%s]" % repo_template_kwds[
"image"]
target_tag = image_name_parts[1]
if tags and (target_tag is None or target_tag in tags):
raise BuildExistsException()
if "push" in command and "error_type" in repo_data and oauth_token:
# Explicitly create the repository so it can be built as public.
create_repository(repo_template_kwds["namespace"], repo_name,
oauth_token)
for channel in channels:
if channel.startswith('file://'):
bind_path = channel.lstrip('file://')
binds.append('/%s:/%s' % (bind_path, bind_path))
channels = ",".join(channels)
target_str = ",".join(map(conda_build_target_str, targets))
bind_str = ",".join(binds)
involucro_args = [
'-f',
'%s/invfile.lua' % DIRNAME,
'-set',
"CHANNELS='%s'" % channels,
'-set',
"TARGETS='%s'" % target_str,
'-set',
"REPO='%s'" % repo,
'-set',
"BINDS='%s'" % bind_str,
]
if DEST_BASE_IMAGE:
involucro_args.extend(
["-set", "DEST_BASE_IMAGE='%s'" % DEST_BASE_IMAGE])
if CONDA_IMAGE:
involucro_args.extend(["-set", "CONDA_IMAGE='%s'" % CONDA_IMAGE])
if verbose:
involucro_args.extend(["-set", "VERBOSE='1'"])
if singularity:
singularity_image_name = repo_template_kwds['image']
involucro_args.extend(["-set", "SINGULARITY='1'"])
involucro_args.extend(
["-set",
"SINGULARITY_IMAGE_NAME='%s'" % singularity_image_name])
involucro_args.extend(
["-set",
"SINGULARITY_IMAGE_DIR='%s'" % singularity_image_dir])
involucro_args.extend(
["-set", "USER_ID='%s:%s'" % (os.getuid(), os.getgid())])
if test:
involucro_args.extend(["-set", "TEST=%s" % shlex_quote(test)])
if conda_version is not None:
verbose = "--verbose" if verbose else "--quiet"
involucro_args.extend([
"-set",
"PREINSTALL='conda install %s --yes conda=%s'" %
(verbose, conda_version)
])
involucro_args.append(command)
if test_files:
test_bind = []
for test_file in test_files:
if ':' not in test_file:
if os.path.exists(test_file):
test_bind.append(
"%s:%s/%s" %
(test_file, DEFAULT_WORKING_DIR, test_file))
else:
if os.path.exists(test_file.split(':')[0]):
test_bind.append(test_file)
if test_bind:
involucro_args.insert(6, '-set')
involucro_args.insert(7, "TEST_BINDS='%s'" % ",".join(test_bind))
print(" ".join(involucro_context.build_command(involucro_args)))
if not dry_run:
ensure_installed(involucro_context, True)
if singularity:
if not os.path.exists(singularity_image_dir):
safe_makedirs(singularity_image_dir)
with open(os.path.join(singularity_image_dir, 'Singularity'),
'w+') as sin_def:
fill_template = SINGULARITY_TEMPLATE % {'container_test': test}
sin_def.write(fill_template)
with PrintProgress():
ret = involucro_context.exec_command(involucro_args)
if singularity:
# we can not remove this folder as it contains the image wich is owned by root
pass
# shutil.rmtree('./singularity_import')
return ret
return 0
def mull_targets(
targets, involucro_context=None,
command="build", channels=DEFAULT_CHANNELS, namespace="biocontainers",
test='true', test_files=None, image_build=None, name_override=None,
repository_template=DEFAULT_REPOSITORY_TEMPLATE, dry_run=False,
conda_version=None, verbose=False, binds=DEFAULT_BINDS, rebuild=True,
oauth_token=None, hash_func="v2", singularity=False,
singularity_image_dir="singularity_import",
):
targets = list(targets)
if involucro_context is None:
involucro_context = InvolucroContext()
image_function = v1_image_name if hash_func == "v1" else v2_image_name
if len(targets) > 1 and image_build is None:
# Force an image build in this case - this seems hacky probably
# shouldn't work this way but single case broken else wise.
image_build = "0"
repo_template_kwds = {
"namespace": namespace,
"image": image_function(targets, image_build=image_build, name_override=name_override)
}
repo = string.Template(repository_template).safe_substitute(repo_template_kwds)
if not rebuild or "push" in command:
repo_name = repo_template_kwds["image"].split(":", 1)[0]
repo_data = quay_repository(repo_template_kwds["namespace"], repo_name)
if not rebuild:
tags = repo_data.get("tags", [])
target_tag = None
if ":" in repo_template_kwds["image"]:
image_name_parts = repo_template_kwds["image"].split(":")
assert len(image_name_parts) == 2, ": not allowed in image name [%s]" % repo_template_kwds["image"]
target_tag = image_name_parts[1]
if tags and (target_tag is None or target_tag in tags):
raise BuildExistsException()
if "push" in command and "error_type" in repo_data and oauth_token:
# Explicitly create the repository so it can be built as public.
create_repository(repo_template_kwds["namespace"], repo_name, oauth_token)
for channel in channels:
if channel.startswith('file://'):
bind_path = channel.lstrip('file://')
binds.append('/%s:/%s' % (bind_path, bind_path))
channels = ",".join(channels)
target_str = ",".join(map(conda_build_target_str, targets))
bind_str = ",".join(binds)
involucro_args = [
'-f', '%s/invfile.lua' % DIRNAME,
'-set', "CHANNELS='%s'" % channels,
'-set', "TEST=%s" % shlex_quote(test),
'-set', "TARGETS='%s'" % target_str,
'-set', "REPO='%s'" % repo,
'-set', "BINDS='%s'" % bind_str,
]
if DEST_BASE_IMAGE:
involucro_args.extend(["-set", "DEST_BASE_IMAGE='%s'" % DEST_BASE_IMAGE])
if CONDA_IMAGE:
involucro_args.extend(["-set", "CONDA_IMAGE='%s'" % CONDA_IMAGE])
if verbose:
involucro_args.extend(["-set", "VERBOSE='1'"])
if singularity:
singularity_image_name = repo_template_kwds['image']
involucro_args.extend(["-set", "SINGULARITY='1'"])
involucro_args.extend(["-set", "SINGULARITY_IMAGE_NAME='%s'" % singularity_image_name])
involucro_args.extend(["-set", "SINGULARITY_IMAGE_DIR='%s'" % singularity_image_dir])
involucro_args.extend(["-set", "USER_ID='%s:%s'" % (os.getuid(), os.getgid())])
if conda_version is not None:
verbose = "--verbose" if verbose else "--quiet"
involucro_args.extend(["-set", "PREINSTALL='conda install %s --yes conda=%s'" % (verbose, conda_version)])
involucro_args.append(command)
if test_files:
test_bind = []
for test_file in test_files:
if ':' not in test_file:
if os.path.exists(test_file):
test_bind.append("%s:%s/%s" % (test_file, DEFAULT_WORKING_DIR, test_file))
else:
if os.path.exists(test_file.split(':')[0]):
test_bind.append(test_file)
if test_bind:
involucro_args.insert(6, '-set')
involucro_args.insert(7, "TEST_BINDS='%s'" % ",".join(test_bind))
print(" ".join(involucro_context.build_command(involucro_args)))
if not dry_run:
ensure_installed(involucro_context, True)
if singularity:
if not os.path.exists(singularity_image_dir):
safe_makedirs(singularity_image_dir)
with open(os.path.join(singularity_image_dir, 'Singularity'), 'w+') as sin_def:
fill_template = SINGULARITY_TEMPLATE % {'container_test': test}
sin_def.write(fill_template)
with PrintProgress():
ret = involucro_context.exec_command(involucro_args)
if singularity:
# we can not remove this folder as it contains the image wich is owned by root
pass
# shutil.rmtree('./singularity_import')
return ret
return 0
def output_secondary_files_dir(self, output_name, create=False):
extra_files_dir = self._output_extra_files_dir(output_name)
secondary_files_dir = os.path.join(extra_files_dir, SECONDARY_FILES_EXTRA_PREFIX)
if create and not os.path.exists(secondary_files_dir):
safe_makedirs(secondary_files_dir)
return secondary_files_dir
def make_files_path():
safe_makedirs(files_path)
