def check_user_library_import_dir(self, user): if getattr(self.app.config, "user_library_import_dir_auto_creation", False): # try to create a user library import directory try: safe_makedirs(os.path.join(self.app.config.user_library_import_dir, user.email)) except ConfigurationError as e: self.log_event(str(e))
def dataset_wrapper_to_file_json(inputs_dir, dataset_wrapper): if dataset_wrapper.ext == "expression.json": with open(dataset_wrapper.file_name, "r") as f: return json.load(f) if dataset_wrapper.ext == "directory": return dataset_wrapper_to_directory_json(inputs_dir, dataset_wrapper) extra_files_path = dataset_wrapper.extra_files_path secondary_files_path = os.path.join(extra_files_path, "__secondary_files__") path = str(dataset_wrapper) raw_file_object = {"class": "File"} if os.path.exists(secondary_files_path): safe_makedirs(inputs_dir) name = os.path.basename(path) new_input_path = os.path.join(inputs_dir, name) os.symlink(path, new_input_path) secondary_files = [] for secondary_file_name in os.listdir(secondary_files_path): secondary_file_path = os.path.join(secondary_files_path, secondary_file_name) target = os.path.join(inputs_dir, secondary_file_name) log.info("linking [%s] to [%s]" % (secondary_file_path, target)) os.symlink(secondary_file_path, target) is_dir = os.path.isdir(os.path.realpath(secondary_file_path)) secondary_files.append({"class": "File" if not is_dir else "Directory", "location": target}) raw_file_object["secondaryFiles"] = secondary_files path = new_input_path raw_file_object["location"] = path raw_file_object["size"] = int(dataset_wrapper.get_size()) set_basename_and_derived_properties(raw_file_object, str(dataset_wrapper.cwl_filename or dataset_wrapper.name)) return raw_file_object
def output_secondary_files_dir(self, output_name, create=False): extra_files_dir = self._output_extra_files_dir(output_name) secondary_files_dir = os.path.join(extra_files_dir, SECONDARY_FILES_EXTRA_PREFIX) if create and not os.path.exists(secondary_files_dir): safe_makedirs(secondary_files_dir) return secondary_files_dir
def check_user_library_import_dir(self, user): if getattr(self.app.config, "user_library_import_dir_auto_creation", False): # try to create a user library import directory try: safe_makedirs(os.path.join(self.app.config.user_library_import_dir, user.email)) except ConfigurationError as e: self.log_event(unicodify(e))
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.cache_directory = kwargs.get( "cache_directory", os.path.join(kwargs['app_info'].container_image_cache_path, "singularity", "mulled")) safe_makedirs(self.cache_directory)
def test_from_directory(): target_dir = tempfile.mkdtemp() dir_1 = os.path.join(target_dir, "dir1") dir_2 = os.path.join(target_dir, "dir2") safe_makedirs(dir_1) safe_makedirs(dir_2) with open(os.path.join(dir_1, "foo.json"), "w") as f: json.dump(EXAMPLE_1, f) with open(os.path.join(dir_2, "seqanalysis.yml"), "w") as f: f.write(""" name: Sequence Analysis type: generic items: - type: workflow id: 12345abcd """) config_param = f"{os.path.abspath(dir_1)},{os.path.abspath(dir_2)}" try: sources = StaticToolBoxViewSources(view_directories=config_param) defs = sources.get_definitions() assert len(defs) == 2 assert defs[0].id == "rna" assert defs[1].id == "seqanalysis" finally: shutil.rmtree(target_dir)
def main(argv=None): mod_docstring = sys.modules[__name__].__doc__ arg_parser = ArgumentParser(description=mod_docstring) arg_parser.add_argument("--conda_prefix", required=True) args = arg_parser.parse_args(argv) conda_prefix = args.conda_prefix safe_makedirs(os.path.dirname(conda_prefix)) conda_context = CondaContext(conda_prefix=conda_prefix, ) install_conda(conda_context)
def setup_external_metadata(self, datasets_dict, sa_session, exec_dir=None, tmp_dir=None, dataset_files_path=None, output_fnames=None, config_root=None, config_file=None, datatypes_config=None, job_metadata=None, compute_tmp_dir=None, include_command=True, max_metadata_value_size=0, kwds=None): assert job_metadata, "setup_external_metadata must be supplied with job_metadata path" kwds = kwds or {} tmp_dir = _init_tmp_dir(tmp_dir) metadata_dir = os.path.join(tmp_dir, "metadata") # may already exist (i.e. metadata collection in the job handler) safe_makedirs(metadata_dir) def job_relative_path(path): path_relative = os.path.relpath(path, tmp_dir) return path_relative outputs = {} for name, dataset in datasets_dict.items(): assert name is not None assert name not in outputs key = name def _metadata_path(what): return os.path.join(metadata_dir, "metadata_%s_%s" % (what, key)) _initialize_metadata_inputs(dataset, _metadata_path, tmp_dir, kwds) outputs[name] = { "filename_override": _get_filename_override(output_fnames, dataset.file_name) } metadata_params_path = os.path.join(metadata_dir, "params.json") metadata_params = { "job_metadata": job_relative_path(job_metadata), "datatypes_config": datatypes_config, "max_metadata_value_size": max_metadata_value_size, "outputs": outputs, "object_store_store_by": galaxy.model.Dataset.object_store.store_by, } with open(metadata_params_path, "w") as f: json.dump(metadata_params, f) if include_command: # return command required to build script_path = os.path.join(metadata_dir, "set.py") with open(script_path, "w") as f: f.write(SET_METADATA_SCRIPT) return 'python "metadata/set.py"' else: # return args to galaxy_ext.metadata.set_metadata required to build return ''
def create(self, obj, **kwargs): """Override `ObjectStore`'s stub by creating any files and folders on disk.""" if not self.exists(obj, **kwargs): path = self._construct_path(obj, **kwargs) dir_only = kwargs.get('dir_only', False) # Create directory if it does not exist dir = path if dir_only else os.path.dirname(path) safe_makedirs(dir) # Create the file if it does not exist if not dir_only: open(path, 'w').close() # Should be rb? umask_fix_perms(path, self.config.umask, 0o666)
def create(self, trans, job_id, payload, **kwargs): """ create( self, trans, job_id, payload, **kwargs ) * POST /api/jobs/{job_id}/files Populate an output file (formal dataset, task split part, working directory file (such as those related to metadata)). This should be a multipart post with a 'file' parameter containing the contents of the actual file to create. :type job_id: str :param job_id: encoded id string of the job :type payload: dict :param payload: dictionary structure containing:: 'job_key' = Key authenticating 'path' = Path to file to create. ..note: This API method is intended only for consumption by job runners, not end users. :rtype: dict :returns: an okay message """ job = self.__authorize_job_access(trans, job_id, **payload) path = payload.get("path") self.__check_job_can_write_to_path(trans, job, path) # Is this writing an unneeded file? Should this just copy in Python? if '__file_path' in payload: file_path = payload.get('__file_path') upload_store = trans.app.config.nginx_upload_job_files_store assert upload_store, ("Request appears to have been processed by" " nginx_upload_module but Galaxy is not" " configured to recognize it") assert file_path.startswith(upload_store), \ ("Filename provided by nginx (%s) is not in correct" " directory (%s)" % (file_path, upload_store)) input_file = open(file_path) else: input_file = payload.get("file", payload.get("__file", None)).file target_dir = os.path.dirname(path) util.safe_makedirs(target_dir) try: shutil.move(input_file.name, path) finally: try: input_file.close() except OSError: # Fails to close file if not using nginx upload because the # tempfile has moved and Python wants to delete it. pass return {"message": "ok"}
def __write_workdir_file(self, config_filename, content, context, is_template=True, strip=False): parent_dir = os.path.dirname(config_filename) if not os.path.exists(parent_dir): safe_makedirs(parent_dir) if is_template: value = fill_template(content, context=context, python_template_version=self.tool.python_template_version) else: value = unicodify(content) if strip: value = value.strip() with open(config_filename, "w", encoding='utf-8') as f: f.write(value) # For running jobs as the actual user, ensure the config file is globally readable os.chmod(config_filename, RW_R__R__)
def _write_to_file(args, f, path): if hasattr(f, "getvalue"): contents = f.getvalue() else: contents = f if args.dry_run: contents_indented = "\n".join(f" |{line}" for line in contents.splitlines()) print(f"Overwriting {path} with the following contents:\n{contents_indented}") print("... skipping because --dry-run is enabled.") else: print(f"Overwriting {path}") safe_makedirs(os.path.dirname(path)) with open(path, "w") as to_f: to_f.write(contents)
def move_directory(output, target_path, output_name=None): assert output["class"] == "Directory" output_path = _possible_uri_to_path(output["location"]) if output_path.startswith("_:"): # No a real path, just copy listing to target path. safe_makedirs(target_path) for listed_file in output["listing"]: # TODO: handle directories assert listed_file["class"] == "File" file_description = file_dict_to_description(listed_file) file_description.write_to(os.path.join(target_path, listed_file["basename"])) else: shutil.move(output_path, target_path) return {"cwl_filename": output["basename"]}
def _write_to_file(args, f, path): dry_run = args.dry_run if hasattr(f, "getvalue"): contents = f.getvalue() else: contents = f contents_indented = "\n".join([" |%s" % l for l in contents.splitlines()]) print("Writing the file contents:\n%s\nto %s" % (contents_indented, path)) if dry_run: print("... skipping because --dry-run is enabled.") else: safe_makedirs(os.path.dirname(path)) with open(path, "w") as to_f: to_f.write(contents)
def move_directory(output, target_path, output_name=None): assert output["class"] == "Directory" output_path = _possible_uri_to_path(output["location"]) if output_path.startswith("_:"): # No a real path, just copy listing to target path. safe_makedirs(target_path) for listed_file in output["listing"]: # TODO: handle directories assert listed_file["class"] == "File" file_description = file_dict_to_description(listed_file) file_description.write_to(os.path.join(target_path, listed_file["basename"])) else: shutil.move(output_path, target_path) return {"created_from_basename": output["basename"]}
def simple_value(input, param_dict_value, cwl_type=None): # Hmm... cwl_type isn't really the cwl type in every case, # like in the case of json for instance. if cwl_type is None: input_type = input.type cwl_type = GALAXY_TO_CWL_TYPES[input_type] if cwl_type == "null": assert param_dict_value is None return None if cwl_type == "File": dataset_wrapper = param_dict_value extra_files_path = dataset_wrapper.extra_files_path secondary_files_path = os.path.join(extra_files_path, "__secondary_files__") path = str(dataset_wrapper) if os.path.exists(secondary_files_path): safe_makedirs(inputs_dir) name = os.path.basename(path) new_input_path = os.path.join(inputs_dir, name) os.symlink(path, new_input_path) for secondary_file_name in os.listdir(secondary_files_path): secondary_file_path = os.path.join(secondary_files_path, secondary_file_name) os.symlink(secondary_file_path, new_input_path + secondary_file_name) path = new_input_path return {"location": path, "class": "File"} elif cwl_type == "integer": return int(str(param_dict_value)) elif cwl_type == "long": return int(str(param_dict_value)) elif cwl_type == "float": return float(str(param_dict_value)) elif cwl_type == "double": return float(str(param_dict_value)) elif cwl_type == "boolean": return string_as_bool(param_dict_value) elif cwl_type == "text": return str(param_dict_value) elif cwl_type == "json": raw_value = param_dict_value.value log.info("raw_value is %s (%s)" % (raw_value, type(raw_value))) return json.loads(raw_value) else: return str(param_dict_value)
def get_dataset(dataset_details, filename=None): parent_basename = dataset_details.get("cwl_file_name") or dataset_details.get("name") file_ext = dataset_details["file_ext"] if file_ext == "directory": # TODO: rename output_directory to outputs_directory because we can have output directories # and this is confusing... the_output_directory = os.path.join(output_directory, parent_basename) safe_makedirs(the_output_directory) destination = self.download_output_to(ctx, dataset_details, the_output_directory, filename=filename) else: destination = self.download_output_to(ctx, dataset_details, output_directory, filename=filename) if filename is None: basename = parent_basename else: basename = os.path.basename(filename) return {"path": destination, "basename": basename}
def dataset_wrapper_to_file_json(inputs_dir, dataset_wrapper): if dataset_wrapper.ext == "expression.json": with open(dataset_wrapper.file_name) as f: return json.load(f) if dataset_wrapper.ext == "directory": return dataset_wrapper_to_directory_json(inputs_dir, dataset_wrapper) extra_files_path = dataset_wrapper.extra_files_path secondary_files_path = os.path.join(extra_files_path, "__secondary_files__") path = str(dataset_wrapper) raw_file_object = {"class": "File"} if os.path.exists(secondary_files_path): safe_makedirs(inputs_dir) name = os.path.basename(path) new_input_path = os.path.join(inputs_dir, name) os.symlink(path, new_input_path) secondary_files = [] for secondary_file_name in os.listdir(secondary_files_path): secondary_file_path = os.path.join(secondary_files_path, secondary_file_name) target = os.path.join(inputs_dir, secondary_file_name) log.info("linking [{}] to [{}]".format(secondary_file_path, target)) os.symlink(secondary_file_path, target) is_dir = os.path.isdir(os.path.realpath(secondary_file_path)) secondary_files.append({ "class": "File" if not is_dir else "Directory", "location": target }) raw_file_object["secondaryFiles"] = secondary_files path = new_input_path raw_file_object["location"] = path # Verify it isn't a NoneDataset if dataset_wrapper.unsanitized: raw_file_object["size"] = int(dataset_wrapper.get_size()) set_basename_and_derived_properties( raw_file_object, str(dataset_wrapper.created_from_basename or dataset_wrapper.name)) return raw_file_object
def get_dataset(dataset_details, filename=None): parent_basename = dataset_details.get("cwl_file_name") if not parent_basename: parent_basename = dataset_details.get("name") file_ext = dataset_details["file_ext"] if file_ext == "directory": # TODO: rename output_directory to outputs_directory because we can have output directories # and this is confusing... the_output_directory = os.path.join(output_directory, parent_basename) safe_makedirs(the_output_directory) destination = self.download_output_to(dataset_details, the_output_directory, filename=filename) else: destination = self.download_output_to(dataset_details, output_directory, filename=filename) if filename is None: basename = parent_basename else: basename = os.path.basename(filename) return {"path": destination, "basename": basename}
def handle_galaxy_config_kwds(cls, config): amqp_url = os.environ.get("GALAXY_TEST_AMQP_URL", None) if amqp_url is None: pytest.skip("External AMQP URL not configured for test") jobs_directory = os.path.join(cls._test_driver.mkdtemp(), "pulsar_staging") safe_makedirs(jobs_directory) job_conf_template = string.Template(JOB_CONF_TEMPLATE) job_conf_str = job_conf_template.substitute( amqp_url=AMQP_URL, jobs_directory=jobs_directory, galaxy_home=os.path.join(SCRIPT_DIRECTORY, os.pardir)) with tempfile.NamedTemporaryFile(suffix="_mq_job_conf.yml", mode="w", delete=False) as job_conf: job_conf.write(job_conf_str) config["job_config_file"] = job_conf.name infrastructure_url = "http://localhost:$GALAXY_WEB_PORT" config["galaxy_infrastructure_url"] = infrastructure_url
def rewrite_inputs_for_staging(self): if hasattr(self._cwl_job, "pathmapper"): pass # DO SOMETHING LIKE THE FOLLOWING? # path_rewrites = {} # for f, p in self._cwl_job.pathmapper.items(): # if not p.staged: # continue # if p.type in ("File", "Directory"): # path_rewrites[p.resolved] = p.target # for key, value in self._input_dict.items(): # if key in path_rewrites: # self._input_dict[key]["location"] = path_rewrites[value] else: stagedir = os.path.join(self._job_directory, "cwlstagedir") safe_makedirs(stagedir) def stage_recursive(value): is_list = isinstance(value, list) is_dict = isinstance(value, dict) log.info( f"handling value {value}, is_list {is_list}, is_dict {is_dict}" ) if is_list: for val in value: stage_recursive(val) elif is_dict: if "location" in value and "basename" in value: location = value["location"] basename = value["basename"] if not location.endswith(basename): # TODO: sep()[-1] staged_loc = os.path.join(stagedir, basename) if not os.path.exists(staged_loc): os.symlink(location, staged_loc) value["location"] = staged_loc for dict_value in value.values(): stage_recursive(dict_value) else: log.info("skipping simple value...") stage_recursive(self._input_dict)
def walk_extra_files(items, prefix=""): for item in items: if "elements" in item: name = item.get("name") if not prefix: item_prefix = name else: item_prefix = os.path.join(prefix, name) walk_extra_files(item.get("elements"), prefix=item_prefix) else: name, src_path = _has_src_to_path(upload_config, item) if prefix: rel_path = os.path.join(prefix, name) else: rel_path = name file_output_path = os.path.join(staged_extra_files, rel_path) parent_dir = os.path.dirname(file_output_path) if not os.path.exists(parent_dir): safe_makedirs(parent_dir) shutil.move(src_path, file_output_path)
def dataset_wrapper_to_file_json(inputs_dir, dataset_wrapper): if dataset_wrapper.ext == "expression.json": with open(dataset_wrapper.file_name, "r") as f: return json.load(f) extra_files_path = dataset_wrapper.extra_files_path secondary_files_path = os.path.join(extra_files_path, "__secondary_files__") path = str(dataset_wrapper) if os.path.exists(secondary_files_path): safe_makedirs(inputs_dir) name = os.path.basename(path) new_input_path = os.path.join(inputs_dir, name) os.symlink(path, new_input_path) for secondary_file_name in os.listdir(secondary_files_path): secondary_file_path = os.path.join(secondary_files_path, secondary_file_name) os.symlink(secondary_file_path, new_input_path + secondary_file_name) path = new_input_path return {"location": path, "class": "File"}
def metadata_command(self, output_datasets, output_collections=None): output_collections = output_collections or {} metadata_compute_strategy = get_metadata_compute_strategy( self.app.config, self.job.id) self.metadata_compute_strategy = metadata_compute_strategy exec_dir = None dataset_files_path = self.app.model.Dataset.file_path config_root = self.app.config.root config_file = None datatypes_config = os.path.join(self.job_working_directory, 'metadata', 'registry.xml') safe_makedirs(os.path.join(self.job_working_directory, 'metadata')) self.app.datatypes_registry.to_xml_file(path=datatypes_config) job_metadata = os.path.join(self.tool_working_directory, self.tool.provided_metadata_file) output_fnames = [ DatasetPath(o.dataset.id, o.dataset.file_name, None) for o in output_datasets.values() ] command = metadata_compute_strategy.setup_external_metadata( output_datasets, output_collections, self.app.model.session, exec_dir=exec_dir, tmp_dir=self. job_working_directory, # set in jobs/runners.py - better if was default. dataset_files_path=dataset_files_path, config_root=config_root, config_file=config_file, datatypes_config=datatypes_config, job_metadata=job_metadata, output_fnames=output_fnames, tool=self.tool, job=self.job, object_store_conf=self.app.object_store.to_dict(), max_metadata_value_size=10000) return command
def rewrite_inputs_for_staging(self): if hasattr(self._cwl_job, "pathmapper"): pass # DO SOMETHING LIKE THE FOLLOWING? # path_rewrites = {} # for f, p in self._cwl_job.pathmapper.items(): # if not p.staged: # continue # if p.type in ("File", "Directory"): # path_rewrites[p.resolved] = p.target # for key, value in self._input_dict.items(): # if key in path_rewrites: # self._input_dict[key]["location"] = path_rewrites[value] else: stagedir = os.path.join(self._job_directory, "cwlstagedir") safe_makedirs(stagedir) def stage_recursive(value): is_list = isinstance(value, list) is_dict = isinstance(value, dict) log.info("handling value %s, is_list %s, is_dict %s" % (value, is_list, is_dict)) if is_list: for val in value: stage_recursive(val) elif is_dict: if "location" in value and "basename" in value: location = value["location"] basename = value["basename"] if not location.endswith(basename): # TODO: sep()[-1] staged_loc = os.path.join(stagedir, basename) if not os.path.exists(staged_loc): os.symlink(location, staged_loc) value["location"] = staged_loc for key, dict_value in value.items(): stage_recursive(dict_value) else: log.info("skipping simple value...") stage_recursive(self._input_dict)
def mull_targets( targets, involucro_context=None, command="build", channels=DEFAULT_CHANNELS, namespace="biocontainers", test='true', test_files=None, image_build=None, name_override=None, repository_template=DEFAULT_REPOSITORY_TEMPLATE, dry_run=False, conda_version=None, verbose=False, binds=DEFAULT_BINDS, rebuild=True, oauth_token=None, hash_func="v2", singularity=False, singularity_image_dir="singularity_import", base_image=None, determine_base_image=True, ): targets = list(targets) if involucro_context is None: involucro_context = InvolucroContext() image_function = v1_image_name if hash_func == "v1" else v2_image_name if len(targets) > 1 and image_build is None: # Force an image build in this case - this seems hacky probably # shouldn't work this way but single case broken else wise. image_build = "0" repo_template_kwds = { "namespace": namespace, "image": image_function(targets, image_build=image_build, name_override=name_override) } repo = string.Template(repository_template).safe_substitute( repo_template_kwds) if not rebuild or "push" in command: repo_name = repo_template_kwds["image"].split(":", 1)[0] repo_data = quay_repository(repo_template_kwds["namespace"], repo_name) if not rebuild: tags = repo_data.get("tags", []) target_tag = None if ":" in repo_template_kwds["image"]: image_name_parts = repo_template_kwds["image"].split(":") assert len( image_name_parts ) == 2, f": not allowed in image name [{repo_template_kwds['image']}]" target_tag = image_name_parts[1] if tags and (target_tag is None or target_tag in tags): raise BuildExistsException() if "push" in command and "error_type" in repo_data and oauth_token: # Explicitly create the repository so it can be built as public. create_repository(repo_template_kwds["namespace"], repo_name, oauth_token) for channel in channels: if channel.startswith('file://'): bind_path = channel[7:] binds.append(f'/{bind_path}:/{bind_path}') channels = ",".join(channels) target_str = ",".join(map(conda_build_target_str, targets)) bind_str = ",".join(binds) involucro_args = [ '-f', f'{DIRNAME}/invfile.lua', '-set', f"CHANNELS={channels}", '-set', f"TARGETS={target_str}", '-set', f"REPO={repo}", '-set', f"BINDS={bind_str}", ] dest_base_image = None if base_image: dest_base_image = base_image elif DEST_BASE_IMAGE: dest_base_image = DEST_BASE_IMAGE elif determine_base_image: dest_base_image = base_image_for_targets(targets) if dest_base_image: involucro_args.extend(["-set", f"DEST_BASE_IMAGE={dest_base_image}"]) if CONDA_IMAGE: involucro_args.extend(["-set", f"CONDA_IMAGE={CONDA_IMAGE}"]) if verbose: involucro_args.extend(["-set", "VERBOSE=1"]) if singularity: singularity_image_name = repo_template_kwds['image'] involucro_args.extend(["-set", "SINGULARITY=1"]) involucro_args.extend( ["-set", f"SINGULARITY_IMAGE_NAME={singularity_image_name}"]) involucro_args.extend( ["-set", f"SINGULARITY_IMAGE_DIR={singularity_image_dir}"]) involucro_args.extend(["-set", f"USER_ID={os.getuid()}:{os.getgid()}"]) if test: involucro_args.extend(["-set", f"TEST={test}"]) if conda_version is not None: verbose = "--verbose" if verbose else "--quiet" involucro_args.extend([ "-set", f"PREINSTALL=conda install {verbose} --yes conda={conda_version}" ]) involucro_args.append(command) if test_files: test_bind = [] for test_file in test_files: if ':' not in test_file: if os.path.exists(test_file): test_bind.append( f"{test_file}:{DEFAULT_WORKING_DIR}/{test_file}") else: if os.path.exists(test_file.split(':')[0]): test_bind.append(test_file) if test_bind: involucro_args.insert(6, '-set') involucro_args.insert(7, f"TEST_BINDS={','.join(test_bind)}") cmd = involucro_context.build_command(involucro_args) print(f"Executing: {' '.join(shlex.quote(_) for _ in cmd)}") if dry_run: return 0 ensure_installed(involucro_context, True) if singularity: if not os.path.exists(singularity_image_dir): safe_makedirs(singularity_image_dir) with open(os.path.join(singularity_image_dir, 'Singularity.def'), 'w+') as sin_def: fill_template = SINGULARITY_TEMPLATE % { 'container_test': test, 'base_image': dest_base_image or DEFAULT_BASE_IMAGE } sin_def.write(fill_template) with PrintProgress(): ret = involucro_context.exec_command(involucro_args) if singularity: # we can not remove this folder as it contains the image wich is owned by root pass # shutil.rmtree('./singularity_import') return ret
def make_files_path(): safe_makedirs(files_path)
def __init__(self, results_path): safe_makedirs(results_path) self.lifecycle = AllureLifecycle() self.logger = AllureFileLogger(results_path) self.listener = AllureListener(self.lifecycle)
def _init_cache_directory(self): cacher_class = get_cache_directory_cacher( self.cache_directory_cacher_type) self.cache_directory = cacher_class(self.cache_directory_path, hash_func=self.hash_func) safe_makedirs(self.cache_directory.path)
def setup_external_metadata(self, datasets_dict, out_collections, sa_session, exec_dir=None, tmp_dir=None, dataset_files_path=None, output_fnames=None, config_root=None, use_bin=False, config_file=None, datatypes_config=None, job_metadata=None, provided_metadata_style=None, compute_tmp_dir=None, include_command=True, max_metadata_value_size=0, validate_outputs=False, object_store_conf=None, tool=None, job=None, kwds=None): assert job_metadata, "setup_external_metadata must be supplied with job_metadata path" kwds = kwds or {} if not job: job = sa_session.query(galaxy.model.Job).get(self.job_id) tmp_dir = _init_tmp_dir(tmp_dir) metadata_dir = os.path.join(tmp_dir, "metadata") # may already exist (i.e. metadata collection in the job handler) safe_makedirs(metadata_dir) def job_relative_path(path): path_relative = os.path.relpath(path, tmp_dir) return path_relative outputs = {} output_collections = {} for name, dataset in datasets_dict.items(): assert name is not None assert name not in outputs key = name def _metadata_path(what): return os.path.join(metadata_dir, f"metadata_{what}_{key}") _initialize_metadata_inputs( dataset, _metadata_path, tmp_dir, kwds, real_metadata_object=self.write_object_store_conf) outputs[name] = { "filename_override": _get_filename_override(output_fnames, dataset.file_name), "validate": validate_outputs, "object_store_store_by": dataset.dataset.store_by, 'id': dataset.id, 'model_class': 'LibraryDatasetDatasetAssociation' if isinstance( dataset, galaxy.model.LibraryDatasetDatasetAssociation) else 'HistoryDatasetAssociation' } metadata_params_path = os.path.join(metadata_dir, "params.json") datatypes_config = os.path.relpath(datatypes_config, tmp_dir) metadata_params = { "job_metadata": job_relative_path(job_metadata), "provided_metadata_style": provided_metadata_style, "datatypes_config": datatypes_config, "max_metadata_value_size": max_metadata_value_size, "outputs": outputs, } # export model objects and object store configuration for extended metadata also. export_directory = os.path.join(metadata_dir, "outputs_new") with DirectoryModelExportStore(export_directory, for_edit=True, serialize_dataset_objects=True, serialize_jobs=False) as export_store: for dataset in datasets_dict.values(): export_store.add_dataset(dataset) for name, dataset_collection in out_collections.items(): export_store.add_dataset_collection(dataset_collection) output_collections[name] = { 'id': dataset_collection.id, } if self.write_object_store_conf: with open(os.path.join(metadata_dir, "object_store_conf.json"), "w") as f: json.dump(object_store_conf, f) # setup tool tool_as_dict = {} tool_as_dict["stdio_exit_codes"] = [ e.to_dict() for e in tool.stdio_exit_codes ] tool_as_dict["stdio_regexes"] = [ r.to_dict() for r in tool.stdio_regexes ] tool_as_dict["outputs"] = { name: output.to_dict() for name, output in tool.outputs.items() } tool_as_dict["output_collections"] = { name: output.to_dict() for name, output in tool.output_collections.items() } # setup the rest metadata_params["tool"] = tool_as_dict metadata_params["tool_path"] = tool.config_file metadata_params["job_id_tag"] = job.get_id_tag() metadata_params["job_params"] = job.raw_param_dict() metadata_params["output_collections"] = output_collections with open(metadata_params_path, "w") as f: json.dump(metadata_params, f) if include_command: # return command required to build if use_bin: return "galaxy-set-metadata" else: script_path = os.path.join(metadata_dir, "set.py") with open(script_path, "w") as f: f.write(SET_METADATA_SCRIPT) return 'python "metadata/set.py"' else: # return args to galaxy_ext.metadata.set_metadata required to build return ''
def _init_tmp_dir(tmp_dir): assert tmp_dir is not None safe_makedirs(tmp_dir) return tmp_dir
def ensure_configs_directory(work_dir): configs_dir = os.path.join(work_dir, "configs") if not os.path.exists(configs_dir): safe_makedirs(configs_dir) return configs_dir
def mull_targets( targets, involucro_context=None, command="build", channels=DEFAULT_CHANNELS, namespace="biocontainers", test='true', test_files=None, image_build=None, name_override=None, repository_template=DEFAULT_REPOSITORY_TEMPLATE, dry_run=False, conda_version=None, verbose=False, binds=DEFAULT_BINDS, rebuild=True, oauth_token=None, hash_func="v2", singularity=False, singularity_image_dir="singularity_import", ): targets = list(targets) if involucro_context is None: involucro_context = InvolucroContext() image_function = v1_image_name if hash_func == "v1" else v2_image_name if len(targets) > 1 and image_build is None: # Force an image build in this case - this seems hacky probably # shouldn't work this way but single case broken else wise. image_build = "0" repo_template_kwds = { "namespace": namespace, "image": image_function(targets, image_build=image_build, name_override=name_override) } repo = string.Template(repository_template).safe_substitute( repo_template_kwds) if not rebuild or "push" in command: repo_name = repo_template_kwds["image"].split(":", 1)[0] repo_data = quay_repository(repo_template_kwds["namespace"], repo_name) if not rebuild: tags = repo_data.get("tags", []) target_tag = None if ":" in repo_template_kwds["image"]: image_name_parts = repo_template_kwds["image"].split(":") assert len( image_name_parts ) == 2, ": not allowed in image name [%s]" % repo_template_kwds[ "image"] target_tag = image_name_parts[1] if tags and (target_tag is None or target_tag in tags): raise BuildExistsException() if "push" in command and "error_type" in repo_data and oauth_token: # Explicitly create the repository so it can be built as public. create_repository(repo_template_kwds["namespace"], repo_name, oauth_token) for channel in channels: if channel.startswith('file://'): bind_path = channel.lstrip('file://') binds.append('/%s:/%s' % (bind_path, bind_path)) channels = ",".join(channels) target_str = ",".join(map(conda_build_target_str, targets)) bind_str = ",".join(binds) involucro_args = [ '-f', '%s/invfile.lua' % DIRNAME, '-set', "CHANNELS='%s'" % channels, '-set', "TARGETS='%s'" % target_str, '-set', "REPO='%s'" % repo, '-set', "BINDS='%s'" % bind_str, ] if DEST_BASE_IMAGE: involucro_args.extend( ["-set", "DEST_BASE_IMAGE='%s'" % DEST_BASE_IMAGE]) if CONDA_IMAGE: involucro_args.extend(["-set", "CONDA_IMAGE='%s'" % CONDA_IMAGE]) if verbose: involucro_args.extend(["-set", "VERBOSE='1'"]) if singularity: singularity_image_name = repo_template_kwds['image'] involucro_args.extend(["-set", "SINGULARITY='1'"]) involucro_args.extend( ["-set", "SINGULARITY_IMAGE_NAME='%s'" % singularity_image_name]) involucro_args.extend( ["-set", "SINGULARITY_IMAGE_DIR='%s'" % singularity_image_dir]) involucro_args.extend( ["-set", "USER_ID='%s:%s'" % (os.getuid(), os.getgid())]) if test: involucro_args.extend(["-set", "TEST=%s" % shlex_quote(test)]) if conda_version is not None: verbose = "--verbose" if verbose else "--quiet" involucro_args.extend([ "-set", "PREINSTALL='conda install %s --yes conda=%s'" % (verbose, conda_version) ]) involucro_args.append(command) if test_files: test_bind = [] for test_file in test_files: if ':' not in test_file: if os.path.exists(test_file): test_bind.append( "%s:%s/%s" % (test_file, DEFAULT_WORKING_DIR, test_file)) else: if os.path.exists(test_file.split(':')[0]): test_bind.append(test_file) if test_bind: involucro_args.insert(6, '-set') involucro_args.insert(7, "TEST_BINDS='%s'" % ",".join(test_bind)) print(" ".join(involucro_context.build_command(involucro_args))) if not dry_run: ensure_installed(involucro_context, True) if singularity: if not os.path.exists(singularity_image_dir): safe_makedirs(singularity_image_dir) with open(os.path.join(singularity_image_dir, 'Singularity'), 'w+') as sin_def: fill_template = SINGULARITY_TEMPLATE % {'container_test': test} sin_def.write(fill_template) with PrintProgress(): ret = involucro_context.exec_command(involucro_args) if singularity: # we can not remove this folder as it contains the image wich is owned by root pass # shutil.rmtree('./singularity_import') return ret return 0
def mull_targets( targets, involucro_context=None, command="build", channels=DEFAULT_CHANNELS, namespace="biocontainers", test='true', test_files=None, image_build=None, name_override=None, repository_template=DEFAULT_REPOSITORY_TEMPLATE, dry_run=False, conda_version=None, verbose=False, binds=DEFAULT_BINDS, rebuild=True, oauth_token=None, hash_func="v2", singularity=False, singularity_image_dir="singularity_import", ): targets = list(targets) if involucro_context is None: involucro_context = InvolucroContext() image_function = v1_image_name if hash_func == "v1" else v2_image_name if len(targets) > 1 and image_build is None: # Force an image build in this case - this seems hacky probably # shouldn't work this way but single case broken else wise. image_build = "0" repo_template_kwds = { "namespace": namespace, "image": image_function(targets, image_build=image_build, name_override=name_override) } repo = string.Template(repository_template).safe_substitute(repo_template_kwds) if not rebuild or "push" in command: repo_name = repo_template_kwds["image"].split(":", 1)[0] repo_data = quay_repository(repo_template_kwds["namespace"], repo_name) if not rebuild: tags = repo_data.get("tags", []) target_tag = None if ":" in repo_template_kwds["image"]: image_name_parts = repo_template_kwds["image"].split(":") assert len(image_name_parts) == 2, ": not allowed in image name [%s]" % repo_template_kwds["image"] target_tag = image_name_parts[1] if tags and (target_tag is None or target_tag in tags): raise BuildExistsException() if "push" in command and "error_type" in repo_data and oauth_token: # Explicitly create the repository so it can be built as public. create_repository(repo_template_kwds["namespace"], repo_name, oauth_token) for channel in channels: if channel.startswith('file://'): bind_path = channel.lstrip('file://') binds.append('/%s:/%s' % (bind_path, bind_path)) channels = ",".join(channels) target_str = ",".join(map(conda_build_target_str, targets)) bind_str = ",".join(binds) involucro_args = [ '-f', '%s/invfile.lua' % DIRNAME, '-set', "CHANNELS='%s'" % channels, '-set', "TEST=%s" % shlex_quote(test), '-set', "TARGETS='%s'" % target_str, '-set', "REPO='%s'" % repo, '-set', "BINDS='%s'" % bind_str, ] if DEST_BASE_IMAGE: involucro_args.extend(["-set", "DEST_BASE_IMAGE='%s'" % DEST_BASE_IMAGE]) if CONDA_IMAGE: involucro_args.extend(["-set", "CONDA_IMAGE='%s'" % CONDA_IMAGE]) if verbose: involucro_args.extend(["-set", "VERBOSE='1'"]) if singularity: singularity_image_name = repo_template_kwds['image'] involucro_args.extend(["-set", "SINGULARITY='1'"]) involucro_args.extend(["-set", "SINGULARITY_IMAGE_NAME='%s'" % singularity_image_name]) involucro_args.extend(["-set", "SINGULARITY_IMAGE_DIR='%s'" % singularity_image_dir]) involucro_args.extend(["-set", "USER_ID='%s:%s'" % (os.getuid(), os.getgid())]) if conda_version is not None: verbose = "--verbose" if verbose else "--quiet" involucro_args.extend(["-set", "PREINSTALL='conda install %s --yes conda=%s'" % (verbose, conda_version)]) involucro_args.append(command) if test_files: test_bind = [] for test_file in test_files: if ':' not in test_file: if os.path.exists(test_file): test_bind.append("%s:%s/%s" % (test_file, DEFAULT_WORKING_DIR, test_file)) else: if os.path.exists(test_file.split(':')[0]): test_bind.append(test_file) if test_bind: involucro_args.insert(6, '-set') involucro_args.insert(7, "TEST_BINDS='%s'" % ",".join(test_bind)) print(" ".join(involucro_context.build_command(involucro_args))) if not dry_run: ensure_installed(involucro_context, True) if singularity: if not os.path.exists(singularity_image_dir): safe_makedirs(singularity_image_dir) with open(os.path.join(singularity_image_dir, 'Singularity'), 'w+') as sin_def: fill_template = SINGULARITY_TEMPLATE % {'container_test': test} sin_def.write(fill_template) with PrintProgress(): ret = involucro_context.exec_command(involucro_args) if singularity: # we can not remove this folder as it contains the image wich is owned by root pass # shutil.rmtree('./singularity_import') return ret return 0