def stage(self, tool_or_workflow, history_id, job=None, job_path=None, use_path_paste=LOAD_TOOLS_FROM_PATH, to_posix_lines=True): files_attached = [False] def upload_func_fetch(upload_target): def _attach_file(upload_payload, uri, index=0): uri = path_or_uri_to_uri(uri) is_path = uri.startswith("file://") if not is_path or use_path_paste: return {"src": "url", "url": uri} else: files_attached[0] = True path = uri[len("file://"):] upload_payload["__files"]["files_%s|file_data" % index] = self._attach_file(path) return {"src": "files"} fetch_payload = None if isinstance(upload_target, FileUploadTarget): file_path = upload_target.path file_type = upload_target.properties.get( 'filetype', None) or DEFAULT_FILE_TYPE dbkey = upload_target.properties.get('dbkey', None) or DEFAULT_DBKEY fetch_payload = _fetch_payload( history_id, file_type=file_type, dbkey=dbkey, to_posix_lines=to_posix_lines, ) name = _file_path_to_name(file_path) if file_path is not None: src = _attach_file(fetch_payload, file_path) fetch_payload["targets"][0]["elements"][0].update(src) if upload_target.composite_data: composite_items = [] for i, composite_data in enumerate( upload_target.composite_data): composite_item_src = _attach_file(fetch_payload, composite_data, index=i) composite_items.append(composite_item_src) fetch_payload["targets"][0]["elements"][0][ "src"] = "composite" fetch_payload["targets"][0]["elements"][0]["composite"] = { "items": composite_items, } tags = upload_target.properties.get("tags") if tags: fetch_payload["targets"][0]["elements"][0]["tags"] = tags fetch_payload["targets"][0]["elements"][0]["name"] = name elif isinstance(upload_target, FileLiteralTarget): fetch_payload = _fetch_payload(history_id) # For file literals - take them as is - never convert line endings. fetch_payload["targets"][0]["elements"][0].update({ "src": "pasted", "paste_content": upload_target.contents, "to_posix_lines": False, }) tags = upload_target.properties.get("tags") if tags: fetch_payload["targets"][0]["elements"][0]["tags"] = tags elif isinstance(upload_target, DirectoryUploadTarget): fetch_payload = _fetch_payload(history_id, file_type="directory") fetch_payload["targets"][0].pop("elements") tar_path = upload_target.path src = _attach_file(fetch_payload, tar_path) fetch_payload["targets"][0]["elements_from"] = src else: content = json.dumps(upload_target.object) fetch_payload = _fetch_payload(history_id, file_type="expression.json") fetch_payload["targets"][0]["elements"][0].update({ "src": "pasted", "paste_content": content, }) tags = upload_target.properties.get("tags") fetch_payload["targets"][0]["elements"][0]["tags"] = tags return self._fetch_post(fetch_payload, files_attached=files_attached[0]) # Save legacy upload_func to target older Galaxy servers def upload_func(upload_target): def _attach_file(upload_payload, uri, index=0): uri = path_or_uri_to_uri(uri) is_path = uri.startswith("file://") if not is_path or use_path_paste: upload_payload["inputs"]["files_%d|url_paste" % index] = uri else: files_attached[0] = True path = uri[len("file://"):] upload_payload["__files"]["files_%d|file_data" % index] = self._attach_file(path) if isinstance(upload_target, FileUploadTarget): file_path = upload_target.path file_type = upload_target.properties.get( 'filetype', None) or DEFAULT_FILE_TYPE dbkey = upload_target.properties.get('dbkey', None) or DEFAULT_DBKEY upload_payload = _upload_payload( history_id, file_type=file_type, to_posix_lines=dbkey, ) name = _file_path_to_name(file_path) upload_payload["inputs"]["files_0|auto_decompress"] = False upload_payload["inputs"]["auto_decompress"] = False if file_path is not None: _attach_file(upload_payload, file_path) upload_payload["inputs"]["files_0|NAME"] = name if upload_target.secondary_files: _attach_file(upload_payload, upload_target.secondary_files, index=1) upload_payload["inputs"]["files_1|type"] = "upload_dataset" upload_payload["inputs"]["files_1|auto_decompress"] = True upload_payload["inputs"]["file_count"] = "2" upload_payload["inputs"]["force_composite"] = "True" # galaxy.exceptions.RequestParameterInvalidException: Not input source type # defined for input '{'class': 'File', 'filetype': 'imzml', 'composite_data': # ['Example_Continuous.imzML', 'Example_Continuous.ibd']}'.\n"}]] if upload_target.composite_data: for i, composite_data in enumerate( upload_target.composite_data): upload_payload["inputs"]["files_%s|type" % i] = "upload_dataset" _attach_file(upload_payload, composite_data, index=i) self._log("upload_payload is %s" % upload_payload) return self._tools_post(upload_payload, files_attached=files_attached[0]) elif isinstance(upload_target, FileLiteralTarget): # For file literals - take them as is - never convert line endings. payload = _upload_payload(history_id, file_type="auto", auto_decompress=False, to_posix_lines=False) payload["inputs"]["files_0|url_paste"] = upload_target.contents return self._tools_post(payload) elif isinstance(upload_target, DirectoryUploadTarget): tar_path = upload_target.tar_path upload_payload = _upload_payload( history_id, file_type="tar", ) upload_payload["inputs"]["files_0|auto_decompress"] = False _attach_file(upload_payload, tar_path) tar_upload_response = self._tools_post( upload_payload, files_attached=files_attached[0]) convert_payload = dict( tool_id="CONVERTER_tar_to_directory", tool_inputs={ "input1": { "src": "hda", "id": tar_upload_response["outputs"][0]["id"] } }, history_id=history_id, ) convert_response = self._tools_post(convert_payload) assert "outputs" in convert_response, convert_response return convert_response else: content = json.dumps(upload_target.object) payload = _upload_payload(history_id, file_type="expression.json") payload["files_0|url_paste"] = content return self._tools_post(payload) def create_collection_func(element_identifiers, collection_type): payload = { "name": "dataset collection", "instance_type": "history", "history_id": history_id, "element_identifiers": element_identifiers, "collection_type": collection_type, "fields": None if collection_type != "record" else "auto", } return self._post("dataset_collections", payload) if job_path is not None: assert job is None with open(job_path) as f: job = yaml.safe_load(f) job_dir = os.path.dirname(os.path.abspath(job_path)) else: assert job is not None # Figure out what "." should be here instead. job_dir = "." if self.use_fetch_api: upload = upload_func_fetch else: upload = upload_func job_dict, datasets = galactic_job_json( job, job_dir, upload, create_collection_func, tool_or_workflow, ) return job_dict, datasets
def stage_in(ctx, runnable, config, user_gi, history_id, job_path, **kwds): files_attached = [False] def upload_func(upload_target): def _attach_file(upload_payload, uri, index=0): uri = path_or_uri_to_uri(uri) is_path = uri.startswith("file://") if not is_path or config.use_path_paste: upload_payload["inputs"]["files_%d|url_paste" % index] = uri else: files_attached[0] = True path = uri[len("file://"):] upload_payload["files_%d|file_data" % index] = attach_file(path) if isinstance(upload_target, FileUploadTarget): file_path = upload_target.path upload_payload = user_gi.tools._upload_payload( history_id, file_type=upload_target.properties.get('filetype', None) or "auto", ) name = os.path.basename(file_path) upload_payload["inputs"]["files_0|auto_decompress"] = False upload_payload["inputs"]["auto_decompress"] = False _attach_file(upload_payload, file_path) upload_payload["inputs"]["files_0|NAME"] = name if upload_target.secondary_files: _attach_file(upload_payload, upload_target.secondary_files, index=1) upload_payload["inputs"]["files_1|type"] = "upload_dataset" upload_payload["inputs"]["files_1|auto_decompress"] = True upload_payload["inputs"]["file_count"] = "2" upload_payload["inputs"]["force_composite"] = "True" ctx.vlog("upload_payload is %s" % upload_payload) return user_gi.tools._post(upload_payload, files_attached=files_attached[0]) elif isinstance(upload_target, DirectoryUploadTarget): tar_path = upload_target.tar_path upload_payload = user_gi.tools._upload_payload( history_id, file_type="tar", ) upload_payload["inputs"]["files_0|auto_decompress"] = False _attach_file(upload_payload, tar_path) tar_upload_response = user_gi.tools._post(upload_payload, files_attached=files_attached[0]) convert_response = user_gi.tools.run_tool( tool_id="CONVERTER_tar_to_directory", tool_inputs={"input1": {"src": "hda", "id": tar_upload_response["outputs"][0]["id"]}}, history_id=history_id, ) assert "outputs" in convert_response, convert_response return convert_response else: content = json.dumps(upload_target.object) return user_gi.tools.paste_content( content, history_id, file_type="expression.json", ) def create_collection_func(element_identifiers, collection_type): payload = { "name": "dataset collection", "instance_type": "history", "history_id": history_id, "element_identifiers": element_identifiers, "collection_type": collection_type, "fields": None if collection_type != "record" else "auto", } dataset_collections_url = user_gi.url + "/dataset_collections" dataset_collection = Client._post(user_gi.histories, payload, url=dataset_collections_url) return dataset_collection with open(job_path, "r") as f: job = yaml.safe_load(f) # Figure out what "." should be here instead. job_dir = os.path.dirname(job_path) job_dict, datasets = galactic_job_json( job, job_dir, upload_func, create_collection_func, tool_or_workflow="tool" if runnable.type in [RunnableType.cwl_tool, RunnableType.galaxy_tool] else "workflow", ) if datasets: final_state = _wait_for_history(ctx, user_gi, history_id) for (dataset, path) in datasets: dataset_details = user_gi.histories.show_dataset( history_id, dataset["id"], ) ctx.vlog("Uploaded dataset for path [%s] with metadata [%s]" % (path, dataset_details)) else: # Mark uploads as ok because nothing to do. final_state = "ok" ctx.vlog("final state is %s" % final_state) if final_state != "ok": msg = "Failed to run job final job state is [%s]." % final_state summarize_history(ctx, user_gi, history_id) with open("errored_galaxy.log", "w") as f: f.write(log_contents_str(config)) raise Exception(msg) return job_dict, datasets