def stage(self,
tool_or_workflow,
history_id,
job=None,
job_path=None,
use_path_paste=LOAD_TOOLS_FROM_PATH,
to_posix_lines=True):
files_attached = [False]
def upload_func_fetch(upload_target):
def _attach_file(upload_payload, uri, index=0):
uri = path_or_uri_to_uri(uri)
is_path = uri.startswith("file://")
if not is_path or use_path_paste:
return {"src": "url", "url": uri}
else:
files_attached[0] = True
path = uri[len("file://"):]
upload_payload["__files"]["files_%s|file_data" %
index] = self._attach_file(path)
return {"src": "files"}
fetch_payload = None
if isinstance(upload_target, FileUploadTarget):
file_path = upload_target.path
file_type = upload_target.properties.get(
'filetype', None) or DEFAULT_FILE_TYPE
dbkey = upload_target.properties.get('dbkey',
None) or DEFAULT_DBKEY
fetch_payload = _fetch_payload(
history_id,
file_type=file_type,
dbkey=dbkey,
to_posix_lines=to_posix_lines,
)
name = _file_path_to_name(file_path)
if file_path is not None:
src = _attach_file(fetch_payload, file_path)
fetch_payload["targets"][0]["elements"][0].update(src)
if upload_target.composite_data:
composite_items = []
for i, composite_data in enumerate(
upload_target.composite_data):
composite_item_src = _attach_file(fetch_payload,
composite_data,
index=i)
composite_items.append(composite_item_src)
fetch_payload["targets"][0]["elements"][0][
"src"] = "composite"
fetch_payload["targets"][0]["elements"][0]["composite"] = {
"items": composite_items,
}
tags = upload_target.properties.get("tags")
if tags:
fetch_payload["targets"][0]["elements"][0]["tags"] = tags
fetch_payload["targets"][0]["elements"][0]["name"] = name
elif isinstance(upload_target, FileLiteralTarget):
fetch_payload = _fetch_payload(history_id)
# For file literals - take them as is - never convert line endings.
fetch_payload["targets"][0]["elements"][0].update({
"src":
"pasted",
"paste_content":
upload_target.contents,
"to_posix_lines":
False,
})
tags = upload_target.properties.get("tags")
if tags:
fetch_payload["targets"][0]["elements"][0]["tags"] = tags
elif isinstance(upload_target, DirectoryUploadTarget):
fetch_payload = _fetch_payload(history_id,
file_type="directory")
fetch_payload["targets"][0].pop("elements")
tar_path = upload_target.path
src = _attach_file(fetch_payload, tar_path)
fetch_payload["targets"][0]["elements_from"] = src
else:
content = json.dumps(upload_target.object)
fetch_payload = _fetch_payload(history_id,
file_type="expression.json")
fetch_payload["targets"][0]["elements"][0].update({
"src":
"pasted",
"paste_content":
content,
})
tags = upload_target.properties.get("tags")
fetch_payload["targets"][0]["elements"][0]["tags"] = tags
return self._fetch_post(fetch_payload,
files_attached=files_attached[0])
# Save legacy upload_func to target older Galaxy servers
def upload_func(upload_target):
def _attach_file(upload_payload, uri, index=0):
uri = path_or_uri_to_uri(uri)
is_path = uri.startswith("file://")
if not is_path or use_path_paste:
upload_payload["inputs"]["files_%d|url_paste" %
index] = uri
else:
files_attached[0] = True
path = uri[len("file://"):]
upload_payload["__files"]["files_%d|file_data" %
index] = self._attach_file(path)
if isinstance(upload_target, FileUploadTarget):
file_path = upload_target.path
file_type = upload_target.properties.get(
'filetype', None) or DEFAULT_FILE_TYPE
dbkey = upload_target.properties.get('dbkey',
None) or DEFAULT_DBKEY
upload_payload = _upload_payload(
history_id,
file_type=file_type,
to_posix_lines=dbkey,
)
name = _file_path_to_name(file_path)
upload_payload["inputs"]["files_0|auto_decompress"] = False
upload_payload["inputs"]["auto_decompress"] = False
if file_path is not None:
_attach_file(upload_payload, file_path)
upload_payload["inputs"]["files_0|NAME"] = name
if upload_target.secondary_files:
_attach_file(upload_payload,
upload_target.secondary_files,
index=1)
upload_payload["inputs"]["files_1|type"] = "upload_dataset"
upload_payload["inputs"]["files_1|auto_decompress"] = True
upload_payload["inputs"]["file_count"] = "2"
upload_payload["inputs"]["force_composite"] = "True"
# galaxy.exceptions.RequestParameterInvalidException: Not input source type
# defined for input '{'class': 'File', 'filetype': 'imzml', 'composite_data':
# ['Example_Continuous.imzML', 'Example_Continuous.ibd']}'.\n"}]]
if upload_target.composite_data:
for i, composite_data in enumerate(
upload_target.composite_data):
upload_payload["inputs"]["files_%s|type" %
i] = "upload_dataset"
_attach_file(upload_payload, composite_data, index=i)
self._log("upload_payload is %s" % upload_payload)
return self._tools_post(upload_payload,
files_attached=files_attached[0])
elif isinstance(upload_target, FileLiteralTarget):
# For file literals - take them as is - never convert line endings.
payload = _upload_payload(history_id,
file_type="auto",
auto_decompress=False,
to_posix_lines=False)
payload["inputs"]["files_0|url_paste"] = upload_target.contents
return self._tools_post(payload)
elif isinstance(upload_target, DirectoryUploadTarget):
tar_path = upload_target.tar_path
upload_payload = _upload_payload(
history_id,
file_type="tar",
)
upload_payload["inputs"]["files_0|auto_decompress"] = False
_attach_file(upload_payload, tar_path)
tar_upload_response = self._tools_post(
upload_payload, files_attached=files_attached[0])
convert_payload = dict(
tool_id="CONVERTER_tar_to_directory",
tool_inputs={
"input1": {
"src": "hda",
"id": tar_upload_response["outputs"][0]["id"]
}
},
history_id=history_id,
)
convert_response = self._tools_post(convert_payload)
assert "outputs" in convert_response, convert_response
return convert_response
else:
content = json.dumps(upload_target.object)
payload = _upload_payload(history_id,
file_type="expression.json")
payload["files_0|url_paste"] = content
return self._tools_post(payload)
def create_collection_func(element_identifiers, collection_type):
payload = {
"name": "dataset collection",
"instance_type": "history",
"history_id": history_id,
"element_identifiers": element_identifiers,
"collection_type": collection_type,
"fields": None if collection_type != "record" else "auto",
}
return self._post("dataset_collections", payload)
if job_path is not None:
assert job is None
with open(job_path) as f:
job = yaml.safe_load(f)
job_dir = os.path.dirname(os.path.abspath(job_path))
else:
assert job is not None
# Figure out what "." should be here instead.
job_dir = "."
if self.use_fetch_api:
upload = upload_func_fetch
else:
upload = upload_func
job_dict, datasets = galactic_job_json(
job,
job_dir,
upload,
create_collection_func,
tool_or_workflow,
)
return job_dict, datasets
def stage_in(ctx, runnable, config, user_gi, history_id, job_path, **kwds):
files_attached = [False]
def upload_func(upload_target):
def _attach_file(upload_payload, uri, index=0):
uri = path_or_uri_to_uri(uri)
is_path = uri.startswith("file://")
if not is_path or config.use_path_paste:
upload_payload["inputs"]["files_%d|url_paste" % index] = uri
else:
files_attached[0] = True
path = uri[len("file://"):]
upload_payload["files_%d|file_data" % index] = attach_file(path)
if isinstance(upload_target, FileUploadTarget):
file_path = upload_target.path
upload_payload = user_gi.tools._upload_payload(
history_id,
file_type=upload_target.properties.get('filetype', None) or "auto",
)
name = os.path.basename(file_path)
upload_payload["inputs"]["files_0|auto_decompress"] = False
upload_payload["inputs"]["auto_decompress"] = False
_attach_file(upload_payload, file_path)
upload_payload["inputs"]["files_0|NAME"] = name
if upload_target.secondary_files:
_attach_file(upload_payload, upload_target.secondary_files, index=1)
upload_payload["inputs"]["files_1|type"] = "upload_dataset"
upload_payload["inputs"]["files_1|auto_decompress"] = True
upload_payload["inputs"]["file_count"] = "2"
upload_payload["inputs"]["force_composite"] = "True"
ctx.vlog("upload_payload is %s" % upload_payload)
return user_gi.tools._post(upload_payload, files_attached=files_attached[0])
elif isinstance(upload_target, DirectoryUploadTarget):
tar_path = upload_target.tar_path
upload_payload = user_gi.tools._upload_payload(
history_id,
file_type="tar",
)
upload_payload["inputs"]["files_0|auto_decompress"] = False
_attach_file(upload_payload, tar_path)
tar_upload_response = user_gi.tools._post(upload_payload, files_attached=files_attached[0])
convert_response = user_gi.tools.run_tool(
tool_id="CONVERTER_tar_to_directory",
tool_inputs={"input1": {"src": "hda", "id": tar_upload_response["outputs"][0]["id"]}},
history_id=history_id,
)
assert "outputs" in convert_response, convert_response
return convert_response
else:
content = json.dumps(upload_target.object)
return user_gi.tools.paste_content(
content,
history_id,
file_type="expression.json",
)
def create_collection_func(element_identifiers, collection_type):
payload = {
"name": "dataset collection",
"instance_type": "history",
"history_id": history_id,
"element_identifiers": element_identifiers,
"collection_type": collection_type,
"fields": None if collection_type != "record" else "auto",
}
dataset_collections_url = user_gi.url + "/dataset_collections"
dataset_collection = Client._post(user_gi.histories, payload, url=dataset_collections_url)
return dataset_collection
with open(job_path, "r") as f:
job = yaml.safe_load(f)
# Figure out what "." should be here instead.
job_dir = os.path.dirname(job_path)
job_dict, datasets = galactic_job_json(
job,
job_dir,
upload_func,
create_collection_func,
tool_or_workflow="tool" if runnable.type in [RunnableType.cwl_tool, RunnableType.galaxy_tool] else "workflow",
)
if datasets:
final_state = _wait_for_history(ctx, user_gi, history_id)
for (dataset, path) in datasets:
dataset_details = user_gi.histories.show_dataset(
history_id,
dataset["id"],
)
ctx.vlog("Uploaded dataset for path [%s] with metadata [%s]" % (path, dataset_details))
else:
# Mark uploads as ok because nothing to do.
final_state = "ok"
ctx.vlog("final state is %s" % final_state)
if final_state != "ok":
msg = "Failed to run job final job state is [%s]." % final_state
summarize_history(ctx, user_gi, history_id)
with open("errored_galaxy.log", "w") as f:
f.write(log_contents_str(config))
raise Exception(msg)
return job_dict, datasets