def stage_in(ctx, runnable, config, user_gi, history_id, job_path, **kwds):
def upload_func(upload_target):
if isinstance(upload_target, FileUploadTarget):
file_path = upload_target.path
upload_payload = user_gi.tools._upload_payload(
history_id,
file_type="auto",
)
name = os.path.basename(file_path)
upload_payload["inputs"]["files_0|auto_decompress"] = False
upload_payload["inputs"]["auto_decompress"] = False
upload_payload["inputs"]["files_0|url_paste"] = "file://%s" % os.path.abspath(file_path)
upload_payload["inputs"]["files_0|NAME"] = name
if upload_target.secondary_files:
upload_payload["inputs"]["files_1|url_paste"] = "file://%s" % os.path.abspath(upload_target.secondary_files)
upload_payload["inputs"]["files_1|type"] = "upload_dataset"
upload_payload["inputs"]["files_1|auto_decompress"] = True
upload_payload["inputs"]["file_count"] = "2"
upload_payload["inputs"]["force_composite"] = "True"
ctx.vlog("upload_payload is %s" % upload_payload)
return user_gi.tools._tool_post(upload_payload, files_attached=False)
elif isinstance(upload_target, DirectoryUploadTarget):
tar_path = upload_target.tar_path
upload_payload = user_gi.tools._upload_payload(
history_id,
file_type="tar",
)
upload_payload["inputs"]["files_0|auto_decompress"] = False
upload_payload["inputs"]["files_0|url_paste"] = "file://%s" % tar_path
tar_upload_response = user_gi.tools._tool_post(upload_payload, files_attached=False)
convert_response = user_gi.tools.run_tool(
tool_id="CONVERTER_tar_to_directory",
tool_inputs={"input1": {"src": "hda", "id": tar_upload_response["outputs"][0]["id"]}},
history_id=history_id,
)
assert "outputs" in convert_response, convert_response
return convert_response
else:
content = json.dumps(upload_target.object)
return user_gi.tools.paste_content(
content,
history_id,
file_type="expression.json",
)
def create_collection_func(element_identifiers, collection_type):
payload = {
"name": "dataset collection",
"instance_type": "history",
"history_id": history_id,
"element_identifiers": element_identifiers,
"collection_type": collection_type,
"fields": None if collection_type != "record" else "auto",
}
dataset_collections_url = user_gi.url + "/dataset_collections"
dataset_collection = Client._post(user_gi.histories, payload, url=dataset_collections_url)
return dataset_collection
with open(job_path, "r") as f:
job = yaml.load(f)
# Figure out what "." should be here instead.
job_dir = os.path.dirname(job_path)
job_dict, datasets = galactic_job_json(
job,
job_dir,
upload_func,
create_collection_func,
tool_or_workflow="tool" if runnable.type in [RunnableType.cwl_tool, RunnableType.galaxy_tool] else "workflow",
)
if datasets:
final_state = _wait_for_history(ctx, user_gi, history_id)
for (dataset, path) in datasets:
dataset_details = user_gi.histories.show_dataset(
history_id,
dataset["id"],
)
ctx.vlog("Uploaded dataset for path [%s] with metadata [%s]" % (path, dataset_details))
else:
# Mark uploads as ok because nothing to do.
final_state = "ok"
ctx.vlog("final state is %s" % final_state)
if final_state != "ok":
msg = "Failed to run job final job state is [%s]." % final_state
summarize_history(ctx, user_gi, history_id)
with open("errored_galaxy.log", "w") as f:
f.write(config.log_contents)
raise Exception(msg)
galaxy_paths = []
for (dataset, upload_target) in datasets:
if isinstance(upload_target, FileUploadTarget):
local_path = upload_target.path
ctx.vlog("fetching full dataset for %s, %s" % (dataset, local_path))
dataset_full = user_gi.datasets.show_dataset(dataset["id"])
galaxy_path = dataset_full["file_name"]
ctx.vlog("galaxy_path is %s" % galaxy_path)
job_path = os.path.join(job_dir, local_path)
galaxy_paths.append((job_path, galaxy_path))
ctx.vlog("galaxy_paths are %s" % galaxy_paths)
return galaxy_paths, job_dict, datasets
def stage_in(ctx, runnable, config, user_gi, history_id, job_path, **kwds):
files_attached = [False]
def upload_func(upload_target):
def _attach_file(upload_payload, uri, index=0):
uri = path_or_uri_to_uri(uri)
is_path = uri.startswith("file://")
if not is_path or config.use_path_paste:
upload_payload["inputs"]["files_%d|url_paste" % index] = uri
else:
files_attached[0] = True
path = uri[len("file://"):]
upload_payload["files_%d|file_data" % index] = attach_file(path)
if isinstance(upload_target, FileUploadTarget):
file_path = upload_target.path
upload_payload = user_gi.tools._upload_payload(
history_id,
file_type=upload_target.properties.get('filetype', None) or "auto",
)
name = os.path.basename(file_path)
upload_payload["inputs"]["files_0|auto_decompress"] = False
upload_payload["inputs"]["auto_decompress"] = False
_attach_file(upload_payload, file_path)
upload_payload["inputs"]["files_0|NAME"] = name
if upload_target.secondary_files:
_attach_file(upload_payload, upload_target.secondary_files, index=1)
upload_payload["inputs"]["files_1|type"] = "upload_dataset"
upload_payload["inputs"]["files_1|auto_decompress"] = True
upload_payload["inputs"]["file_count"] = "2"
upload_payload["inputs"]["force_composite"] = "True"
ctx.vlog("upload_payload is %s" % upload_payload)
return user_gi.tools._tool_post(upload_payload, files_attached=files_attached[0])
elif isinstance(upload_target, DirectoryUploadTarget):
tar_path = upload_target.tar_path
upload_payload = user_gi.tools._upload_payload(
history_id,
file_type="tar",
)
upload_payload["inputs"]["files_0|auto_decompress"] = False
_attach_file(upload_payload, tar_path)
tar_upload_response = user_gi.tools._tool_post(upload_payload, files_attached=files_attached[0])
convert_response = user_gi.tools.run_tool(
tool_id="CONVERTER_tar_to_directory",
tool_inputs={"input1": {"src": "hda", "id": tar_upload_response["outputs"][0]["id"]}},
history_id=history_id,
)
assert "outputs" in convert_response, convert_response
return convert_response
else:
content = json.dumps(upload_target.object)
return user_gi.tools.paste_content(
content,
history_id,
file_type="expression.json",
)
def create_collection_func(element_identifiers, collection_type):
payload = {
"name": "dataset collection",
"instance_type": "history",
"history_id": history_id,
"element_identifiers": element_identifiers,
"collection_type": collection_type,
"fields": None if collection_type != "record" else "auto",
}
dataset_collections_url = user_gi.url + "/dataset_collections"
dataset_collection = Client._post(user_gi.histories, payload, url=dataset_collections_url)
return dataset_collection
with open(job_path, "r") as f:
job = yaml.safe_load(f)
# Figure out what "." should be here instead.
job_dir = os.path.dirname(job_path)
job_dict, datasets = galactic_job_json(
job,
job_dir,
upload_func,
create_collection_func,
tool_or_workflow="tool" if runnable.type in [RunnableType.cwl_tool, RunnableType.galaxy_tool] else "workflow",
)
if datasets:
final_state = _wait_for_history(ctx, user_gi, history_id)
for (dataset, path) in datasets:
dataset_details = user_gi.histories.show_dataset(
history_id,
dataset["id"],
)
ctx.vlog("Uploaded dataset for path [%s] with metadata [%s]" % (path, dataset_details))
else:
# Mark uploads as ok because nothing to do.
final_state = "ok"
ctx.vlog("final state is %s" % final_state)
if final_state != "ok":
msg = "Failed to run job final job state is [%s]." % final_state
summarize_history(ctx, user_gi, history_id)
with open("errored_galaxy.log", "w") as f:
f.write(log_contents_str(config))
raise Exception(msg)
return job_dict, datasets