def generate_base_submission():
file_name = "base_submission.json"
path = os.path.join(TEST_SUBMISSIONS_DIR, file_name)
mdfcc = MDFConnectClient() # service_instance is irrelevant
mdfcc.create_dc_block(title="Base Deploy Testing Dataset",
authors="jgaff",
affiliations="UChicago")
mdfcc.add_data_source(DATA_SOURCES)
mdfcc.set_test(True)
mdfcc.update = UPDATE_FLAG
submission = mdfcc.get_submission()
with open(path, 'w') as f:
json.dump(submission, f)
return {"success": True}
def generate_base_failure():
file_name = "base_failure.json"
path = os.path.join(TEST_SUBMISSIONS_DIR, file_name)
mdfcc = MDFConnectClient() # service_instance is irrelevant
mdfcc.create_dc_block(title="Base Deploy Fail Dataset",
authors="jgaff",
affiliations="UChicago")
mdfcc.add_data_source(DATA_SOURCES)
# Failure point: Submission not in nanomfg organization,
# but attempting to write to nanomfg project block
mdfcc.set_project_block("nanomfg", {"catalyst": "bad data testing"})
mdfcc.set_test(True)
mdfcc.update = UPDATE_FLAG
submission = mdfcc.get_submission()
with open(path, 'w') as f:
json.dump(submission, f)
return {"success": True}
download_folder(client, download_path, sample_folder)
with open(os.path.join(download_path, "recipe.json"), 'w') as outfile:
json.dump(s, outfile, cls=GresqEncoder)
zip_path = os.path.join("..", "output", s.sample_id + ".zip")
zipf = zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED)
zipdir(download_path, zipf)
zipf.close()
box_file = staging_folder.upload(zip_path, s.sample_id + ".zip")
experiment_date = datetime.strptime(s.experiment_date, "%Y-%m-%d")
mdfcc.create_dc_block(title="Graphene Synthesis Sample " +
s.identifier,
authors=[s.contributor],
affiliations=[s.group],
publication_year=experiment_date.year)
mdfcc.add_data(box_file.get_shared_link_download_url(access='open'))
# Don't publish specific recipes. Later on, we will bundle datasets and
# and publish an omnibus dataset
# mdfcc.add_service("globus_publish")
mdfcc.set_source_name(s.sample_id)
submission = mdfcc.get_submission()
submission["projects"] = {}
submission["projects"]["nanomfg"] = {
"catalyst": s.catalyst,
"max_temperature": max_temp(s),
class MDFAdaptor:
def __init__(self):
self.mdfcc = MDFConnectClient(test=True, service_instance="prod")
def upload_recipe(self, recipe, box_file):
experiment_date = datetime.now()
self.mdfcc.create_dc_block(
title="Graphene Synthesis Sample " + "TBD",
authors=[
"%s, %s" % (auth["last_name"], auth["first_name"])
for auth in recipe.authors
],
affiliations=[auth["institution"] for auth in recipe.authors],
publication_year=recipe.experiment_year,
)
self.mdfcc.add_data_source(
box_file.get_shared_link_download_url(access="open"))
# Don't publish specific recipes. Later on, we will bundle datasets and
# and publish an omnibus dataset
# mdfcc.add_service("globus_publish")
self.mdfcc.set_source_name(str(uuid.uuid4()))
submission = self.mdfcc.get_submission()
submission["projects"] = {}
submission["projects"]["nanomfg"] = {
"catalyst": recipe.catalyst,
"max_temperature": recipe.max_temp(),
"carbon_source": recipe.carbon_source(),
"base_pressure": recipe.base_pressure
# "sample_surface_area": recipe.sample_surface_area,
# "sample_thickness": recipe.thickness,
# "orientation": "",
# "grain_size": ""
}
print("\n\n\n\n------>", submission)
try:
mdf_result = self.mdfcc.submit_dataset(submission=submission)
except Exception as e:
print("Exception submitting dataset to mdf ", str(e))
raise MDFException(e)
if not mdf_result["success"]:
self.mdfcc.reset_submission()
print("\n\n\n--->Error-----> " + mdf_result["error"])
raise MDFException(mdf_result["error"])
print("Submitted to MDF -----> " + str(mdf_result))
self.mdfcc.reset_submission()
return mdf_result["source_id"]
def upload_raman_analysis(self, recipe, recipe_dataset_id, raman_set,
raman_box_file):
self.mdfcc.create_dc_block(
title="Graphene Synthesis Raman Analysis",
authors=[
"%s, %s" % (auth["last_name"], auth["first_name"])
for auth in recipe.authors
],
affiliations=[auth["institution"] for auth in recipe.authors],
publication_year=recipe.experiment_year,
related_dois=recipe_dataset_id,
)
self.mdfcc.add_data_source(
raman_box_file.get_shared_link_download_url(access="open"))
self.mdfcc.set_source_name(str(uuid.uuid4()))
submission = self.mdfcc.get_submission()
submission["projects"] = {}
submission["projects"]["nanomfg"] = {
"d_to_g": raman_set["d_to_g"],
"gp_to_g": raman_set["gp_to_g"],
"d_peak_shift": raman_set["d_peak_shift"],
"d_peak_amplitude": raman_set["d_peak_amplitude"],
"d_fwhm": raman_set["d_fwhm"],
"g_peak_shift": raman_set["g_peak_shift"],
"g_peak_amplitude": raman_set["g_peak_amplitude"],
"g_fwhm": raman_set["g_fwhm"],
"g_prime_peak_shift": raman_set["g_prime_peak_shift"],
"g_prime_peak_amplitude": raman_set["g_prime_peak_amplitude"],
"g_prime_fwhm": raman_set["g_prime_fwhm"],
}
print("\n\n\n\n------>", submission)
try:
mdf_result = self.mdfcc.submit_dataset(submission=submission)
except Exception as e:
print("Exception submitting raman analysis dataset to mdf ",
str(e))
raise MDFException(e)
if not mdf_result["success"]:
self.mdfcc.reset_submission()
print("\n\n\n--->Error-----> " + mdf_result["error"])
raise MDFException(mdf_result["error"])
print("Submitted raman analysis to MDF -----> " + str(mdf_result))
self.mdfcc.reset_submission()
return mdf_result["source_id"]
def mdf_submit(local_json_path,
globus_uri,
test=True,
with_doi=True,
update=False):
"""Submit dataset to MDF Connect.
Arguments:
local_json_path (str): The path to the local JSON metadata file.
globus_uri (str): The URI to the Globus Endpoint and path, in the form:
"globus://[endpoint id]/[path to data directory]"
test (bool): Is this a test submission (test submissions generate test DOIs
and populate the Search index "mdf-test" instead of "mdf")?
Default True.
with_doi (bool): Should a DOI be minted? (Includes test DOI.)
Default True.
update (bool): Has this submission been made before? If so, an update will be made
to the previous submission. Test submissions and non-test submissions
are separate.
"""
import json
import os
from mdf_connect_client import MDFConnectClient
mapping = {
"custom.dynamic_mean_window_size": "xpcs.dynamic_mean_window_size",
"custom.lld": "xpcs.lld",
"custom.sigma": "xpcs.sigma",
"custom.snophi": "xpcs.snophi",
"custom.snoq": "xpcs.snoq"
}
mdfcc = MDFConnectClient()
with open(local_json_path) as f:
md = json.load(f)
# DC block (title, authors, publisher, pub year, subjects)
mdfcc.create_dc_block(
title=os.path.basename(local_json_path).replace(".json", ""),
authors=[creator["creatorName"] for creator in md["creators"]],
publisher=md.get("publisher"),
publication_year=md.get("publication_year"),
subjects=[subject["subject"] for subject in md.get("subjects", [])])
# Add data
mdfcc.add_data_source(globus_uri)
# Add JSON mapping
mdfcc.add_index("json", mapping)
# Set test flag
mdfcc.set_test(test)
# Add XPCS as organization
mdfcc.add_organization("XPCS 8-ID")
# Set group-by-dir flag
mdfcc.set_conversion_config({"group_by_dir": True})
# Add MDF Publish service
if with_doi:
mdfcc.add_service("mdf_publish")
# Submit dataset
sub_res = mdfcc.submit_dataset(update=update)
if not sub_res["success"]:
raise RuntimeError(sub_res["error"])
else:
print("Submission '{}' started".format(sub_res["source_id"]))
return "Submission '{}' started".format(sub_res["source_id"])