def get(self, dataset_uuid, study_id=None, user=None):
""" Fetch a specific dataset for a given study """
args = self._get_parser.parse_args()
prop_id_to_name = get_property_map(key="id", value="name")
prop_name_to_id = reverse_map(prop_id_to_name)
# Used for helper route using only dataset_uuid
if study_id is None:
study_id = find_study_id_from_lvl1_uuid("dataset", dataset_uuid,
prop_name_to_id)
if study_id is None:
raise Exception(
f"Dataset not found in any study (uuid = {dataset_uuid})")
study = Study.objects().get(id=study_id)
study_json = marshal(study, study_model)
# The converter is used for its get_entry_by_name() method
study_converter = FormatConverter(mapper=prop_id_to_name)
study_converter.add_api_format(study_json["entries"])
datasets_entry = study_converter.get_entry_by_name("datasets")
dataset_nested_entry = datasets_entry.value.find_nested_entry(
"uuid", dataset_uuid)[0]
# The "dataset_nested_entry" entry is a NestedEntry (return list of dict)
if args["entry_format"] == "api":
return dataset_nested_entry.get_api_format()
elif args["entry_format"] == "form":
return dataset_nested_entry.get_form_format()
def test_get_entry_by_name(self):
"""Get an entry based on its prop_name"""
mapper = {"1": "user", "2": "username", "3": "phone"}
input_format = [{
"prop":
"1",
"value": [
{
"prop": "2",
"value": "Edward"
},
{
"prop": "3",
"value": "+33(0)123456789"
},
],
}]
c = FormatConverter(key_name="prop", value_name="value", mapper=mapper)
c.add_api_format(input_format)
user_entry = c.entries[0]
self.assertEqual(
user_entry.value.get_entry_by_name("username").value, "Edward")
def delete(self, dataset_uuid, study_id=None, user=None):
""" Delete a dataset from a study given its unique identifier """
prop_id_to_name = get_property_map(key="id", value="name")
prop_name_to_id = reverse_map(prop_id_to_name)
# Used for helper route using only dataset_uuid
if study_id is None:
study_id = find_study_id_from_lvl1_uuid("dataset", dataset_uuid,
prop_name_to_id)
if study_id is None:
raise Exception(
f"Dataset not found in any study (uuid = {dataset_uuid})")
# 1. Get study data
study = Study.objects().get(id=study_id)
study_json = marshal(study, study_model)
study_converter = FormatConverter(mapper=prop_id_to_name)
study_converter.add_api_format(study_json["entries"])
# 2. Delete specific dataset
datasets_entry = study_converter.get_entry_by_name("datasets")
datasets_entry.value.delete_nested_entry("uuid", dataset_uuid)
if len(datasets_entry.value.value) == 0:
study_converter.remove_entries(prop_names=["datasets"])
# 3. Update study state, data and ulpoad on DB
message = f"Deleted dataset"
update_study(study, study_converter, api.payload, message, user)
return {"message": message}
def put(self, dataset_uuid, study_id=None, user=None):
""" Update a dataset for a given study """
prop_id_to_name = get_property_map(key="id", value="name")
prop_name_to_id = reverse_map(prop_id_to_name)
# Used for helper route using only dataset_uuid
if study_id is None:
study_id = find_study_id_from_lvl1_uuid("dataset", dataset_uuid,
prop_name_to_id)
if study_id is None:
raise Exception(
f"Dataset not found in any study (uuid = {dataset_uuid})")
payload = api.payload
# 1. Split payload
form_name = payload["form_name"]
entries = payload["entries"]
entry_format = payload.get("entry_format", "api")
# 2. Get study data
study = Study.objects().get(id=study_id)
study_json = marshal(study, study_model)
study_converter = FormatConverter(mapper=prop_id_to_name)
study_converter.add_api_format(study_json["entries"])
# 3. Get current dataset data
datasets_entry = study_converter.get_entry_by_name("datasets")
dataset_nested_entry = datasets_entry.value.find_nested_entry(
"uuid", dataset_uuid)[0]
dataset_converter = FormatConverter(mapper=prop_id_to_name)
dataset_converter.entries = dataset_nested_entry.value
# 4. Get new dataset data and get entries to remove
new_dataset_converter, entries_to_remove = get_entity_converter(
entries, entry_format, prop_id_to_name, prop_name_to_id)
# 5. Update current dataset by adding, updating and deleting entries
# Nested entries not present in the original form are ignored (example: dataset["samples"])
# won't be deleted if not present in the new data), it needs to be None or "" to be deleted
dataset_converter.add_or_update_entries(new_dataset_converter.entries)
dataset_converter.remove_entries(entries=entries_to_remove)
dataset_nested_entry.value = dataset_converter.entries
# 6. Validate dataset data against form
validate_form_format_against_form(form_name,
dataset_converter.get_form_format())
# 7. Update study state, data and ulpoad on DB
message = "Updated dataset"
update_study(study, study_converter, payload, message, user)
return {"message": message}
def test_api_to_form_format_case_3(self):
"""A nested form field. Only one form field is accepted"""
mapper = {"1": "storage", "2": "location", "3": "number_of_files"}
input_format = [{
"prop":
"1",
"value": [
{
"prop": "2",
"value": "C:/Documents"
},
{
"prop": "3",
"value": 1000
},
],
}]
expected_form_format = {
"storage": {
"location": "C:/Documents",
"number_of_files": 1000
}
}
c = FormatConverter(key_name="prop", value_name="value", mapper=mapper)
actual_form_format = c.add_api_format(input_format).get_form_format()
self.assertEqual(expected_form_format, actual_form_format)
def post(self, study_id, user=None):
""" Add a new dataset for a given study """
payload = api.payload
prop_id_to_name = get_property_map(key="id", value="name")
prop_name_to_id = reverse_map(prop_id_to_name)
# 1. Split payload
form_name = payload["form_name"]
entries = payload["entries"]
entry_format = payload.get("entry_format", "api")
# 2. Get study data
study = Study.objects().get(id=study_id)
study_json = marshal(study, study_model)
study_converter = FormatConverter(mapper=prop_id_to_name)
study_converter.add_api_format(study_json["entries"])
# 3. Create dataset entry and append it to "datasets"
# Format and clean entity
dataset_converter, _ = get_entity_converter(entries, entry_format,
prop_id_to_name,
prop_name_to_id)
# Generate UUID
dataset_converter, dataset_uuid = add_uuid_entry_if_missing(
dataset_converter, prop_name_to_id)
study_converter = add_entity_to_study_nested_list(
study_converter=study_converter,
entity_converter=dataset_converter,
prop_name_to_id=prop_name_to_id,
study_list_prop="datasets",
)
# 4. Validate dataset data against form
validate_form_format_against_form(form_name,
dataset_converter.get_form_format())
# 5. Update study state, data and ulpoad on DB
message = "Added dataset"
update_study(study, study_converter, payload, message, user)
return {"message": message, "uuid": dataset_uuid}, 201
def delete(self, study_id, user=None):
""" Delete all samples from a study given its unique identifier """
prop_id_to_name = get_property_map(key="id", value="name")
# 1. Get study data
study = Study.objects().get(id=study_id)
study_json = marshal(study, study_model)
study_converter = FormatConverter(mapper=prop_id_to_name)
study_converter.add_api_format(study_json["entries"])
# 2. Delete samples
study_converter.remove_entries(prop_names=["samples"])
# 3. Update study state, data and ulpoad on DB
message = "Deleted samples"
update_study(study, study_converter, api.payload, message, user)
return {"message": message}
def test_api_to_api_format_case_2(self):
"""List of simple values"""
mapper = {"1": "username"}
input_format = [{"prop": "1", "value": ["Edward", "Annie"]}]
c = FormatConverter(key_name="prop", value_name="value", mapper=mapper)
reconstructed_api_format = c.add_api_format(
input_format).get_api_format()
self.assertEqual(input_format, reconstructed_api_format)
def test_api_to_form_format_case_1(self):
"""Simple value"""
mapper = {"1": "username"}
input_format = [{"prop": "1", "value": "Edward"}]
expected_form_format = {"username": "******"}
c = FormatConverter(key_name="prop", value_name="value", mapper=mapper)
actual_form_format = c.add_api_format(input_format).get_form_format()
self.assertEqual(expected_form_format, actual_form_format)
def test_find_nested_entry(self):
"""Get a specific NestedEntry based on the value of a property"""
mapper = {"1": "users", "2": "username", "3": "phone"}
input_format = [{
"prop":
"1",
"value": [
[
{
"prop": "2",
"value": "Edward"
},
{
"prop": "3",
"value": "+33(0)123456789"
},
],
[
{
"prop": "2",
"value": "John"
},
{
"prop": "3",
"value": "+33(9)876543210"
},
],
],
}]
c = FormatConverter(key_name="prop", value_name="value", mapper=mapper)
c.add_api_format(input_format)
users_entry = c.entries[0]
nested_entry, position = users_entry.value.find_nested_entry(
"username", "John")
self.assertEqual(nested_entry.get_form_format()["phone"],
"+33(9)876543210")
self.assertEqual(position, 1)
def upload_study_related_entity(data, url, method, property_url, headers):
"""Send data to the API (study related entity) in form format"""
entry_format = data.pop("entry_format", "form")
# Format data (cleaning + conversion from "form format" to "api format")
if entry_format == "form":
prop_name_to_id = map_key_value(property_url, key="name", value="id")
converter = FormatConverter(mapper=prop_name_to_id)
converter.add_form_format(data["entries"])
elif entry_format == "api":
prop_id_to_name = map_key_value(property_url, key="id", value="name")
converter = FormatConverter(mapper=prop_id_to_name)
converter.add_api_format(data["entries"])
converter.clean_data()
data["entries"] = converter.get_api_format()
data["entry_format"] = "api"
# Send data to API
if method == "post":
res = requests.post(url=url, json=data, headers=headers)
if res.status_code != 201:
message = f"Failed to POST study related entity. {res.json()}"
success = False
else:
message = "Succeded to POST study related entity"
success = True
elif method == "put":
res = requests.put(url=url, json=data, headers=headers)
if res.status_code != 200:
message = f"Failed to PUT study related entity. {res.json()}"
success = False
else:
message = "Succeded to PUT study related entity"
success = True
return (res.json(), message, success)
def get(self, study_id, user=None):
""" Fetch a list of all datasets for a given study """
args = self._get_parser.parse_args()
study = Study.objects().get(id=study_id)
study_json = marshal(study, study_model)
prop_map = get_property_map(key="id", value="name")
# The converter is used for its get_entry_by_name() method
study_converter = FormatConverter(mapper=prop_map)
study_converter.add_api_format(study_json["entries"])
datasets_entry = study_converter.get_entry_by_name("datasets")
if datasets_entry is not None:
# The "datasets" entry is a NestedListEntry (return list of list)
if args["entry_format"] == "api":
return datasets_entry.get_api_format()
elif args["entry_format"] == "form":
return datasets_entry.get_form_format()
else:
return []
def test_api_to_form_format_case_4(self):
"""List of nested form fields. Each form field is incorporated into a list"""
mapper = {"1": "contacts", "3": "name", "4": "phone"}
input_format = [{
"prop":
"1",
"value": [
[
{
"prop": "3",
"value": "Edward"
},
{
"prop": "4",
"value": "903 367 2072"
},
],
[
{
"prop": "3",
"value": "Annie"
},
{
"prop": "4",
"value": "731 222 8842"
},
],
],
}]
expected_form_format = {
"contacts": [
{
"name": "Edward",
"phone": "903 367 2072"
},
{
"name": "Annie",
"phone": "731 222 8842"
},
]
}
c = FormatConverter(key_name="prop", value_name="value", mapper=mapper)
actual_form_format = c.add_api_format(input_format).get_form_format()
self.assertEqual(expected_form_format, actual_form_format)
def post(self, dataset_uuid, study_id=None, user=None):
""" Add a new processing event for a given dataset """
prop_id_to_name = get_property_map(key="id", value="name")
prop_name_to_id = reverse_map(prop_id_to_name)
# Used for helper route using only dataset_uuid
if study_id is None:
study_id = find_study_id_from_lvl1_uuid("dataset", dataset_uuid,
prop_name_to_id)
if study_id is None:
raise Exception(
f"Dataset not found in any study (uuid = {dataset_uuid})")
payload = api.payload
# 1. Split payload
form_name = payload["form_name"]
entries = payload["entries"]
entry_format = payload.get("entry_format", "api")
# 2. Get study and dataset data
study = Study.objects().get(id=study_id)
study_json = marshal(study, study_model)
study_converter = FormatConverter(mapper=prop_id_to_name)
study_converter.add_api_format(study_json["entries"])
datasets_entry = study_converter.get_entry_by_name("datasets")
dataset_nested_entry, dataset_position = datasets_entry.value.find_nested_entry(
"uuid", dataset_uuid)
# 3. Format and clean processing event data
pe_converter, _ = get_entity_converter(entries, entry_format,
prop_id_to_name,
prop_name_to_id)
# 4. Generate UUID
pe_converter, pe_uuid = add_uuid_entry_if_missing(pe_converter,
prop_name_to_id,
replace=False)
pe_nested_entry = NestedEntry(pe_converter)
pe_nested_entry.value = pe_converter.entries
# 5. Check if "process_events"" entry already exist study, creates it if it doesn't
pes_entry = dataset_nested_entry.get_entry_by_name("process_events")
if pes_entry is not None:
pes_entry.value.value.append(pe_nested_entry)
else:
pes_entry = Entry(
FormatConverter(prop_name_to_id)).add_form_format(
"process_events", [pe_nested_entry.get_form_format()])
dataset_nested_entry.value.append(pes_entry)
datasets_entry.value.value[dataset_position] = dataset_nested_entry
# 6. Validate processing data against form
validate_form_format_against_form(form_name,
pe_converter.get_form_format())
# 7. Update study state, data and ulpoad on DB
message = "Added processing event"
update_study(study, study_converter, payload, message, user)
return {"message": message, "uuid": pe_uuid}, 201
def put(self, sample_uuid, study_id=None, user=None):
""" Update a sample for a given study """
prop_id_to_name = get_property_map(key="id", value="name")
prop_name_to_id = reverse_map(prop_id_to_name)
# Used for helper route using only sample_uuid
if study_id is None:
study_id = find_study_id_from_lvl1_uuid("sample", sample_uuid,
prop_name_to_id)
if study_id is None:
raise Exception(
f"Sample not found in any study (uuid = {sample_uuid})")
payload = api.payload
# 1. Split payload
validate_dict = payload.get("validate", None)
form_names = payload.get("form_names", None)
entries = payload["entries"]
entry_format = payload.get("entry_format", "api")
# 2. Get forms for validation
forms = {}
for key, validate in validate_dict.items():
if validate:
forms[key] = app.form_manager.get_form_by_name(
form_name=form_names[key])
# 3. Get study data
study = Study.objects().get(id=study_id)
study_json = marshal(study, study_model)
study_converter = FormatConverter(mapper=prop_id_to_name)
study_converter.add_api_format(study_json["entries"])
# 3. Get current sample data
samples_entry = study_converter.get_entry_by_name("samples")
sample_nested_entry = samples_entry.value.find_nested_entry(
"uuid", sample_uuid)[0]
sample_converter = FormatConverter(mapper=prop_id_to_name)
sample_converter.entries = sample_nested_entry.value
# 4. Unify UUIDs with existing entities (including nested ones)
# Format and clean entity
new_sample_converter, _ = get_entity_converter(entries, entry_format,
prop_id_to_name,
prop_name_to_id)
new_sample_form_format = new_sample_converter.get_form_format()
[new_sample_form_format] = unify_sample_entities_uuids(
existing_samples=study_converter.get_form_format().get(
"samples", []),
new_samples=[new_sample_form_format],
)
# 5. Clean new data and get entries to remove
# Format and clean entity
new_sample_converter, entries_to_remove = get_entity_converter(
entries=new_sample_form_format,
entry_format="form",
prop_id_to_name=None,
prop_name_to_id=prop_name_to_id,
)
# 6. Update current sample by adding, updating and deleting entries
# Nested entries not present in the original form are ignored
# won't be deleted if not present in the new data), it needs to be None or "" to be deleted
sample_converter.add_or_update_entries(new_sample_converter.entries)
sample_converter.remove_entries(entries=entries_to_remove)
sample_nested_entry.value = sample_converter.entries
# 7. Validate data against form
validate_sample_against_form(sample_converter.get_form_format(),
validate_dict, forms)
# 8. Update study state, data and ulpoad on DB
message = "Updated sample"
update_study(study, study_converter, payload, message, user)
return {"message": message}
def post(self, study_id, user=None):
""" Add a new sample for a given study """
payload = api.payload
prop_id_to_name = get_property_map(key="id", value="name")
prop_name_to_id = reverse_map(prop_id_to_name)
# 1. Split payload
validate_dict = payload.get("validate", None)
form_names = payload.get("form_names", None)
entries = payload["entries"]
entry_format = payload.get("entry_format", "api")
# 2. Get forms for validation
forms = {}
for key, validate in validate_dict.items():
if validate:
forms[key] = app.form_manager.get_form_by_name(
form_name=form_names[key])
# 3. Get study data
study = Study.objects().get(id=study_id)
study_json = marshal(study, study_model)
study_converter = FormatConverter(mapper=prop_id_to_name)
study_converter.add_api_format(study_json["entries"])
# 4. Unify UUIDs with existing entities (including nested ones)
# Format and clean entity
sample_converter, _ = get_entity_converter(entries, entry_format,
prop_id_to_name,
prop_name_to_id)
new_sample_form_format = sample_converter.get_form_format()
[new_sample_form_format] = unify_sample_entities_uuids(
existing_samples=study_converter.get_form_format().get(
"samples", []),
new_samples=[new_sample_form_format],
)
# 5. Append new samples to "samples" in study
# Format and clean entity
sample_converter, _ = get_entity_converter(
entries=new_sample_form_format,
entry_format="form",
prop_id_to_name=None,
prop_name_to_id=prop_name_to_id,
)
# Generate UUID (redundant, UUIDs already generated by unify_sample_entities_uuids)
sample_converter, sample_uuid = add_uuid_entry_if_missing(
sample_converter, prop_name_to_id)
study_converter = add_entity_to_study_nested_list(
study_converter=study_converter,
entity_converter=sample_converter,
prop_name_to_id=prop_name_to_id,
study_list_prop="samples",
)
# 6. Validate data against form
validate_sample_against_form(sample_converter.get_form_format(),
validate_dict, forms)
# 7. Update study state, data and ulpoad on DB
message = "Added sample"
update_study(study, study_converter, payload, message, user)
return {"message": message, "uuid": sample_uuid}, 201