def load_schemas() -> dict:
"""
Load all JSON schemas into a dictionary keyed on the
schema directory. Values are dictionaries mapping entity
names to loaded and validated entity schemas.
"""
schemas = {}
for root, _, paths in os.walk(SCHEMA_DIR):
root_schemas = {}
for path in paths:
if not path.endswith(".json"):
continue
schema_path = os.path.join(root, path)
full_json = _load_dont_validate_schema(schema_path, SCHEMA_DIR)
resolver = jsonschema.RefResolver(f"file://{SCHEMA_DIR}/schemas",
full_json)
def json_to_html(ref: str) -> dict:
"""Update refs to refer to the URL of the corresponding documentation."""
url = ref.replace(".json", ".html")
url = url.replace("properties/", "")
url = url.replace("definitions/", "")
url = url.replace("/", ".")
# resolve ref, extract resolved['description']
with resolver.resolving(ref) as resolved:
if "description" in resolved:
description = resolved["description"]
else:
description = ""
return {"url": url, "description": description}
schema = _load_dont_validate_schema(schema_path,
SCHEMA_DIR,
on_refs=json_to_html)
assert path.endswith(".json")
schema_name = path[:-5].replace("/", ".")
root_schemas[schema_name] = (schema, full_json)
relative_root = root.replace(SCHEMA_DIR, "").replace("/", ".")
relative_root = relative_root.replace(".", "", 1)
schemas[relative_root] = root_schemas
return schemas
def test_special_keywords():
# load the schema
schema_root = SCHEMA_DIR
schema_path = os.path.join(SCHEMA_DIR,
"templates/metadata/cytof_template.json")
# we don't validate it because it's a template, not a schema
schema = _load_dont_validate_schema(schema_path, schema_root)
tmp1 = schema["properties"]["worksheets"]["Acquisition and Preprocessing"]
tmp2 = tmp1["data_columns"]["Preprocessing"]["processed fcs filename"]
assert "is_multi_artifact" not in tmp2
assert "is_artifact" in tmp2
def test_convert_api_to_template_rna():
rna_api = {
"cimac id": [
{ # use first entry as example
"filter_group": "alignment",
"file_path_template": "analysis/star/{id}/{id}.sorted.bam",
"short_description": "star alignment output",
"long_description": "file sorted_bam file sorted_bam file sorted_bam file",
"file_purpose": "Analysis view",
}
]
}
rna_json = {
"title": "RNAseq level 1 analysis template",
"description":
"Metadata information for RNAseq level 1 Analysis output.",
"prism_template_root_object_schema":
"assays/components/ngs/rna/rna_analysis.json",
"prism_template_root_object_pointer": "/analysis/rna_analysis",
"properties": {
"worksheets": {
"RNAseq Analysis": {
"preamble_rows": {
"protocol identifier": {
"merge_pointer":
"2/protocol_identifier",
"type_ref":
"clinical_trial.json#properties/protocol_identifier",
},
"folder": {
"do_not_merge": True,
"type": "string",
"allow_empty": True,
},
},
"prism_data_object_pointer": "/level_1/-",
"data_columns": {
"RNAseq Runs": {
"cimac id": {
"merge_pointer":
"/cimac_id",
"type_ref":
"sample.json#properties/cimac_id",
"process_as": [{
"parse_through":
"lambda id: f'{folder or \"\"}analysis/star/{id}/{id}.sorted.bam'",
"merge_pointer": "0/star/sorted_bam",
"gcs_uri_format":
"{protocol identifier}/rna/{cimac id}/analysis/star/sorted.bam",
"type_ref":
"assays/components/local_file.json#properties/file_path",
"is_artifact": 1,
}],
},
"comments": {
"type_ref":
"assays/components/ngs/rna/rna_level1_analysis.json#properties/comments",
"merge_pointer": "0/comments",
"allow_empty": True,
},
}
},
},
**_excluded_samples_worksheet_snippet,
}
},
}
assay_schema = _load_dont_validate_schema(
"assays/components/ngs/rna/rna_analysis.json")
rna_output = _convert_api_to_template("rna", rna_api, assay_schema)
assert DeepDiff(rna_json, rna_output) == {}
rna_api_bad_key = {"foo": [{}]}
with pytest.raises(InvalidMergeTargetException,
match="corresponding entry"):
_convert_api_to_template("rna", rna_api_bad_key, assay_schema)
rna_api_no_target = {
"cimac id": [{
"filter_group": "alignment",
"file_path_template": "foo", # used to generate merge_pointer
"short_description": "star alignment output",
"long_description":
"file sorted_bam file sorted_bam file sorted_bam file",
"file_purpose": "Analysis view",
}]
}
with pytest.raises(InvalidMergeTargetException, match="cannot be mapped"):
_convert_api_to_template("rna", rna_api_no_target, assay_schema)
rna_api_merge_collision = {
"cimac id": [
{
"filter_group": "alignment",
"file_path_template": "analysis/star/{id}/{id}.sorted.bam",
"short_description": "star alignment output",
"long_description": "file sorted_bam file sorted_bam file sorted_bam file",
"file_purpose": "Analysis view",
},
{ # direct repeat will collide
"filter_group": "alignment",
"file_path_template": "analysis/star/{id}/{id}.sorted.bam",
"short_description": "star alignment output",
"long_description": "file sorted_bam file sorted_bam file sorted_bam file",
"file_purpose": "Analysis view",
},
]
}
with pytest.raises(InvalidMergeTargetException,
match="collision for inferred merge target"):
_convert_api_to_template("rna", rna_api_merge_collision, assay_schema)
def test_convert_api_to_template_wes():
wes_api = {
"run id": [{
"filter_group": "clonality/clonality_pyclone",
"file_path_template":
"analysis/clonality/{run id}/{run id}_pyclone.tsv",
"short_description": "clonality_pyclone file",
"long_description":
"clonality_pyclone file clonality_pyclone file",
"file_purpose": "Miscellaneous",
}],
"tumor cimac id": [{
"filter_group": "alignment/align_sorted_dedup",
"file_path_template":
"analysis/align/{tumor cimac id}/{tumor cimac id}.sorted.dedup.bam",
"short_description": "align_sorted_dedup file",
"long_description":
"align_sorted_dedup file align_sorted_dedup file",
"file_purpose": "Miscellaneous",
}],
}
wes_json = {
"title": "WES analysis template",
"description": "Metadata information for WES Analysis output.",
"prism_template_root_object_schema": "assays/wes_analysis.json",
"prism_template_root_object_pointer": "/analysis/wes_analysis",
"properties": {
"worksheets": {
"WES Analysis": {
"preamble_rows": {
"protocol identifier": {
"merge_pointer":
"2/protocol_identifier",
"type_ref":
"clinical_trial.json#properties/protocol_identifier",
},
"folder": {
"do_not_merge": True,
"type": "string",
"allow_empty": True,
},
},
"prism_data_object_pointer": "/pair_runs/-",
"data_columns": {
"WES Runs": {
"run id": {
"merge_pointer":
"/run_id",
"type_ref":
"assays/wes_analysis.json#definitions/pair_analysis/properties/run_id",
"process_as": [{
"parse_through":
"lambda run: f'{folder or \"\"}analysis/clonality/{run}/{run}_pyclone.tsv'",
"merge_pointer":
"/clonality/clonality_pyclone",
"gcs_uri_format":
"{protocol identifier}/wes/{run id}/analysis/clonality_pyclone.tsv",
"type_ref":
"assays/components/local_file.json#properties/file_path",
"is_artifact": 1,
}],
},
"tumor cimac id": {
"merge_pointer":
"/tumor/cimac_id",
"type_ref":
"sample.json#properties/cimac_id",
"process_as": [{
"parse_through":
"lambda id: f'{folder or \"\"}analysis/align/{id}/{id}.sorted.dedup.bam'",
"merge_pointer":
"/tumor/alignment/align_sorted_dedup",
"gcs_uri_format":
"{protocol identifier}/wes/{run id}/analysis/tumor/{tumor cimac id}/sorted.dedup.bam",
"type_ref":
"assays/components/local_file.json#properties/file_path",
"is_artifact": 1,
}],
},
"comments": {
"type_ref":
"assays/wes_analysis.json#definitions/pair_analysis/properties/comments",
"merge_pointer": "0/comments",
"allow_empty": True,
},
}
},
},
**_excluded_samples_worksheet_snippet,
}
},
}
assay_schema = _load_dont_validate_schema("assays/wes_analysis.json")
wes_output = _convert_api_to_template("wes", wes_api, assay_schema)
assert DeepDiff(wes_json, wes_output) == {}