def validate_manifest(ctx, manifest_path, data_type, json_schema):
"""
Running CLI for manifest validation.
"""
data_type = fill_in_from_config("data_type", data_type,
("manifest", "data_type"))
json_schema = fill_in_from_config("json_schema",
json_schema,
("model", "input", "validation_schema"),
allow_none=True)
jsonld = get_from_config(CONFIG.DATA, ("model", "input", "location"))
model_file_type = get_from_config(CONFIG.DATA,
("model", "input", "file_type"))
metadata_model = MetadataModel(inputMModelLocation=jsonld,
inputMModelLocationType=model_file_type)
errors = metadata_model.validateModelManifest(manifestPath=manifest_path,
rootNode=data_type,
jsonSchema=json_schema)
if not errors:
click.echo("Your manifest has been validated successfully. "
"There are no errors in your manifest, and it can "
"be submitted without any modifications.")
else:
click.echo(errors)
def submit_manifest(ctx, manifest_path, dataset_id, validate_component):
"""
Running CLI with manifest validation (optional) and submission options.
"""
jsonld = get_from_config(CONFIG.DATA, ("model", "input", "location"))
model_file_type = get_from_config(CONFIG.DATA,
("model", "input", "file_type"))
metadata_model = MetadataModel(inputMModelLocation=jsonld,
inputMModelLocationType=model_file_type)
try:
success = metadata_model.submit_metadata_manifest(
manifest_path=manifest_path,
dataset_id=dataset_id,
validate_component=validate_component)
if success:
logger.info(
f"File at '{manifest_path}' was successfully associated "
f"with dataset '{dataset_id}'.")
except ValueError:
logger.error(
f"Component '{validate_component}' is not present in '{jsonld}', or is invalid."
)
except ValidationError:
logger.error(
f"Validation errors resulted while validating with '{validate_component}'."
)
def submit_manifest(
ctx,
manifest_path,
dataset_id,
validate_component,
manifest_record_type,
use_schema_label,
hide_blanks,
restrict_rules,
project_scope,
):
"""
Running CLI with manifest validation (optional) and submission options.
"""
jsonld = get_from_config(CONFIG.DATA, ("model", "input", "location"))
model_file_type = get_from_config(CONFIG.DATA,
("model", "input", "file_type"))
metadata_model = MetadataModel(inputMModelLocation=jsonld,
inputMModelLocationType=model_file_type)
try:
manifest_id = metadata_model.submit_metadata_manifest(
path_to_json_ld=jsonld,
manifest_path=manifest_path,
dataset_id=dataset_id,
validate_component=validate_component,
manifest_record_type=manifest_record_type,
restrict_rules=restrict_rules,
use_schema_label=use_schema_label,
hide_blanks=hide_blanks,
project_scope=project_scope,
)
'''
if censored_manifest_id:
logger.info(
f"File at '{manifest_path}' was censored and successfully associated "
f"with dataset '{dataset_id}'. "
f"An uncensored version has also been associated with dataset '{dataset_id}' "
f"and submitted to the Synapse Access Control Team to begin the process "
f"of adding terms of use or review board approval."
)
'''
if manifest_id:
logger.info(
f"File at '{manifest_path}' was successfully associated "
f"with dataset '{dataset_id}'.")
except ValueError:
logger.error(
f"Component '{validate_component}' is not present in '{jsonld}', or is invalid."
)
except ValidationError:
logger.error(
f"Validation errors resulted while validating with '{validate_component}'."
)
def metadata_model(helpers):
metadata_model = MetadataModel(
inputMModelLocation=helpers.get_data_path("example.model.jsonld"),
inputMModelLocationType="local",
)
yield metadata_model
def validate_manifest_route(schema_url, data_type):
# call config_handler()
config_handler()
#Get path to temp file where manifest file contents will be saved
temp_path = csv_path_handler()
# get path to temporary JSON-LD file
jsonld = get_temp_jsonld(schema_url)
metadata_model = MetadataModel(inputMModelLocation=jsonld,
inputMModelLocationType="local")
errors = metadata_model.validateModelManifest(manifestPath=temp_path,
rootNode=data_type)
return errors
def validate_manifest(ctx, manifest_path, data_type, json_schema,
restrict_rules, project_scope):
"""
Running CLI for manifest validation.
"""
data_type = fill_in_from_config("data_type", data_type,
("manifest", "data_type"))
try:
len(data_type) == 1
except:
logger.error(
f"Can only validate a single data_type at a time. Please provide a single data_type"
)
data_type = data_type[0]
json_schema = fill_in_from_config(
"json_schema",
json_schema,
("model", "input", "validation_schema"),
allow_none=True,
)
jsonld = get_from_config(CONFIG.DATA, ("model", "input", "location"))
model_file_type = get_from_config(CONFIG.DATA,
("model", "input", "file_type"))
metadata_model = MetadataModel(inputMModelLocation=jsonld,
inputMModelLocationType=model_file_type)
errors, warnings = metadata_model.validateModelManifest(
manifestPath=manifest_path,
rootNode=data_type,
jsonSchema=json_schema,
restrict_rules=restrict_rules,
project_scope=project_scope,
)
if not errors:
click.echo("Your manifest has been validated successfully. "
"There are no errors in your manifest, and it can "
"be submitted without any modifications.")
else:
click.echo(errors)
def populate_manifest_route(schema_url, title=None, data_type=None):
# call config_handler()
config_handler()
# get path to temporary JSON-LD file
jsonld = get_temp_jsonld(schema_url)
# Get path to temp file where manifest file contents will be saved
temp_path = csv_path_handler()
#Initalize MetadataModel
metadata_model = MetadataModel(inputMModelLocation=jsonld,
inputMModelLocationType='local')
#Call populateModelManifest class
populated_manifest_link = metadata_model.populateModelManifest(
title=title, manifestPath=temp_path, rootNode=data_type)
return populated_manifest_link
import os
import yaml
from schematic.models.metadata import MetadataModel
from schematic import CONFIG
with open("config.yaml") as app_config_file:
app_config = yaml.safe_load(app_config_file)
schematic_config = app_config["schematic_config"]
schematic_config_dir = os.path.dirname(schematic_config)
config = CONFIG.load_config(app_config["schematic_config"])
inputMModelLocation = CONFIG["model"]["input"]["location"]
inputMModelLocationType = CONFIG["model"]["input"]["file_type"]
if not os.path.isabs(inputMModelLocation):
inputMModelLocation = os.path.join(schematic_config_dir,
inputMModelLocation)
inputMModelLocation = os.path.normpath(inputMModelLocation)
metadata_model = MetadataModel(inputMModelLocation, inputMModelLocationType)
from schematic import CONFIG
config = CONFIG.load_config("./schematic/config.yml")
#inputMModelLocation = "./schemas/exampleSchemaReq.jsonld"
#inputMModelLocation = "./HTAN-data-pipeline/schemas/scRNASeq.jsonld"
# inputMModelLocation = "./HTAN-data-pipeline/schemas/HTAPP.jsonld"
inputMModelLocation = CONFIG["model"]["input"]["location"]
inputMModelLocationType = CONFIG["model"]["input"]["file_type"]
manifest_title = CONFIG["manifest"]["title"]
manifest_data_type = CONFIG["manifest"]["data_type"]
# datasetType = "scRNASeq"
# modelType = "TableA"
metadata_model = MetadataModel(inputMModelLocation, inputMModelLocationType)
metadata_model.getModelManifest(title=manifest_title,
rootNode=manifest_data_type)
### function for getting model Manifest
# mm.getModelManifest(modelType, additionalMetadata = {"Filename":["MantonCB1_HiSeq_1_S1_L001_R1_001.fastq.gz"]} )
# getModelManifest = mm.getModelManifest
# ### function for validating manifest
# # mm.validateModelManifest(manifest_path, datasetType)
# validateModelManifest = mm.validateModelManifest
# ### populates manifest with path to csv
# populateModelManifest = mm.populateModelManifest
# ### gets dependencies
def initalize_metadata_model(schema_url):
jsonld = get_temp_jsonld(schema_url)
metadata_model = MetadataModel(inputMModelLocation=jsonld,
inputMModelLocationType="local")
return metadata_model