def validate_manifest(ctx, manifest_path, data_type, json_schema): """ Running CLI for manifest validation. """ data_type = fill_in_from_config("data_type", data_type, ("manifest", "data_type")) json_schema = fill_in_from_config("json_schema", json_schema, ("model", "input", "validation_schema"), allow_none=True) jsonld = get_from_config(CONFIG.DATA, ("model", "input", "location")) model_file_type = get_from_config(CONFIG.DATA, ("model", "input", "file_type")) metadata_model = MetadataModel(inputMModelLocation=jsonld, inputMModelLocationType=model_file_type) errors = metadata_model.validateModelManifest(manifestPath=manifest_path, rootNode=data_type, jsonSchema=json_schema) if not errors: click.echo("Your manifest has been validated successfully. " "There are no errors in your manifest, and it can " "be submitted without any modifications.") else: click.echo(errors)
def submit_manifest(ctx, manifest_path, dataset_id, validate_component): """ Running CLI with manifest validation (optional) and submission options. """ jsonld = get_from_config(CONFIG.DATA, ("model", "input", "location")) model_file_type = get_from_config(CONFIG.DATA, ("model", "input", "file_type")) metadata_model = MetadataModel(inputMModelLocation=jsonld, inputMModelLocationType=model_file_type) try: success = metadata_model.submit_metadata_manifest( manifest_path=manifest_path, dataset_id=dataset_id, validate_component=validate_component) if success: logger.info( f"File at '{manifest_path}' was successfully associated " f"with dataset '{dataset_id}'.") except ValueError: logger.error( f"Component '{validate_component}' is not present in '{jsonld}', or is invalid." ) except ValidationError: logger.error( f"Validation errors resulted while validating with '{validate_component}'." )
def submit_manifest( ctx, manifest_path, dataset_id, validate_component, manifest_record_type, use_schema_label, hide_blanks, restrict_rules, project_scope, ): """ Running CLI with manifest validation (optional) and submission options. """ jsonld = get_from_config(CONFIG.DATA, ("model", "input", "location")) model_file_type = get_from_config(CONFIG.DATA, ("model", "input", "file_type")) metadata_model = MetadataModel(inputMModelLocation=jsonld, inputMModelLocationType=model_file_type) try: manifest_id = metadata_model.submit_metadata_manifest( path_to_json_ld=jsonld, manifest_path=manifest_path, dataset_id=dataset_id, validate_component=validate_component, manifest_record_type=manifest_record_type, restrict_rules=restrict_rules, use_schema_label=use_schema_label, hide_blanks=hide_blanks, project_scope=project_scope, ) ''' if censored_manifest_id: logger.info( f"File at '{manifest_path}' was censored and successfully associated " f"with dataset '{dataset_id}'. " f"An uncensored version has also been associated with dataset '{dataset_id}' " f"and submitted to the Synapse Access Control Team to begin the process " f"of adding terms of use or review board approval." ) ''' if manifest_id: logger.info( f"File at '{manifest_path}' was successfully associated " f"with dataset '{dataset_id}'.") except ValueError: logger.error( f"Component '{validate_component}' is not present in '{jsonld}', or is invalid." ) except ValidationError: logger.error( f"Validation errors resulted while validating with '{validate_component}'." )
def metadata_model(helpers): metadata_model = MetadataModel( inputMModelLocation=helpers.get_data_path("example.model.jsonld"), inputMModelLocationType="local", ) yield metadata_model
def validate_manifest_route(schema_url, data_type): # call config_handler() config_handler() #Get path to temp file where manifest file contents will be saved temp_path = csv_path_handler() # get path to temporary JSON-LD file jsonld = get_temp_jsonld(schema_url) metadata_model = MetadataModel(inputMModelLocation=jsonld, inputMModelLocationType="local") errors = metadata_model.validateModelManifest(manifestPath=temp_path, rootNode=data_type) return errors
def validate_manifest(ctx, manifest_path, data_type, json_schema, restrict_rules, project_scope): """ Running CLI for manifest validation. """ data_type = fill_in_from_config("data_type", data_type, ("manifest", "data_type")) try: len(data_type) == 1 except: logger.error( f"Can only validate a single data_type at a time. Please provide a single data_type" ) data_type = data_type[0] json_schema = fill_in_from_config( "json_schema", json_schema, ("model", "input", "validation_schema"), allow_none=True, ) jsonld = get_from_config(CONFIG.DATA, ("model", "input", "location")) model_file_type = get_from_config(CONFIG.DATA, ("model", "input", "file_type")) metadata_model = MetadataModel(inputMModelLocation=jsonld, inputMModelLocationType=model_file_type) errors, warnings = metadata_model.validateModelManifest( manifestPath=manifest_path, rootNode=data_type, jsonSchema=json_schema, restrict_rules=restrict_rules, project_scope=project_scope, ) if not errors: click.echo("Your manifest has been validated successfully. " "There are no errors in your manifest, and it can " "be submitted without any modifications.") else: click.echo(errors)
def populate_manifest_route(schema_url, title=None, data_type=None): # call config_handler() config_handler() # get path to temporary JSON-LD file jsonld = get_temp_jsonld(schema_url) # Get path to temp file where manifest file contents will be saved temp_path = csv_path_handler() #Initalize MetadataModel metadata_model = MetadataModel(inputMModelLocation=jsonld, inputMModelLocationType='local') #Call populateModelManifest class populated_manifest_link = metadata_model.populateModelManifest( title=title, manifestPath=temp_path, rootNode=data_type) return populated_manifest_link
import os import yaml from schematic.models.metadata import MetadataModel from schematic import CONFIG with open("config.yaml") as app_config_file: app_config = yaml.safe_load(app_config_file) schematic_config = app_config["schematic_config"] schematic_config_dir = os.path.dirname(schematic_config) config = CONFIG.load_config(app_config["schematic_config"]) inputMModelLocation = CONFIG["model"]["input"]["location"] inputMModelLocationType = CONFIG["model"]["input"]["file_type"] if not os.path.isabs(inputMModelLocation): inputMModelLocation = os.path.join(schematic_config_dir, inputMModelLocation) inputMModelLocation = os.path.normpath(inputMModelLocation) metadata_model = MetadataModel(inputMModelLocation, inputMModelLocationType)
from schematic import CONFIG config = CONFIG.load_config("./schematic/config.yml") #inputMModelLocation = "./schemas/exampleSchemaReq.jsonld" #inputMModelLocation = "./HTAN-data-pipeline/schemas/scRNASeq.jsonld" # inputMModelLocation = "./HTAN-data-pipeline/schemas/HTAPP.jsonld" inputMModelLocation = CONFIG["model"]["input"]["location"] inputMModelLocationType = CONFIG["model"]["input"]["file_type"] manifest_title = CONFIG["manifest"]["title"] manifest_data_type = CONFIG["manifest"]["data_type"] # datasetType = "scRNASeq" # modelType = "TableA" metadata_model = MetadataModel(inputMModelLocation, inputMModelLocationType) metadata_model.getModelManifest(title=manifest_title, rootNode=manifest_data_type) ### function for getting model Manifest # mm.getModelManifest(modelType, additionalMetadata = {"Filename":["MantonCB1_HiSeq_1_S1_L001_R1_001.fastq.gz"]} ) # getModelManifest = mm.getModelManifest # ### function for validating manifest # # mm.validateModelManifest(manifest_path, datasetType) # validateModelManifest = mm.validateModelManifest # ### populates manifest with path to csv # populateModelManifest = mm.populateModelManifest # ### gets dependencies
def initalize_metadata_model(schema_url): jsonld = get_temp_jsonld(schema_url) metadata_model = MetadataModel(inputMModelLocation=jsonld, inputMModelLocationType="local") return metadata_model