def validate_manifest(ctx, manifest_path, data_type, json_schema):
"""
Running CLI for manifest validation.
"""
data_type = fill_in_from_config("data_type", data_type,
("manifest", "data_type"))
json_schema = fill_in_from_config("json_schema",
json_schema,
("model", "input", "validation_schema"),
allow_none=True)
jsonld = get_from_config(CONFIG.DATA, ("model", "input", "location"))
model_file_type = get_from_config(CONFIG.DATA,
("model", "input", "file_type"))
metadata_model = MetadataModel(inputMModelLocation=jsonld,
inputMModelLocationType=model_file_type)
errors = metadata_model.validateModelManifest(manifestPath=manifest_path,
rootNode=data_type,
jsonSchema=json_schema)
if not errors:
click.echo("Your manifest has been validated successfully. "
"There are no errors in your manifest, and it can "
"be submitted without any modifications.")
else:
click.echo(errors)
def test_fill_in_from_config(self, mocker):
jsonld = "/path/to/one"
jsonld_none = None
mock_config = {"model": {"path": "/path/to/two"}}
mock_keys = ["model", "path"]
mock_keys_invalid = ["model", "file"]
mocker.patch("schematic.CONFIG.DATA", mock_config)
result1 = cli_utils.fill_in_from_config(
"jsonld", jsonld, mock_keys
)
result2 = cli_utils.fill_in_from_config(
"jsonld", jsonld, mock_keys
)
result3 = cli_utils.fill_in_from_config(
"jsonld_none", jsonld_none, mock_keys
)
assert result1 == "/path/to/one"
assert result2 == "/path/to/one"
assert result3 == "/path/to/two"
with pytest.raises(MissingConfigAndArgumentValueError):
cli_utils.fill_in_from_config(
"jsonld_none", jsonld_none, mock_keys_invalid
)
def get_manifest(ctx, title, data_type, jsonld, dataset_id, sheet_url,
output_csv, use_annotations, oauth, json_schema):
"""
Running CLI with manifest generation options.
"""
# optional parameters that need to be passed to ManifestGenerator()
# can be read from config.yml as well
data_type = fill_in_from_config("data_type", data_type,
("manifest", "data_type"))
jsonld = fill_in_from_config("jsonld", jsonld,
("model", "input", "location"))
title = fill_in_from_config("title",
title, ("manifest", "title"),
allow_none=True)
json_schema = fill_in_from_config("json_schema",
json_schema,
("model", "input", "validation_schema"),
allow_none=True)
# create object of type ManifestGenerator
manifest_generator = ManifestGenerator(
path_to_json_ld=jsonld,
title=title,
root=data_type,
oauth=oauth,
use_annotations=use_annotations,
)
# call get_manifest() on manifest_generator
result = manifest_generator.get_manifest(
dataset_id=dataset_id,
sheet_url=sheet_url,
json_schema=json_schema,
)
if sheet_url:
logger.info("Find the manifest template using this Google Sheet URL:")
click.echo(result)
elif isinstance(result, pd.DataFrame):
if output_csv is None:
prefix, _ = os.path.splitext(jsonld)
prefix_root, prefix_ext = os.path.splitext(prefix)
if prefix_ext == ".model":
prefix = prefix_root
output_csv = f"{prefix}.{data_type}.manifest.csv"
logger.info(
f"Find the manifest template using this CSV file path: {output_csv}"
)
result.to_csv(output_csv, index=False)
return result
def validate_manifest(ctx, manifest_path, data_type, json_schema,
restrict_rules, project_scope):
"""
Running CLI for manifest validation.
"""
data_type = fill_in_from_config("data_type", data_type,
("manifest", "data_type"))
try:
len(data_type) == 1
except:
logger.error(
f"Can only validate a single data_type at a time. Please provide a single data_type"
)
data_type = data_type[0]
json_schema = fill_in_from_config(
"json_schema",
json_schema,
("model", "input", "validation_schema"),
allow_none=True,
)
jsonld = get_from_config(CONFIG.DATA, ("model", "input", "location"))
model_file_type = get_from_config(CONFIG.DATA,
("model", "input", "file_type"))
metadata_model = MetadataModel(inputMModelLocation=jsonld,
inputMModelLocationType=model_file_type)
errors, warnings = metadata_model.validateModelManifest(
manifestPath=manifest_path,
rootNode=data_type,
jsonSchema=json_schema,
restrict_rules=restrict_rules,
project_scope=project_scope,
)
if not errors:
click.echo("Your manifest has been validated successfully. "
"There are no errors in your manifest, and it can "
"be submitted without any modifications.")
else:
click.echo(errors)
def get_manifest(
ctx,
title,
data_type,
jsonld,
dataset_id,
sheet_url,
output_csv,
use_annotations,
oauth,
json_schema,
output_xlsx,
alphabetize_valid_values,
):
"""
Running CLI with manifest generation options.
"""
# optional parameters that need to be passed to ManifestGenerator()
# can be read from config.yml as well
data_type = fill_in_from_config("data_type", data_type,
("manifest", "data_type"))
jsonld = fill_in_from_config("jsonld", jsonld,
("model", "input", "location"))
title = fill_in_from_config("title",
title, ("manifest", "title"),
allow_none=True)
json_schema = fill_in_from_config(
"json_schema",
json_schema,
("model", "input", "validation_schema"),
allow_none=True,
)
def create_single_manifest(data_type, output_csv=None, output_xlsx=None):
# create object of type ManifestGenerator
manifest_generator = ManifestGenerator(
path_to_json_ld=jsonld,
title=t,
root=data_type,
oauth=oauth,
use_annotations=use_annotations,
alphabetize_valid_values=alphabetize_valid_values,
)
# call get_manifest() on manifest_generator
result = manifest_generator.get_manifest(
dataset_id=dataset_id,
sheet_url=sheet_url,
json_schema=json_schema,
)
if sheet_url:
logger.info(
"Find the manifest template using this Google Sheet URL:")
click.echo(result)
if output_csv is None and output_xlsx is None:
prefix, _ = os.path.splitext(jsonld)
prefix_root, prefix_ext = os.path.splitext(prefix)
if prefix_ext == ".model":
prefix = prefix_root
output_csv = f"{prefix}.{data_type}.manifest.csv"
elif output_xlsx:
export_manifest_excel(output_excel=output_xlsx, manifest=result)
logger.info(
f"Find the manifest template using this Excel file path: {output_xlsx}"
)
return result
export_manifest_csv(file_name=output_csv, manifest=result)
logger.info(
f"Find the manifest template using this CSV file path: {output_csv}"
)
return result
if type(data_type) is str:
data_type = [data_type]
if data_type[0] == 'all manifests':
sg = SchemaGenerator(path_to_json_ld=jsonld)
component_digraph = sg.se.get_digraph_by_edge_type('requiresComponent')
components = component_digraph.nodes()
for component in components:
t = f'{title}.{component}.manifest'
result = create_single_manifest(data_type=component)
else:
for dt in data_type:
if len(data_type) > 1:
t = f'{title}.{dt}.manifest'
else:
t = title
result = create_single_manifest(data_type=dt,
output_csv=output_csv,
output_xlsx=output_xlsx)
return result