def users_validate(jsonfile, verbose, debug):
"""Check users validation."""
click.secho('Validate user file', fg='green')
path = current_jsonschemas.url_to_path(get_schema_for_resource('ptrn'))
ptrn_schema = current_jsonschemas.get_schema(path=path)
ptrn_schema = records_state.replace_refs(ptrn_schema)
# TODO: get user schema path programaticly
# path = current_jsonschemas.url_to_path(get_schema_for_resource('user'))
path = 'users/user-v0.0.1.json'
user_schema = current_jsonschemas.get_schema(path=path)
user_schema = records_state.replace_refs(user_schema)
merger_schema = {"properties": {"required": {"mergeStrategy": "append"}}}
merger = Merger(merger_schema)
schema = merger.merge(user_schema, ptrn_schema)
schema['required'] = [
s for s in schema['required'] if s not in ['$schema', 'user_id']
]
datas = read_json_record(jsonfile)
for idx, data in enumerate(datas):
if verbose:
click.echo(f'\tTest record: {idx}')
try:
validate(data, schema)
except ValidationError as err:
click.secho(
f'Error validate in record: {idx} pid: {data.get("pid")}',
fg='red')
if debug:
click.secho(str(err))
else:
trace_lines = traceback.format_exc(1).split('\n')
click.secho(trace_lines[3].strip())
def get_remote_json(self, uri, **kwargs):
"""Get remote json.
Adds loading of $ref locally for the application instance.
See: github invenio-jsonschemas ext.py.
:param uri: The URI of the JSON document to load.
:param kwargs: Keyword arguments passed to json.loads().
:returns: resolved json schema.
"""
path = current_jsonschemas.url_to_path(uri)
if path:
result = current_jsonschemas.get_schema(path=path)
else:
result = super(JsonLoader, self).get_remote_json(uri, **kwargs)
return result
def get_document_types_from_schema(schema='doc'):
"""Create document type definition from schema."""
path = current_jsonschemas.url_to_path(get_schema_for_resource(schema))
schema = current_jsonschemas.get_schema(path=path)
schema = _records_state.replace_refs(schema)
schema_types = schema.get('properties',
{}).get('type', {}).get('items',
{}).get('oneOf', [])
doc_types = {}
for schema_type in schema_types:
doc_types[schema_type['title']] = {}
sub_types = schema_type.get('properties', {}).get('subtype',
{}).get('enum', [])
for sub_type in sub_types:
doc_types[schema_type['title']][sub_type] = True
return doc_types
def create_csv(record_type, json_file, output_directory, lazy, verbose,
create_pid):
"""Create csv files from json.
:param verbose: Verbose.
"""
click.secho(f"Create CSV files for: {record_type} from: {json_file}",
fg='green')
path = current_jsonschemas.url_to_path(
get_schema_for_resource(record_type))
add_schema = get_schema_for_resource(record_type)
schema = current_jsonschemas.get_schema(path=path)
schema = _records_state.replace_refs(schema)
count = 0
errors_count = 0
with open(json_file) as infile:
if lazy:
# try to lazy read json file (slower, better memory management)
records = read_json_record(infile)
else:
# load everything in memory (faster, bad memory management)
records = json.load(infile)
file_name_pidstore = os.path.join(output_directory,
f'{record_type}_pidstore.csv')
click.secho(f'\t{file_name_pidstore}', fg='green')
file_pidstore = open(file_name_pidstore, 'w')
file_name_metadata = os.path.join(output_directory,
f'{record_type}_metadata.csv')
click.secho(f'\t{file_name_metadata}', fg='green')
file_metadata = open(file_name_metadata, 'w')
file_name_pids = os.path.join(output_directory,
f'{record_type}_pids.csv')
click.secho(f'\t{file_name_pids}', fg='green')
file_pids = open(file_name_pids, 'w')
file_name_errors = os.path.join(output_directory,
f'{record_type}_errors.json')
file_errors = open(file_name_errors, 'w')
file_errors.write('[')
for count, record in enumerate(records, 1):
pid = record.get('pid')
if create_pid:
pid = str(count)
record['pid'] = pid
uuid = str(uuid4())
if verbose:
click.secho(f'{count}\t{record_type}\t{pid}:{uuid}')
date = str(datetime.utcnow())
record['$schema'] = add_schema
try:
validate(record, schema)
file_metadata.write(csv_metadata_line(record, uuid, date))
file_pidstore.write(
csv_pidstore_line(record_type, pid, uuid, date))
file_pids.write(pid + '\n')
except Exception as err:
click.secho(
f'{count}\t{record_type}: Error validate in record: ',
fg='red')
click.secho(str(err))
if errors_count > 0:
file_errors.write(',')
errors_count += 1
file_errors.write('\n')
for line in json.dumps(record, indent=2).split('\n'):
file_errors.write(' ' + line + '\n')
file_pidstore.close()
file_metadata.close()
file_pids.close()
file_errors.write('\n]')
file_errors.close()
if errors_count == 0:
os.remove(file_name_errors)
click.secho(f'Created: {count-errors_count} Errors: {errors_count}',
fg='yellow')
def users_validate(jsonfile, verbose, debug):
"""Check users validation."""
click.secho('Validate user file: ', fg='green', nl=False)
click.echo(f'{jsonfile.name}')
path = current_jsonschemas.url_to_path(get_schema_for_resource('ptrn'))
ptrn_schema = current_jsonschemas.get_schema(path=path)
ptrn_schema = records_state.replace_refs(ptrn_schema)
# TODO: get user schema path programaticly
# path = current_jsonschemas.url_to_path(get_schema_for_resource('user'))
path = 'users/user-v0.0.1.json'
user_schema = current_jsonschemas.get_schema(path=path)
user_schema = records_state.replace_refs(user_schema)
merger_schema = {"properties": {"required": {"mergeStrategy": "append"}}}
merger = Merger(merger_schema)
schema = merger.merge(user_schema, ptrn_schema)
schema['required'] = [
s for s in schema['required'] if s not in ['$schema', 'user_id']
]
datas = read_json_record(jsonfile)
librarien_roles_users = {}
for idx, data in enumerate(datas):
if verbose:
click.echo(f'\tTest record: {idx} pid: {data.get("pid")}')
try:
validate(data, schema)
patron = data.get('patron', {})
if patron and patron.get('communication_channel') == 'email'\
and data.get('email') is None \
and patron.get('additional_communication_email') is None:
raise ValidationError('At least one email should be defined '
'for an email communication channel.')
librarian_roles = [
Patron.ROLE_SYSTEM_LIBRARIAN, Patron.ROLE_LIBRARIAN
]
roles = data.get('roles', [])
if any(role in librarian_roles for role in roles):
if not data.get('libraries'):
raise ValidationError('Missing libraries')
# test multiple librarien, roles for same user
username = data.get('username')
if username in librarien_roles_users:
raise ValidationError('Multiple librarian roles')
else:
librarien_roles_users[username] = 1
birth_date = data.get('birth_date')
if birth_date[0] == '0':
raise ValidationError(f'Wrong birth date: {birth_date}')
except ValidationError as err:
click.secho(
f'Error validate in record: {idx} pid: {data.get("pid")} '
f'username: {data.get("username")}',
fg='red')
if debug:
click.secho(str(err))
else:
trace_lines = traceback.format_exc(1).split('\n')
click.secho(trace_lines[3].strip())
def build_docs(schemas):
"""Generates API docs for included / specified data models."""
from json_schema_for_humans.generate import generate_from_file_object
for schema_path in schemas or current_jsonschemas.list_schemas():
click.secho(f'Generating docs for schema {schema_path}')
schema = current_jsonschemas.get_schema(schema_path,
with_refs=False,
resolved=False)
try:
schema = JsonRef.replace_refs(
schema,
jsonschema=True,
base_uri=current_app.config.get('JSONSCHEMAS_HOST'),
loader=_records_state.loader_cls(),
)
# TODO: this is necessary to resolve JSONRefs in allOf
schema = json.loads(
json.dumps(schema, default=lambda x: x.__subject__))
# Resolve definition schemas
if 'definitions' in schema:
definitions = list(schema['definitions'].keys())
# Consider only a first definition as a schema for now
schema = schema['definitions'][definitions[0]]
click.secho(f'Schema resolved to: {json.dumps(schema)}',
color='blue')
except JsonRefError as e:
click.secho(f'Error resolving schema: {e}. Skipping...',
color='red')
continue
# Generate and save html docs for the schema
with tempfile.NamedTemporaryFile(mode="w+") as schema_source:
schema_source.write(json.dumps(schema))
schema_source.flush()
with open(
f'docs/schemas/{basename(schema_path.rstrip(".json"))}.html',
mode='w+') as result_file:
click.secho(f'Writing schema docs to {result_file.name}',
color='green')
generate_from_file_object(schema_file=schema_source,
result_file=result_file,
minify=True,
expand_buttons=True)
# Generate and save schema index page
index_md = r"""---
layout: default
---
# Data Models Schema Docs
"""
for f in os.listdir('docs/schemas/'):
if f.endswith('.html'):
index_md += f'- [{f.rstrip(".html")}](./{f})\n'
with open(f'docs/schemas/index.md', mode='w+') as index_file:
index_file.write(index_md)
def get_resolved_record_schema(self, obj):
"""Resolve refs in record schema."""
schema = current_jsonschemas.get_schema(obj.record_path,
with_refs=True,
resolved=True)
return copy.deepcopy(schema) # so all the JSONRefs get resoved