def check_and_update_data_with_schema(data, ana): """ Checks if the analysis type is included in the schema, or adds it. It also checks if the schema provided is valid. """ schema = data.get('$schema') if not schema and not ana: click.secho( 'You need to provide the --ana/-a parameter OR ' 'add the $schema field in your JSON', fg='red') return False try: if schema: if ana: click.secho( "Your data already provide a $schema, --ana will not be used." ) # noqa resolve_schema_by_url(schema) elif ana: data['$schema'] = schema_name_to_url(ana) return True except JSONSchemaNotFound: click.secho('Provided schema is not a valid option.', fg='red') return False
def test_schema_name_to_url(db): db.session.add(Schema(name='my-schema', version='1.5.5')) db.session.add(Schema(name='my-schema', version='2.4.0')) db.session.add(Schema(name='my-schema', version='2.4.3')) db.session.add(Schema(name='different-schema', version='3.5.4')) db.session.commit() assert schema_name_to_url( 'my-schema' ) == 'https://analysispreservation.cern.ch/schemas/deposits/records/my-schema-v2.4.3.json'
def _preprocess_create_data(cls, data, uuid_, owner): """Preprocess metadata for new deposit. :param data: metadata, need to contain $schema|$ana_type field :type data: dict :param id_: specify a UUID to use for the new record, instead of automatically generated :type id_: `uuid.UUID` :param owner: owner of a new deposit (will get all permissions) :type owner: `invenio_accounts.models.User` :returns: processed metadata dictionary :rtype: dict """ if not isinstance(data, dict) or data == {}: raise DepositValidationError('Empty deposit data.') if '$ana_type' in data: try: ana_type = data.pop('$ana_type') data['$schema'] = schema_name_to_url(ana_type) except JSONSchemaNotFound: raise DepositValidationError( f'Schema {ana_type} is not a valid deposit schema.') elif '$schema' not in data: raise DepositValidationError('Schema not specified.') try: schema = resolve_schema_by_url(data['$schema']) data['_experiment'] = schema.experiment except JSONSchemaNotFound: raise DepositValidationError( f'Schema {data["$schema"]} is not a valid deposit schema.') # minting is done by invenio on POST action preprocessing, # if method called programatically mint PID here if '_deposit' not in data: cls.deposit_minter(uuid_, data) if owner: data['_deposit']['owners'] = [owner.id] data['_deposit']['created_by'] = owner.id data['_access'] = { permission: { 'users': [owner.id], 'roles': [] } for permission in DEPOSIT_ACTIONS } else: data['_deposit']['owners'] = [] data['_access'] = copy.deepcopy(EMPTY_ACCESS_OBJECT) return data
def _preprocess_data(cls, data): # data can be sent without specifying particular version of schema, # but just with a type, e.g. cms-analysis # this be resolved to the last version of deposit schema of this type if '$ana_type' in data: try: ana_type = data.pop('$ana_type') data['$schema'] = schema_name_to_url(ana_type) except JSONSchemaNotFound: raise DepositValidationError( 'Schema {} is not a valid deposit schema.'.format( ana_type)) return data
def validate(schema_url, ana_type, ana_version, compare_with, status, export, export_type): """ Validate deposit or record metadata based on their schema. Provide the schema url OR ana-type and version, as well as the schema version that you want to compare the records you get, to. E.g. If you do not provide an ana-version, it will get the latest. If you do not provide a -c parameter, the records will compare the data to their own schema. cap fixtures validate -u https://analysispreservation.cern.ch/schemas/deposits/records/test-v2.0.0.json -c 1.0.0 # noqa cap fixtures validate -a test -c 1.0.0 """ try: if schema_url: schema = resolve_schema_by_url(schema_url) elif ana_type: schema = resolve_schema_by_name_and_version(ana_type, ana_version) else: raise click.UsageError( 'You need to provide the ana-type or the schema-url.') except JSONSchemaNotFound: raise click.UsageError('Schema not found.') except ValueError: raise click.UsageError( 'Version has to be passed as string <major>.<minor>.<patch>.') # differentiate between drafts/published from cap.modules.deposit.api import CAPDeposit if status == 'draft': search_path = 'deposits-records' cap_record_class = CAPDeposit else: search_path = 'records' cap_record_class = CAPRecord # get all the records for this specific schema/type combination records = current_search_client.search( search_path, q=f'_deposit.status: {status} AND ' f'$schema: "{schema_name_to_url(schema.name, schema.version)}"', size=5000)['hits']['hits'] pids = [rec['_id'] for rec in records] click.secho(f'{len(records)} record(s) of {schema.name} found.\n', fg='green') total_errors = [] for pid in pids: cap_record = cap_record_class.get_record(pid) cap_record_pid = cap_record.get('_deposit', {}).get('id') cap_record_cadi_id = cap_record.get('basic_info', {}).get('cadi_id') cap_host = 'https://analysispreservation.cern.ch/drafts' # get the url of the schema version, used for validation if compare_with: cap_record['$schema'] = schema_name_to_url(schema.name, compare_with) try: cap_record.validate() click.secho(f'No errors found in record {pid}', fg='green') except DepositValidationError as exc: if export_type == 'md': msg = '- [ ] Errors in **CADI ID:** ' + \ f'{cap_record_cadi_id or "?"}' + \ f' - **[link]({cap_host}/{cap_record_pid})** :\n' msg += "\n| Field Path | Error | \n| ---------- | ----- | \n" for err in exc.errors: _err = err.res msg += f"| ```{_err.get('field')}``` |" msg += f" {_err.get('message')} | \n" msg += "----\n" else: error_list = '\n'.join(str(err.res) for err in exc.errors) msg = f'Errors in {pid} - CADI ' + \ f'id: {cap_record_cadi_id or "?"}' + \ f' - {cap_host}/{cap_record_pid} :\n{error_list}' click.secho(msg, fg='red') if export: total_errors.append(msg) # export the errors in a file if export: with open(export, 'w') as out: out.writelines('\n\n'.join(err for err in total_errors)) click.secho(f'Errors saved at {export}.', fg='red')
def test_schema_name_to_url_when_schema_doesnt_exist_raises_JSONSchemaNotFound( db): with raises(JSONSchemaNotFound): schema_name_to_url('non-existing-schema')