def master_mixins():
mixins = load_schema('encoded:schemas/mixins.json')
mixin_keys = [
'schema_version',
'uuid',
'accession',
'aliases',
'status',
'submitted',
'release_dates',
'modified',
'references',
'attribution',
'notes',
'documents',
'attachment',
'dbxrefs',
'library',
'antibody_info',
'spikein_info',
'sop_mapping',
'tags',
'badges',
'facets_common'
]
for key in mixin_keys:
assert(mixins[key])
def test_biosample_upgrade_starting_amount_unknown(testapp, biosample_1):
from snovault.schema_utils import load_schema
schema = load_schema('encoded:schemas/biosample.json')
biosample_1['starting_amount'] = 'unknown'
biosample_1['starting_amount_units'] = 'g'
res = testapp.post_json('/biosample?validate=false&render=uuid',
biosample_1)
location = res.location
# The properties are stored un-upgraded.
res = testapp.get(location + '?frame=raw&upgrade=false').maybe_follow()
assert res.json['schema_version'] == '1'
# When the item is fetched, it is upgraded automatically.
res = testapp.get(location).maybe_follow()
assert res.json['schema_version'] == schema['properties'][
'schema_version']['default']
res = testapp.patch_json(location, {})
# The stored properties are now upgraded.
res = testapp.get(location + '?frame=raw&upgrade=false').maybe_follow()
assert res.json['schema_version'] == schema['properties'][
'schema_version']['default']
assert 'starting_amount' not in res.json
assert 'starting_amount_units' not in res.json
def test_fourfront_crawl_schemas(testapp, registry):
from snovault import TYPES
from snovault.schema_utils import load_schema
schema = load_schema('encoded:schemas/experiment_hi_c.json')
field_path = 'files.extra_files.file_size'
field_schema = crawl_schema(registry[TYPES], field_path, schema)
assert isinstance(field_schema, dict)
assert field_schema['title'] == 'File Size'
def test_biosample_upgrade_inline_unknown(testapp, biosample_1):
from snovault.schema_utils import load_schema
schema = load_schema('encoded:schemas/biosample.json')
biosample_1['starting_amount'] = 'Unknown'
res = testapp.post_json('/biosample?validate=false&render=uuid', biosample_1)
location = res.location
res = testapp.patch_json(location, {})
res = testapp.get(location + '?frame=raw&upgrade=false').maybe_follow()
assert res.json['schema_version'] == schema['properties']['schema_version']['default']
assert 'starting_amount' not in res.json
def compute_master_mixins():
mixins = load_schema('encoded:schemas/mixins.json')
mixin_keys = [
'schema_version', 'uuid', 'accession', 'aliases', 'status',
'submitted', 'modified', 'attribution', 'notes', 'documents',
'attachment', 'dbxrefs', 'alternative_ids', 'static_embeds', 'tags',
'facets_common', 'supplementary_files'
]
for key in mixin_keys:
assert (mixins[key])
def test_biosample_upgrade_inline_unknown(testapp, biosample_1):
from snovault.schema_utils import load_schema
schema = load_schema('encoded:schemas/biosample.json')
biosample_1['starting_amount'] = 'Unknown'
res = testapp.post_json('/biosample?validate=false&render=uuid',
biosample_1)
location = res.location
res = testapp.patch_json(location, {})
res = testapp.get(location + '?frame=raw&upgrade=false').maybe_follow()
assert res.json['schema_version'] == schema['properties'][
'schema_version']['default']
assert 'starting_amount' not in res.json
def test_biosample_upgrade_inline(testapp, biosample_1, heart):
from snovault.schema_utils import load_schema
schema = load_schema('encoded:schemas/biosample.json')
res = testapp.post_json('/biosample?validate=false&render=uuid', biosample_1)
location = res.location
# The properties are stored un-upgraded.
res = testapp.get(location + '?frame=raw&upgrade=false').maybe_follow()
assert res.json['schema_version'] == '1'
# When the item is fetched, it is upgraded automatically.
res = testapp.get(location).maybe_follow()
assert res.json['schema_version'] == schema['properties']['schema_version']['default']
res = testapp.patch_json(location, {})
# The stored properties are now upgraded.
res = testapp.get(location + '?frame=raw&upgrade=false').maybe_follow()
assert res.json['schema_version'] == schema['properties']['schema_version']['default']
def test_facets_and_columns_orders(schema, testapp):
'''This tests depends on Python 3.6's ordered dicts'''
loaded_schema = load_schema('encoded:schemas/%s' % schema)
if "properties" in loaded_schema and ("columns" in loaded_schema
or "facets" in loaded_schema):
loaded_schema_copy = deepcopy(loaded_schema)
loaded_schema_copy = order_schema_columns_and_facets(
loaded_schema_copy)
failed = False
if "columns" in loaded_schema:
failed = json.dumps(loaded_schema["columns"]) != json.dumps(
loaded_schema_copy["columns"])
if not failed and "facets" in loaded_schema:
# Avoid running if already failed.
failed = json.dumps(loaded_schema["facets"]) != json.dumps(
loaded_schema_copy["facets"])
assert not failed, '''
Order of facets or columns in %s file does not match the ordering based on "order" values. \
Please run `poetry run order-schema-columns-and-facets`. \
If you don't want this test to fail ever again, please consider adding the follow as "pre-commit" \
file in your .git/hooks directory in order to automatically amend your commits with proper order when \
schemas change.
> #!/bin/sh
>
> CHANGED=`git diff HEAD@{0} --stat -- $GIT_DIR/../src/encoded/schemas/ | wc -l`
> if [ $CHANGED -gt 0 ];
> then
> echo "Schemas have changed! Sorting columns and facets..."
> python3 $GIT_DIR/../src/encoded/commands/order_schema_columns_and_facets.py
> git add $GIT_DIR/../src/encoded/schemas/
> fi
''' % schema
def test_biosample_upgrade_starting_amount_explicit_patch(testapp, biosample_1):
from snovault.schema_utils import load_schema
schema = load_schema('encoded:schemas/biosample.json')
biosample_1['starting_amount'] = 0.632
biosample_1['starting_amount_units'] = 'g'
res = testapp.post_json('/biosample?validate=false&render=uuid', biosample_1)
location = res.location
# The properties are stored un-upgraded.
res = testapp.get(location + '?frame=raw&upgrade=false').maybe_follow()
assert res.json['schema_version'] == '1'
# When the item is fetched, it is upgraded automatically.
res = testapp.get(location).maybe_follow()
assert res.json['schema_version'] == schema['properties']['schema_version']['default']
res = testapp.patch_json(location, {'starting_amount': 0.263})
# The stored properties are now upgraded.
res = testapp.get(location + '?frame=raw&upgrade=false').maybe_follow()
assert res.json['schema_version'] == schema['properties']['schema_version']['default']
assert res.json['starting_amount'] == 0.263
assert res.json['starting_amount_units'] == 'g'
def test_mixinProperties():
from snovault.schema_utils import load_schema
schema = load_schema('snowflakes:schemas/access_key.json')
assert schema['properties']['uuid']['type'] == 'string'
def test_load_schema(schema):
from snovault.schema_utils import load_schema
assert load_schema('snowflakes:schemas/%s' % schema)
def test_load_schema(schema, master_mixins, registry, pattern_fields, testapp):
abstract = [
'file.json', 'individual.json', 'quality_metric.json', 'note.json',
'workflow_run.json', 'user_content.json', 'evidence.json',
'higlass_view_config.json'
]
loaded_schema = load_schema('encoded:schemas/%s' % schema)
assert (loaded_schema)
typename = schema.replace('.json', '')
collection_names = [camel_case(typename), pluralize(typename)]
# see if there is a pattern field and if so test the regex (uses fixture)
schema_props = loaded_schema.get('properties')
for field, test_values in pattern_fields.items():
if schema_props and field in schema_props:
pattern = schema_props.get(field).get(
'pattern') # will sometimes be None if mixed in from elsewhere
if pattern is None:
continue
if not test_values:
continue
regex = re.compile(pattern)
good_vals = test_values.get('good', [])
bad_vals = test_values.get('bad', [])
assert all([regex.search(tv) for tv in good_vals if good_vals])
assert not any([regex.search(bv) for bv in bad_vals if bad_vals])
# check the mixin properties for each schema
if not schema == ('mixins.json'):
verify_mixins(loaded_schema, master_mixins)
if schema not in ['namespaces.json', 'mixins.json']:
# check that schema.id is same as /profiles/schema
idtag = loaded_schema['id']
idtag = idtag.replace('/profiles/', '')
# special case for access_key.json
if schema == 'access_key.json':
idtag = idtag.replace('_admin', '')
assert schema == idtag
# check for pluralized and camel cased in collection_names
val = None
for name in collection_names:
assert name in registry[COLLECTIONS]
if val is not None:
assert registry[COLLECTIONS][name] == val
else:
val = registry[COLLECTIONS][name]
if schema not in abstract:
# check schema w/o json extension is in registry[TYPES]
assert typename in registry[TYPES].by_item_type
assert typename in registry[COLLECTIONS]
assert registry[COLLECTIONS][typename] == val
shared_properties = [
'uuid', 'schema_version', 'aliases', 'date_created',
'submitted_by', 'last_modified', 'status'
]
no_alias_or_attribution = [
'user.json', 'project.json', 'institution.json',
'organism.json', 'page.json', 'static_section.json',
'badge.json', 'tracking_item.json', 'file_format.json',
'experiment_type.json'
]
for prop in shared_properties:
if schema == 'experiment.json':
# currently experiment is abstract and has no mixin properties
continue
if schema == 'access_key.json' and prop not in [
'uuid', 'schema_version'
]:
continue
if schema in no_alias_or_attribution and prop in [
'aliases', 'institution', 'project'
]:
continue
verify_property(loaded_schema, prop)
def test_load_schema(schema):
from snovault.schema_utils import load_schema
assert load_schema('encoded:schemas/%s' % schema)
def test_mixinProperties():
schema = load_schema('encoded:schemas/access_key.json')
assert schema['properties']['uuid']['type'] == 'string'
def test_mixinProperties():
from snovault.schema_utils import load_schema
schema = load_schema('snowflakes:schemas/access_key.json')
assert schema['properties']['uuid']['type'] == 'string'
def test_load_schema(schema):
from snovault.schema_utils import load_schema
assert load_schema('snowflakes:schemas/%s' % schema)
def test_load_schema(schema, master_mixins, registry):
from snovault import TYPES
from snovault import COLLECTIONS
abstract = [
'microscope_setting.json',
'experiment.json',
'file.json',
'individual.json',
'quality_metric.json',
'treatment.json',
'workflow_run.json',
'user_content.json'
]
loaded_schema = load_schema('encoded:schemas/%s' % schema)
assert(loaded_schema)
typename = schema.replace('.json', '')
collection_names = [camel_case(typename), pluralize(typename)]
# check the mixin properties for each schema
if not schema == ('mixins.json'):
verify_mixins(loaded_schema, master_mixins)
if schema not in ['namespaces.json', 'mixins.json']:
# check that schema.id is same as /profiles/schema
idtag = loaded_schema['id']
idtag = idtag.replace('/profiles/', '')
# special case for access_key.json
if schema == 'access_key.json':
idtag = idtag.replace('_admin', '')
assert schema == idtag
# check for pluralized and camel cased in collection_names
val = None
for name in collection_names:
assert name in registry[COLLECTIONS]
if val is not None:
assert registry[COLLECTIONS][name] == val
else:
val = registry[COLLECTIONS][name]
if schema not in abstract:
# check schema w/o json extension is in registry[TYPES]
assert typename in registry[TYPES].by_item_type
assert typename in registry[COLLECTIONS]
assert registry[COLLECTIONS][typename] == val
shared_properties = [
'uuid',
'schema_version',
'aliases',
'lab',
'award',
'date_created',
'submitted_by',
'last_modified',
'status'
]
no_alias_or_attribution = [
'user.json', 'award.json', 'lab.json', 'organism.json',
'ontology.json', 'ontology_term.json', 'sysinfo.json', 'page.json',
'static_section.json', 'badge.json', 'tracking_item.json',
'file_format.json', 'experiment_type.json', 'higlass_view_config.json'
]
for prop in shared_properties:
if schema == 'experiment.json':
# currently experiment is abstract and has no mixin properties
continue
if schema == 'access_key.json' and prop not in ['uuid', 'schema_version']:
continue
if schema in no_alias_or_attribution and prop in ['aliases', 'lab', 'award']:
continue
verify_property(loaded_schema, prop)