def _validate_metadata_errors(vals, meta, expected_key, expected_dev_message): mv = MetadataValidatorSet(vals) # with raises(Exception) as got: errors = mv.validate_metadata(meta, return_error_detail=True) assert len(errors) == 1 assert str(errors[0]['key']) == str(expected_key) assert str(errors[0]['dev_message']) == str(expected_dev_message)
def test_set_call_validator(): mv = MetadataValidatorSet([ MetadataValidator('key1', [lambda k, v: (k, v, 1), lambda k, v: (k, v, 2)]), MetadataValidator('key2', [lambda k, v: (k, v, 3)]) ]) assert mv.call_validator('key1', 0, {'foo', 'bar'}) == ('key1', {'foo', 'bar'}, 1) assert mv.call_validator('key1', 1, {'foo', 'bat'}) == ('key1', {'foo', 'bat'}, 2) assert mv.call_validator('key2', 0, {'foo', 'baz'}) == ('key2', {'foo', 'baz'}, 3)
def get_validators(url: str) -> MetadataValidatorSet: ''' Given a url pointing to a config file, initialize any metadata validators present in the configuration. :param url: The URL for a config file for the metadata validators. :returns: A set of metadata validators. ''' # TODO VALIDATOR make validator CLI try: with _urllib.request.urlopen(url) as res: cfg = _yaml.safe_load(res) except _URLError as e: raise ValueError( f'Failed to open validator configuration file at {url}: {str(e.reason)}' ) from e except _ParserError as e: raise ValueError( f'Failed to open validator configuration file at {url}: {str(e)}' ) from e _validate(instance=cfg, schema=_META_VAL_JSONSCHEMA) mvals = _get_validators( cfg.get('validators', {}), 'Metadata', lambda k, v, m: _MetadataValidator(k, v, metadata=m)) mvals.extend( _get_validators( cfg.get('prefix_validators', {}), 'Prefix metadata', lambda k, v, m: _MetadataValidator(k, prefix_validators=v, metadata=m))) return MetadataValidatorSet(mvals)
def test_set_with_prefix_validators_multiple_matches(): results = [] mv = MetadataValidatorSet([ MetadataValidator('somekey', [lambda k, v: results.append((k, v))]), MetadataValidator('somekeya', prefix_validators=[ lambda p, k, v: exec( 'raise ValueError("test failed somekeya")') ]), MetadataValidator( 'somekex', prefix_validators=[ lambda p, k, v: exec('raise ValueError("test failed somekex")') ]), MetadataValidator( 'somekey', prefix_validators=[lambda p, k, v: results.append((p, k, v))]), MetadataValidator( 'somekez', prefix_validators=[ lambda p, k, v: exec('raise ValueError("test failed somekez")') ]), MetadataValidator( 'someke', prefix_validators=[lambda p, k, v: results.append((p, k, v))]), MetadataValidator( 's', prefix_validators=[lambda p, k, v: results.append( (p, k, v))]), MetadataValidator( 't', prefix_validators=[ lambda p, k, v: exec('raise ValueError("test failed t")') ]), ]) md = {'somekey': {'x', 'y'}} mv.validate_metadata(md) print(results) assert results == [ ('somekey', {'x', 'y'}), ('s', 'somekey', {'x', 'y'}), ('someke', 'somekey', {'x', 'y'}), ('somekey', 'somekey', {'x', 'y'}), ]
def test_set_call_prefix_validator(): mv = MetadataValidatorSet([ MetadataValidator('p1', prefix_validators=[ lambda p, k, v: (p, k, v, 1), lambda p, k, v: (p, k, v, 2) ]), MetadataValidator('p2', prefix_validators=[lambda p, k, v: (p, k, v, 3)]) ]) assert mv.call_prefix_validator('p1', 0, 'key1', {'foo', 'bar'}) == ('p1', 'key1', {'foo', 'bar'}, 1) assert mv.call_prefix_validator('p1', 1, 'key11', {'foo', 'bat'}) == ('p1', 'key11', {'foo', 'bat'}, 2) assert mv.call_prefix_validator('p2', 0, 'key2', {'foo', 'baz'}) == ('p2', 'key2', {'foo', 'baz'}, 3)
def test_set_with_validators(): mv = MetadataValidatorSet([ # this is vile MetadataValidator('key1', [ lambda k, v: exec('assert k == "key1"'), lambda k, v: exec('assert v == {"a": "b"}') ]), MetadataValidator('key2', [lambda k, v: exec('assert k == "key2"')]) ]) md = {'key1': {'a': 'b'}, 'key2': {'foo': 'bar'}} mv.validate_metadata(md) mv.validate_metadata(maps.FrozenMap(md)) assert mv.keys() == ['key1', 'key2'] assert mv.prefix_keys() == [] assert mv.validator_count('key1') == 2 assert mv.validator_count('key2') == 1
def _prefix_key_metadata_fail_(vals, keys, expected): mv = MetadataValidatorSet(vals) with raises(Exception) as got: mv.prefix_key_metadata(keys) assert_exception_correct(got.value, expected) with raises(Exception) as got: mv.prefix_key_metadata(keys, exact_match=True) assert_exception_correct(got.value, expected)
def test_set_with_prefix_validators(): mv = MetadataValidatorSet([ # this is vile MetadataValidator('pre1', prefix_validators=[ lambda p, k, v: exec('assert p == "pre1"'), lambda p, k, v: exec('assert k == "pre1stuff"'), lambda p, k, v: exec('assert v == {"a": "b"}') ]), MetadataValidator( 'pre2', prefix_validators=[lambda p, k, v: exec('assert p == "pre2"')]) ]) md = {'pre1stuff': {'a': 'b'}, 'pre2thingy': {'foo': 'bar'}} mv.validate_metadata(md) mv.validate_metadata(maps.FrozenMap(md)) assert mv.keys() == [] assert mv.prefix_keys() == ['pre1', 'pre2'] assert mv.prefix_validator_count('pre1') == 3 assert mv.prefix_validator_count('pre2') == 1
def test_set_with_prefix_validators_and_standard_validator(): mv = MetadataValidatorSet([ # this is vile MetadataValidator('pre1', prefix_validators=[ lambda p, k, v: exec('assert p == "pre1"'), lambda p, k, v: exec('assert k == "pre1stuff"'), lambda p, k, v: exec('assert v == {"a": "b"}') ]), MetadataValidator( 'pre2', prefix_validators=[lambda p, k, v: exec('assert p == "pre2"')]), # test that non-prefix validator with same name is ok MetadataValidator('pre2', [lambda k, v: exec('raise ValueError()')]) ]) md = {'pre1stuff': {'a': 'b'}, 'pre2thingy': {'foo': 'bar'}} mv.validate_metadata(md) mv.validate_metadata(maps.FrozenMap(md)) assert mv.keys() == ['pre2'] assert mv.prefix_keys() == ['pre1', 'pre2'] assert mv.prefix_validator_count('pre1') == 3 assert mv.prefix_validator_count('pre2') == 1
def test_prefix_key_metadata_fail_prefix_match(): mv = MetadataValidatorSet([ MetadataValidator('abcdef', prefix_validators=[_noop], metadata={'f': 'g'}), MetadataValidator('abcdefhi', prefix_validators=[_noop], metadata={'f': 'g'}) ]) with raises(Exception) as got: mv.prefix_key_metadata(None, exact_match=False) assert_exception_correct(got.value, ValueError('keys cannot be None')) with raises(Exception) as got: mv.prefix_key_metadata(['abcde'], exact_match=False) assert_exception_correct( got.value, IllegalParameterError('No prefix metadata keys matching key abcde'))
def _validate_metadata_fail(vals, meta, expected): mv = MetadataValidatorSet(vals) with raises(Exception) as got: mv.validate_metadata(meta) assert_exception_correct(got.value, expected)
def _fail_construct_set(validators, expected): with raises(Exception) as got: MetadataValidatorSet(validators) assert_exception_correct(got.value, expected)
def _call_prefix_validator_fail(vals, prefix, index, key, expected): mv = MetadataValidatorSet(vals) with raises(Exception) as got: mv.call_prefix_validator(prefix, index, key, {}) assert_exception_correct(got.value, expected)
def test_set_with_key_metadata(): mv = MetadataValidatorSet([ MetadataValidator('pre1', prefix_validators=[_noop], metadata={ 'a': 'b', 'c': 'd' }), MetadataValidator('pre2', prefix_validators=[_noop]), MetadataValidator('pre3', prefix_validators=[_noop], metadata={'c': 'd'}), MetadataValidator('pre1', [_noop]), MetadataValidator('pre2', [_noop], metadata={ 'e': 'f', 'h': 'i' }), MetadataValidator('pre3', [_noop], metadata={'h': 'i'}) ]) assert mv.key_metadata([]) == {} assert mv.prefix_key_metadata([]) == {} assert mv.key_metadata(['pre1']) == {'pre1': {}} assert mv.prefix_key_metadata(['pre1'], exact_match=True) == { 'pre1': { 'a': 'b', 'c': 'd' } } assert mv.key_metadata(['pre1', 'pre3']) == { 'pre1': {}, 'pre3': { 'h': 'i' } } assert mv.prefix_key_metadata(['pre1', 'pre2']) == { 'pre1': { 'a': 'b', 'c': 'd' }, 'pre2': {} } assert mv.key_metadata(['pre1', 'pre2', 'pre3']) == { 'pre1': {}, 'pre2': { 'e': 'f', 'h': 'i' }, 'pre3': { 'h': 'i' } } assert mv.prefix_key_metadata(['pre1', 'pre2', 'pre3']) == { 'pre1': { 'a': 'b', 'c': 'd' }, 'pre2': {}, 'pre3': { 'c': 'd' } }
def _prefix_validator_count_fail(vals, prefix, expected): mv = MetadataValidatorSet(vals) with raises(Exception) as got: mv.prefix_validator_count(prefix) assert_exception_correct(got.value, expected)
def _validator_count_fail(vals, key, expected): mv = MetadataValidatorSet(vals) with raises(Exception) as got: mv.validator_count(key) assert_exception_correct(got.value, expected)
def test_set_with_prefix_match_key_metadata(): mv = MetadataValidatorSet([ MetadataValidator('a', prefix_validators=[_noop], metadata={'a': 'b'}), MetadataValidator('abc', prefix_validators=[_noop], metadata={'c': 'd'}), MetadataValidator('abcdef', prefix_validators=[_noop], metadata={'f': 'g'}), MetadataValidator('abcdefhi', prefix_validators=[_noop], metadata={'f': 'g'}), MetadataValidator('abzhi', prefix_validators=[_noop], metadata={'z': 'w'}), MetadataValidator('abzhijk', prefix_validators=[_noop], metadata={'q': 'q'}), MetadataValidator('b', prefix_validators=[_noop], metadata={'bbb': 'bbb'}), ]) assert mv.prefix_key_metadata(['abcdef']) == {'abcdef': {'f': 'g'}} assert mv.prefix_key_metadata(['abcdef'], exact_match=False) == { 'a': { 'a': 'b' }, 'abc': { 'c': 'd' }, 'abcdef': { 'f': 'g' } } assert mv.prefix_key_metadata(['abcdefh'], exact_match=False) == { 'a': { 'a': 'b' }, 'abc': { 'c': 'd' }, 'abcdef': { 'f': 'g' } } assert mv.prefix_key_metadata(['abzhij'], exact_match=False) == { 'a': { 'a': 'b' }, 'abzhi': { 'z': 'w' } } assert mv.prefix_key_metadata(['abcdef', 'abzhij'], exact_match=False) == { 'a': { 'a': 'b' }, 'abc': { 'c': 'd' }, 'abcdef': { 'f': 'g' }, 'abzhi': { 'z': 'w' } }
def _key_metadata_fail_(vals, keys, expected): mv = MetadataValidatorSet(vals) with raises(Exception) as got: mv.key_metadata(keys) assert_exception_correct(got.value, expected)
def test_empty_set(): mv = MetadataValidatorSet() assert mv.keys() == [] assert mv.prefix_keys() == []
def build_samples( config: Dict[str, str]) -> Tuple[Samples, KBaseUserLookup, List[str]]: ''' Build the sample service instance from the SDK server provided parameters. :param cfg: The SDK generated configuration. :returns: A samples instance. ''' if not config: raise ValueError('config is empty, cannot start service') arango_url = _check_string_req(config.get('arango-url'), 'config param arango-url') arango_db = _check_string_req(config.get('arango-db'), 'config param arango-db') arango_user = _check_string_req(config.get('arango-user'), 'config param arango-user') arango_pwd = _check_string_req(config.get('arango-pwd'), 'config param arango-pwd') col_sample = _check_string_req(config.get('sample-collection'), 'config param sample-collection') col_version = _check_string_req(config.get('version-collection'), 'config param version-collection') col_ver_edge = _check_string_req(config.get('version-edge-collection'), 'config param version-edge-collection') col_node = _check_string_req(config.get('node-collection'), 'config param node-collection') col_node_edge = _check_string_req(config.get('node-edge-collection'), 'config param node-edge-collection') col_data_link = _check_string_req(config.get('data-link-collection'), 'config param data-link-collection') col_ws_obj_ver = _check_string_req( config.get('workspace-object-version-shadow-collection'), 'config param workspace-object-version-shadow-collection') col_schema = _check_string_req(config.get('schema-collection'), 'config param schema-collection') auth_root_url = _check_string_req(config.get('auth-root-url'), 'config param auth-root-url') auth_token = _check_string_req(config.get('auth-token'), 'config param auth-token') full_roles = split_value(config, 'auth-full-admin-roles') read_roles = split_value(config, 'auth-read-admin-roles') read_exempt_roles = split_value(config, 'auth-read-exempt-roles') ws_url = _check_string_req(config.get('workspace-url'), 'config param workspace-url') ws_token = _check_string_req(config.get('workspace-read-admin-token'), 'config param workspace-read-admin-token') kafka_servers = _check_string(config.get('kafka-bootstrap-servers'), 'config param kafka-bootstrap-servers', optional=True) kafka_topic = None if kafka_servers: # have to start the server twice to test no kafka scenario kafka_topic = _check_string(config.get('kafka-topic'), 'config param kafka-topic') metaval_url = _check_string(config.get('metadata-validator-config-url'), 'config param metadata-validator-config-url', optional=True) # meta params may have info that shouldn't be logged so don't log any for now. # Add code to deal with this later if needed print(f''' Starting server with config: arango-url: {arango_url} arango-db: {arango_db} arango-user: {arango_user} arango-pwd: [REDACTED FOR YOUR SAFETY AND COMFORT] sample-collection: {col_sample} version-collection: {col_version} version-edge-collection: {col_ver_edge} node-collection: {col_node} node-edge-collection: {col_node_edge} data-link-collection: {col_data_link} workspace-object-version-shadow-collection: {col_ws_obj_ver} schema-collection: {col_schema} auth-root-url: {auth_root_url} auth-token: [REDACTED FOR YOUR CONVENIENCE AND ENJOYMENT] auth-full-admin-roles: {', '.join(full_roles)} auth-read-admin-roles: {', '.join(read_roles)} auth-read-exempt-roles: {', '.join(read_exempt_roles)} workspace-url: {ws_url} workspace-read-admin-token: [REDACTED FOR YOUR ULTIMATE PLEASURE] kafka-bootstrap-servers: {kafka_servers} kafka-topic: {kafka_topic} metadata-validators-config-url: {metaval_url} ''') # build the validators before trying to connect to arango metaval = get_validators( metaval_url) if metaval_url else MetadataValidatorSet() arangoclient = _arango.ArangoClient(hosts=arango_url) arango_db = arangoclient.db(arango_db, username=arango_user, password=arango_pwd, verify=True) storage = _ArangoSampleStorage( arango_db, col_sample, col_version, col_ver_edge, col_node, col_node_edge, col_ws_obj_ver, col_data_link, col_schema, ) storage.start_consistency_checker() kafka = _KafkaNotifer(kafka_servers, _cast( str, kafka_topic)) if kafka_servers else None user_lookup = KBaseUserLookup(auth_root_url, auth_token, full_roles, read_roles) ws = _WS(_Workspace(ws_url, token=ws_token)) return Samples(storage, user_lookup, metaval, ws, kafka), user_lookup, read_exempt_roles
def _validate_metadata_errors(vals, meta, expected): mv = MetadataValidatorSet(vals) # with raises(Exception) as got: errors = mv.validate_metadata(meta, return_error_strings=True) assert len(errors) == 1 assert str(errors[0]) == str(expected)