def _validate_metadata_errors(vals, meta, expected_key, expected_dev_message):
mv = MetadataValidatorSet(vals)
# with raises(Exception) as got:
errors = mv.validate_metadata(meta, return_error_detail=True)
assert len(errors) == 1
assert str(errors[0]['key']) == str(expected_key)
assert str(errors[0]['dev_message']) == str(expected_dev_message)
def test_set_call_validator():
mv = MetadataValidatorSet([
MetadataValidator('key1',
[lambda k, v: (k, v, 1), lambda k, v: (k, v, 2)]),
MetadataValidator('key2', [lambda k, v: (k, v, 3)])
])
assert mv.call_validator('key1', 0,
{'foo', 'bar'}) == ('key1', {'foo', 'bar'}, 1)
assert mv.call_validator('key1', 1,
{'foo', 'bat'}) == ('key1', {'foo', 'bat'}, 2)
assert mv.call_validator('key2', 0,
{'foo', 'baz'}) == ('key2', {'foo', 'baz'}, 3)
def get_validators(url: str) -> MetadataValidatorSet:
'''
Given a url pointing to a config file, initialize any metadata validators present
in the configuration.
:param url: The URL for a config file for the metadata validators.
:returns: A set of metadata validators.
'''
# TODO VALIDATOR make validator CLI
try:
with _urllib.request.urlopen(url) as res:
cfg = _yaml.safe_load(res)
except _URLError as e:
raise ValueError(
f'Failed to open validator configuration file at {url}: {str(e.reason)}'
) from e
except _ParserError as e:
raise ValueError(
f'Failed to open validator configuration file at {url}: {str(e)}'
) from e
_validate(instance=cfg, schema=_META_VAL_JSONSCHEMA)
mvals = _get_validators(
cfg.get('validators', {}), 'Metadata',
lambda k, v, m: _MetadataValidator(k, v, metadata=m))
mvals.extend(
_get_validators(
cfg.get('prefix_validators', {}), 'Prefix metadata', lambda k, v,
m: _MetadataValidator(k, prefix_validators=v, metadata=m)))
return MetadataValidatorSet(mvals)
def test_set_with_prefix_validators_multiple_matches():
results = []
mv = MetadataValidatorSet([
MetadataValidator('somekey', [lambda k, v: results.append((k, v))]),
MetadataValidator('somekeya',
prefix_validators=[
lambda p, k, v: exec(
'raise ValueError("test failed somekeya")')
]),
MetadataValidator(
'somekex',
prefix_validators=[
lambda p, k, v: exec('raise ValueError("test failed somekex")')
]),
MetadataValidator(
'somekey',
prefix_validators=[lambda p, k, v: results.append((p, k, v))]),
MetadataValidator(
'somekez',
prefix_validators=[
lambda p, k, v: exec('raise ValueError("test failed somekez")')
]),
MetadataValidator(
'someke',
prefix_validators=[lambda p, k, v: results.append((p, k, v))]),
MetadataValidator(
's', prefix_validators=[lambda p, k, v: results.append(
(p, k, v))]),
MetadataValidator(
't',
prefix_validators=[
lambda p, k, v: exec('raise ValueError("test failed t")')
]),
])
md = {'somekey': {'x', 'y'}}
mv.validate_metadata(md)
print(results)
assert results == [
('somekey', {'x', 'y'}),
('s', 'somekey', {'x', 'y'}),
('someke', 'somekey', {'x', 'y'}),
('somekey', 'somekey', {'x', 'y'}),
]
def test_set_call_prefix_validator():
mv = MetadataValidatorSet([
MetadataValidator('p1',
prefix_validators=[
lambda p, k, v: (p, k, v, 1), lambda p, k, v:
(p, k, v, 2)
]),
MetadataValidator('p2',
prefix_validators=[lambda p, k, v: (p, k, v, 3)])
])
assert mv.call_prefix_validator('p1', 0, 'key1',
{'foo', 'bar'}) == ('p1', 'key1',
{'foo', 'bar'}, 1)
assert mv.call_prefix_validator('p1', 1, 'key11',
{'foo', 'bat'}) == ('p1', 'key11',
{'foo', 'bat'}, 2)
assert mv.call_prefix_validator('p2', 0, 'key2',
{'foo', 'baz'}) == ('p2', 'key2',
{'foo', 'baz'}, 3)
def test_set_with_validators():
mv = MetadataValidatorSet([
# this is vile
MetadataValidator('key1', [
lambda k, v: exec('assert k == "key1"'),
lambda k, v: exec('assert v == {"a": "b"}')
]),
MetadataValidator('key2', [lambda k, v: exec('assert k == "key2"')])
])
md = {'key1': {'a': 'b'}, 'key2': {'foo': 'bar'}}
mv.validate_metadata(md)
mv.validate_metadata(maps.FrozenMap(md))
assert mv.keys() == ['key1', 'key2']
assert mv.prefix_keys() == []
assert mv.validator_count('key1') == 2
assert mv.validator_count('key2') == 1
def _prefix_key_metadata_fail_(vals, keys, expected):
mv = MetadataValidatorSet(vals)
with raises(Exception) as got:
mv.prefix_key_metadata(keys)
assert_exception_correct(got.value, expected)
with raises(Exception) as got:
mv.prefix_key_metadata(keys, exact_match=True)
assert_exception_correct(got.value, expected)
def test_set_with_prefix_validators():
mv = MetadataValidatorSet([
# this is vile
MetadataValidator('pre1',
prefix_validators=[
lambda p, k, v: exec('assert p == "pre1"'),
lambda p, k, v: exec('assert k == "pre1stuff"'),
lambda p, k, v: exec('assert v == {"a": "b"}')
]),
MetadataValidator(
'pre2',
prefix_validators=[lambda p, k, v: exec('assert p == "pre2"')])
])
md = {'pre1stuff': {'a': 'b'}, 'pre2thingy': {'foo': 'bar'}}
mv.validate_metadata(md)
mv.validate_metadata(maps.FrozenMap(md))
assert mv.keys() == []
assert mv.prefix_keys() == ['pre1', 'pre2']
assert mv.prefix_validator_count('pre1') == 3
assert mv.prefix_validator_count('pre2') == 1
def test_set_with_prefix_validators_and_standard_validator():
mv = MetadataValidatorSet([
# this is vile
MetadataValidator('pre1',
prefix_validators=[
lambda p, k, v: exec('assert p == "pre1"'),
lambda p, k, v: exec('assert k == "pre1stuff"'),
lambda p, k, v: exec('assert v == {"a": "b"}')
]),
MetadataValidator(
'pre2',
prefix_validators=[lambda p, k, v: exec('assert p == "pre2"')]),
# test that non-prefix validator with same name is ok
MetadataValidator('pre2', [lambda k, v: exec('raise ValueError()')])
])
md = {'pre1stuff': {'a': 'b'}, 'pre2thingy': {'foo': 'bar'}}
mv.validate_metadata(md)
mv.validate_metadata(maps.FrozenMap(md))
assert mv.keys() == ['pre2']
assert mv.prefix_keys() == ['pre1', 'pre2']
assert mv.prefix_validator_count('pre1') == 3
assert mv.prefix_validator_count('pre2') == 1
def test_prefix_key_metadata_fail_prefix_match():
mv = MetadataValidatorSet([
MetadataValidator('abcdef',
prefix_validators=[_noop],
metadata={'f': 'g'}),
MetadataValidator('abcdefhi',
prefix_validators=[_noop],
metadata={'f': 'g'})
])
with raises(Exception) as got:
mv.prefix_key_metadata(None, exact_match=False)
assert_exception_correct(got.value, ValueError('keys cannot be None'))
with raises(Exception) as got:
mv.prefix_key_metadata(['abcde'], exact_match=False)
assert_exception_correct(
got.value,
IllegalParameterError('No prefix metadata keys matching key abcde'))
def _validate_metadata_fail(vals, meta, expected):
mv = MetadataValidatorSet(vals)
with raises(Exception) as got:
mv.validate_metadata(meta)
assert_exception_correct(got.value, expected)
def _fail_construct_set(validators, expected):
with raises(Exception) as got:
MetadataValidatorSet(validators)
assert_exception_correct(got.value, expected)
def _call_prefix_validator_fail(vals, prefix, index, key, expected):
mv = MetadataValidatorSet(vals)
with raises(Exception) as got:
mv.call_prefix_validator(prefix, index, key, {})
assert_exception_correct(got.value, expected)
def test_set_with_key_metadata():
mv = MetadataValidatorSet([
MetadataValidator('pre1',
prefix_validators=[_noop],
metadata={
'a': 'b',
'c': 'd'
}),
MetadataValidator('pre2', prefix_validators=[_noop]),
MetadataValidator('pre3',
prefix_validators=[_noop],
metadata={'c': 'd'}),
MetadataValidator('pre1', [_noop]),
MetadataValidator('pre2', [_noop], metadata={
'e': 'f',
'h': 'i'
}),
MetadataValidator('pre3', [_noop], metadata={'h': 'i'})
])
assert mv.key_metadata([]) == {}
assert mv.prefix_key_metadata([]) == {}
assert mv.key_metadata(['pre1']) == {'pre1': {}}
assert mv.prefix_key_metadata(['pre1'], exact_match=True) == {
'pre1': {
'a': 'b',
'c': 'd'
}
}
assert mv.key_metadata(['pre1', 'pre3']) == {
'pre1': {},
'pre3': {
'h': 'i'
}
}
assert mv.prefix_key_metadata(['pre1', 'pre2']) == {
'pre1': {
'a': 'b',
'c': 'd'
},
'pre2': {}
}
assert mv.key_metadata(['pre1', 'pre2', 'pre3']) == {
'pre1': {},
'pre2': {
'e': 'f',
'h': 'i'
},
'pre3': {
'h': 'i'
}
}
assert mv.prefix_key_metadata(['pre1', 'pre2', 'pre3']) == {
'pre1': {
'a': 'b',
'c': 'd'
},
'pre2': {},
'pre3': {
'c': 'd'
}
}
def _prefix_validator_count_fail(vals, prefix, expected):
mv = MetadataValidatorSet(vals)
with raises(Exception) as got:
mv.prefix_validator_count(prefix)
assert_exception_correct(got.value, expected)
def _validator_count_fail(vals, key, expected):
mv = MetadataValidatorSet(vals)
with raises(Exception) as got:
mv.validator_count(key)
assert_exception_correct(got.value, expected)
def test_set_with_prefix_match_key_metadata():
mv = MetadataValidatorSet([
MetadataValidator('a', prefix_validators=[_noop], metadata={'a': 'b'}),
MetadataValidator('abc',
prefix_validators=[_noop],
metadata={'c': 'd'}),
MetadataValidator('abcdef',
prefix_validators=[_noop],
metadata={'f': 'g'}),
MetadataValidator('abcdefhi',
prefix_validators=[_noop],
metadata={'f': 'g'}),
MetadataValidator('abzhi',
prefix_validators=[_noop],
metadata={'z': 'w'}),
MetadataValidator('abzhijk',
prefix_validators=[_noop],
metadata={'q': 'q'}),
MetadataValidator('b',
prefix_validators=[_noop],
metadata={'bbb': 'bbb'}),
])
assert mv.prefix_key_metadata(['abcdef']) == {'abcdef': {'f': 'g'}}
assert mv.prefix_key_metadata(['abcdef'], exact_match=False) == {
'a': {
'a': 'b'
},
'abc': {
'c': 'd'
},
'abcdef': {
'f': 'g'
}
}
assert mv.prefix_key_metadata(['abcdefh'], exact_match=False) == {
'a': {
'a': 'b'
},
'abc': {
'c': 'd'
},
'abcdef': {
'f': 'g'
}
}
assert mv.prefix_key_metadata(['abzhij'], exact_match=False) == {
'a': {
'a': 'b'
},
'abzhi': {
'z': 'w'
}
}
assert mv.prefix_key_metadata(['abcdef', 'abzhij'], exact_match=False) == {
'a': {
'a': 'b'
},
'abc': {
'c': 'd'
},
'abcdef': {
'f': 'g'
},
'abzhi': {
'z': 'w'
}
}
def _key_metadata_fail_(vals, keys, expected):
mv = MetadataValidatorSet(vals)
with raises(Exception) as got:
mv.key_metadata(keys)
assert_exception_correct(got.value, expected)
def test_empty_set():
mv = MetadataValidatorSet()
assert mv.keys() == []
assert mv.prefix_keys() == []
def build_samples(
config: Dict[str, str]) -> Tuple[Samples, KBaseUserLookup, List[str]]:
'''
Build the sample service instance from the SDK server provided parameters.
:param cfg: The SDK generated configuration.
:returns: A samples instance.
'''
if not config:
raise ValueError('config is empty, cannot start service')
arango_url = _check_string_req(config.get('arango-url'),
'config param arango-url')
arango_db = _check_string_req(config.get('arango-db'),
'config param arango-db')
arango_user = _check_string_req(config.get('arango-user'),
'config param arango-user')
arango_pwd = _check_string_req(config.get('arango-pwd'),
'config param arango-pwd')
col_sample = _check_string_req(config.get('sample-collection'),
'config param sample-collection')
col_version = _check_string_req(config.get('version-collection'),
'config param version-collection')
col_ver_edge = _check_string_req(config.get('version-edge-collection'),
'config param version-edge-collection')
col_node = _check_string_req(config.get('node-collection'),
'config param node-collection')
col_node_edge = _check_string_req(config.get('node-edge-collection'),
'config param node-edge-collection')
col_data_link = _check_string_req(config.get('data-link-collection'),
'config param data-link-collection')
col_ws_obj_ver = _check_string_req(
config.get('workspace-object-version-shadow-collection'),
'config param workspace-object-version-shadow-collection')
col_schema = _check_string_req(config.get('schema-collection'),
'config param schema-collection')
auth_root_url = _check_string_req(config.get('auth-root-url'),
'config param auth-root-url')
auth_token = _check_string_req(config.get('auth-token'),
'config param auth-token')
full_roles = split_value(config, 'auth-full-admin-roles')
read_roles = split_value(config, 'auth-read-admin-roles')
read_exempt_roles = split_value(config, 'auth-read-exempt-roles')
ws_url = _check_string_req(config.get('workspace-url'),
'config param workspace-url')
ws_token = _check_string_req(config.get('workspace-read-admin-token'),
'config param workspace-read-admin-token')
kafka_servers = _check_string(config.get('kafka-bootstrap-servers'),
'config param kafka-bootstrap-servers',
optional=True)
kafka_topic = None
if kafka_servers: # have to start the server twice to test no kafka scenario
kafka_topic = _check_string(config.get('kafka-topic'),
'config param kafka-topic')
metaval_url = _check_string(config.get('metadata-validator-config-url'),
'config param metadata-validator-config-url',
optional=True)
# meta params may have info that shouldn't be logged so don't log any for now.
# Add code to deal with this later if needed
print(f'''
Starting server with config:
arango-url: {arango_url}
arango-db: {arango_db}
arango-user: {arango_user}
arango-pwd: [REDACTED FOR YOUR SAFETY AND COMFORT]
sample-collection: {col_sample}
version-collection: {col_version}
version-edge-collection: {col_ver_edge}
node-collection: {col_node}
node-edge-collection: {col_node_edge}
data-link-collection: {col_data_link}
workspace-object-version-shadow-collection: {col_ws_obj_ver}
schema-collection: {col_schema}
auth-root-url: {auth_root_url}
auth-token: [REDACTED FOR YOUR CONVENIENCE AND ENJOYMENT]
auth-full-admin-roles: {', '.join(full_roles)}
auth-read-admin-roles: {', '.join(read_roles)}
auth-read-exempt-roles: {', '.join(read_exempt_roles)}
workspace-url: {ws_url}
workspace-read-admin-token: [REDACTED FOR YOUR ULTIMATE PLEASURE]
kafka-bootstrap-servers: {kafka_servers}
kafka-topic: {kafka_topic}
metadata-validators-config-url: {metaval_url}
''')
# build the validators before trying to connect to arango
metaval = get_validators(
metaval_url) if metaval_url else MetadataValidatorSet()
arangoclient = _arango.ArangoClient(hosts=arango_url)
arango_db = arangoclient.db(arango_db,
username=arango_user,
password=arango_pwd,
verify=True)
storage = _ArangoSampleStorage(
arango_db,
col_sample,
col_version,
col_ver_edge,
col_node,
col_node_edge,
col_ws_obj_ver,
col_data_link,
col_schema,
)
storage.start_consistency_checker()
kafka = _KafkaNotifer(kafka_servers, _cast(
str, kafka_topic)) if kafka_servers else None
user_lookup = KBaseUserLookup(auth_root_url, auth_token, full_roles,
read_roles)
ws = _WS(_Workspace(ws_url, token=ws_token))
return Samples(storage, user_lookup, metaval, ws,
kafka), user_lookup, read_exempt_roles
def _validate_metadata_errors(vals, meta, expected):
mv = MetadataValidatorSet(vals)
# with raises(Exception) as got:
errors = mv.validate_metadata(meta, return_error_strings=True)
assert len(errors) == 1
assert str(errors[0]) == str(expected)