def _to_gob_value(entity, field, spec, resolve_secure=False):
"""
Transforms a entity field value into a GOB type value
Attention:
Resolve secure is normally False as this is all handled by the Authority classes
For enhanced views however, this is not possible.
These queries are based upon a view and cannot directly be related to a GOB Model
If the field names of the view are properly named the decryption will be handled here
:param entity:
:param field:
:param spec:
:param resolve_secure:
:return:
"""
entity_value = getattr(entity, field, None)
if isinstance(spec, dict):
gob_type = get_gob_type_from_info(spec)
if resolve_secure and Authority.is_secure_type(spec):
# Transform the value into a secure type value
secure_type = Authority.get_secure_type(gob_type, spec,
entity_value)
# Return decrypted value
return Authority.get_secured_value(secure_type)
return gob_type.from_value(entity_value, **spec)
else:
gob_type = get_gob_type_from_sql_type(spec)
return gob_type.from_value(entity_value)
def test_secured_value(self, mock_user, mock_request):
authority = Authority('cat', 'col')
mock_user.return_value = "any user"
mock_secure_type = mock.MagicMock()
result = authority.get_secured_value(mock_secure_type)
mock_user.assert_called_with(mock_request)
mock_secure_type.get_value.assert_called_with("any user")
def _autorized_order(order, catalog_name, collection_name):
"""
Filter the order (list of columns) on columns that are not suppressed given the current request
"""
authority = Authority(catalog_name, collection_name)
suppress_columns = authority.get_suppressed_columns()
return [o for o in order if o not in suppress_columns]
def _dump_entities_to_table(self, entities, model):
authority = Authority(self.catalog_name, self.collection_name)
suppress_columns = authority.get_suppressed_columns()
connection = self.datastore.connection
stream = CSVStream(csv_entities(entities, model, suppress_columns), STREAM_PER)
with connection.cursor() as cursor:
yield "Export data"
commit = COMMIT_PER
while stream.has_items():
stream.reset_count()
cursor.copy_expert(
sql=f"COPY {self.schema}.{self.tmp_collection_name} FROM STDIN DELIMITER ';' CSV HEADER;",
file=stream,
size=BUFFER_PER
)
if stream.total_count >= commit:
connection.commit()
commit += COMMIT_PER
yield f"\n{self.collection_name}: {stream.total_count:,}"
else:
# Let client know we're still working.
yield "."
yield f"\nExported {stream.total_count} rows\n"
connection.commit()
def test_exposed_value(self):
mock_type = MagicMock()
mock_type.get_value = lambda user: '******'
mock_type.from_value_secure = lambda value, spec: mock_type
info = {'gob_type': mock_type, 'spec': 'any spec'}
value = Authority.exposed_value('any value', info)
self.assertEqual(value, 'protected value')
value = Authority.exposed_value(None, info)
self.assertEqual(value, None)
def test_is_secured(self):
testcases = [
('any catalog', 'any collection', True),
('secure catalog', 'any collection', True),
('any catalog', 'some other collection', False),
('open catalog', 'collection', False),
]
for cat, coll, result in testcases:
authority = Authority(cat, coll)
self.assertEqual(result, authority.is_secured())
def test_get_secured_columns(self, mock_model):
mock_model.return_value.get_collection.return_value = {
'fields': {
'secure column': 'any spec'
}
}
authority = Authority('secure catalog', 'any col')
secure_columns = authority.get_secured_columns()
self.assertEqual(
secure_columns, {
'secure column': {
'gob_type': gob_secure_types.SecureString,
'spec': 'any spec'
}
})
def _allows_access(rule, *args, **kwargs):
"""
Check access to paths with variable catalog/collection names
"""
catalog_name = kwargs.get('catalog_name')
collection_name = kwargs.get('collection_name')
return Authority(catalog_name, collection_name).allows_access()
def get_curl(catalog_name, collection_name, path):
"""
Get the curl statement to retrieve the dataset in the Amsterdam Schema format
:param catalog_name:
:param collection_name:
:return:
"""
auth_header = None
# Make request to secure API if secured colllection
if Authority(args.catalog, args.collection).is_secured():
if '/gob/secure' not in path:
path = path.replace('/gob', '/gob/secure')
auth_header = os.getenv('GOB_API_AUTH_HEADER')
if not auth_header:
print_yellow(
"Generating a curl request for a secure endpoint, but missing Auth header env variable\n"
"Use login_keycloak.py to generate the Auth header to use with the secure endpoint.\n"
"Generated curl request contains a placeholder for now\n")
auth_header = 'AUTH_HEADER_PLACEHOLDER'
query = get_graphql_query(catalog_name, collection_name).replace('\n', '')
query = '{"query":"%s"}' % query
header = 'Content-Type: application/x-ndjson'
auth = f"--header 'Authorization: {auth_header}' " if auth_header else ""
url = get_url(catalog_name, collection_name, path)
return f"curl -s --location --request POST '{url}' --header '{header}' {auth}--data-raw '{query}'"
def sql(self):
self._reset()
# Relation without parent is main relation
base_collection = [
k for k, v in self.relation_parents.items() if v is None
][0]
self._collect_relation_info(base_collection, base_collection)
base_info = self._get_relation_info(base_collection)
select_fields = [
self._select_expression(base_info, field)
for field in [FIELD.GOBID] +
self.selects[base_collection]['fields']
]
self.select_expressions.extend(select_fields)
# Add catalog and collection to allow for value resolution
self.select_expressions.extend([
f"'{base_info['catalog_name']}' AS {CATALOG_NAME}",
f"'{base_info['collection_name']}' AS {COLLECTION_NAME}",
])
authority = Authority(base_info['catalog_name'],
base_info['collection_name'])
if not authority.allows_access():
raise NoAccessException
arguments = self._get_arguments_with_defaults(
self.selects[base_collection]['arguments'])
self.joins.append(
self._build_from_table(arguments, base_info['tablename'],
base_info['alias']))
del self.selects[base_collection]
self._join_relations(self.selects)
select = ',\n'.join(self.select_expressions)
table_select = '\n'.join(self.joins)
order_by = f"ORDER BY {base_info['alias']}.{FIELD.GOBID}"
query = f"SELECT\n{select}\n{table_select}\n{order_by}"
return query
def test_get_secure_type(self):
mock_gob_type = mock.MagicMock()
mock_gob_type.from_value_secure.return_value = "secure GOB type"
result = Authority.get_secure_type(mock_gob_type, 'any spec',
'any value')
self.assertEqual(result, "secure GOB type")
mock_gob_type.from_value_secure.assert_called_with(
'any value', 'any spec')
def test_filter_row(self):
authority = Authority('cat', 'col')
authority.get_suppressed_columns = lambda: ['b', 'd']
row = {'a': 1, 'b': 2, 'c': 3}
authority.filter_row(row)
self.assertEqual(row, {'a': 1, 'b': None, 'c': 3})
authority.allows_access = lambda: False
row = {'a': 1, 'b': 2, 'c': 3}
authority.filter_row(row)
self.assertEqual(row, {'a': None, 'b': None, 'c': None})
def test_is_secure_type(self):
authority = Authority('secure catalog', 'any col')
spec = {"type": "GOB.String"}
self.assertFalse(authority.is_secure_type(spec))
spec = {"type": "GOB.SecureString"}
self.assertTrue(authority.is_secure_type(spec))
spec = {
"type": "GOB.JSON",
"attributes": {
"attr": {
"type": "GOB.String"
}
}
}
self.assertFalse(authority.is_secure_type(spec))
spec = {
"type": "GOB.JSON",
"attributes": {
"attr": {
"type": "GOB.String"
},
"attr": {
"type": "GOB.SecureString"
}
}
}
self.assertTrue(authority.is_secure_type(spec))
def test_get_secured_json_columns(self, mock_model):
mock_model.return_value.get_collection.return_value = {
'fields': {
'secure column': {
'type': 'GOB.JSON',
'attributes': {
'attr1': {
'type': 'GOB.SecureString'
},
'attr2': {
'type': 'GOB.String'
}
}
}
}
}
authority = Authority('secure catalog', 'any col')
secure_columns = authority.get_secured_columns()
expect = {
'secure column': {
'gob_type': gob_types.JSON,
'spec': {
'type': 'GOB.JSON',
'gob_type': gob_types.JSON,
'attributes': {
'attr1': {
'type': 'GOB.SecureString',
'gob_type': gob_secure_types.SecureString
},
'attr2': {
'type': 'GOB.String',
'gob_type': gob_types.String
}
},
}
}
}
print(secure_columns)
self.assertEqual(secure_columns, expect)
def test_handle_secured_columns(self):
authority = Authority('secure catalog', 'any col')
authority.get_secured_columns = lambda: {'col': 'any info'}
authority.exposed_value = lambda value, info: 'exposed value'
row = {}
authority._handle_secured_columns({}, row)
self.assertEqual(row, {})
row = {'col': 'any value', 'any other col': 'any value'}
authority._handle_secured_columns({}, row)
self.assertEqual(row, {
'col': 'exposed value',
'any other col': 'any value'
})
row = {'mapped col': 'any value', 'any other col': 'any value'}
mapping = {'col': 'mapped col'}
authority._handle_secured_columns(mapping, row)
self.assertEqual(row, {
'mapped col': 'exposed value',
'any other col': 'any value'
})
def __init__(self, catalog_name: str, collection_name: str, config: dict):
self.catalog_name = catalog_name
self.collection_name = collection_name
self.tmp_collection_name = f"tmp_{collection_name}"
self.datastore = DatastoreFactory.get_datastore(config['db'])
self.datastore.connect()
self.schema = self._get_dst_schema(config, catalog_name, collection_name)
_, self.model = get_table_and_model(catalog_name, collection_name)
# Set attributes for create_table
self.model['catalog'] = catalog_name
self.model['collection'] = collection_name
self.suppressed_columns = Authority(self.catalog_name, self.collection_name).get_suppressed_columns()
def resolve_row(self, row, result):
"""
Resolve all values in the row
Update the resolved value in the result
:param row:
:param result:
:return:
"""
catalog_name = row.get(CATALOG_NAME)
collection_name = row.get(COLLECTION_NAME)
self._init_catalog_collection(catalog_name, collection_name)
# Filter row and result for columns that do not match with the roles of the current request
authority = Authority(catalog_name, collection_name)
authority.filter_row(row, mapping=self._attributes[catalog_name][collection_name])
authority.filter_row(result, mapping=self._attributes[catalog_name][collection_name])
for attr in [name for name in [CATALOG_NAME, COLLECTION_NAME] if name in row]:
# Once a row has been resolved, don't resolve it twice
del row[attr]
def test_allows_access(self):
authority = Authority('secure catalog', 'any col')
authority.get_roles = lambda: []
self.assertFalse(authority.allows_access())
authority.get_roles = lambda: [role_b]
self.assertTrue(authority.allows_access())
authority._catalog = "secure catalog collection"
authority._collection = "secure collection"
authority.get_roles = lambda: []
self.assertFalse(authority.allows_access())
authority.get_roles = lambda: [role_b]
self.assertTrue(authority.allows_access())
authority._collection = "any collection"
authority.get_roles = lambda: []
self.assertTrue(authority.allows_access())
def test_create(self):
authority = Authority('cat', 'col')
self.assertEqual(authority._catalog, 'cat')
self.assertEqual(authority._collection, 'col')
self.assertEqual(authority._auth_scheme, GOB_AUTH_SCHEME)