def _allows_access(rule, *args, **kwargs): """ Check access to paths with variable catalog/collection names """ catalog_name = kwargs.get('catalog_name') collection_name = kwargs.get('collection_name') return Authority(catalog_name, collection_name).allows_access()
def test_secured_value(self, mock_user, mock_request): authority = Authority('cat', 'col') mock_user.return_value = "any user" mock_secure_type = mock.MagicMock() result = authority.get_secured_value(mock_secure_type) mock_user.assert_called_with(mock_request) mock_secure_type.get_value.assert_called_with("any user")
def get_curl(catalog_name, collection_name, path): """ Get the curl statement to retrieve the dataset in the Amsterdam Schema format :param catalog_name: :param collection_name: :return: """ auth_header = None # Make request to secure API if secured colllection if Authority(args.catalog, args.collection).is_secured(): if '/gob/secure' not in path: path = path.replace('/gob', '/gob/secure') auth_header = os.getenv('GOB_API_AUTH_HEADER') if not auth_header: print_yellow( "Generating a curl request for a secure endpoint, but missing Auth header env variable\n" "Use login_keycloak.py to generate the Auth header to use with the secure endpoint.\n" "Generated curl request contains a placeholder for now\n") auth_header = 'AUTH_HEADER_PLACEHOLDER' query = get_graphql_query(catalog_name, collection_name).replace('\n', '') query = '{"query":"%s"}' % query header = 'Content-Type: application/x-ndjson' auth = f"--header 'Authorization: {auth_header}' " if auth_header else "" url = get_url(catalog_name, collection_name, path) return f"curl -s --location --request POST '{url}' --header '{header}' {auth}--data-raw '{query}'"
def test_is_secure_type(self): authority = Authority('secure catalog', 'any col') spec = {"type": "GOB.String"} self.assertFalse(authority.is_secure_type(spec)) spec = {"type": "GOB.SecureString"} self.assertTrue(authority.is_secure_type(spec)) spec = { "type": "GOB.JSON", "attributes": { "attr": { "type": "GOB.String" } } } self.assertFalse(authority.is_secure_type(spec)) spec = { "type": "GOB.JSON", "attributes": { "attr": { "type": "GOB.String" }, "attr": { "type": "GOB.SecureString" } } } self.assertTrue(authority.is_secure_type(spec))
def _autorized_order(order, catalog_name, collection_name): """ Filter the order (list of columns) on columns that are not suppressed given the current request """ authority = Authority(catalog_name, collection_name) suppress_columns = authority.get_suppressed_columns() return [o for o in order if o not in suppress_columns]
def _dump_entities_to_table(self, entities, model): authority = Authority(self.catalog_name, self.collection_name) suppress_columns = authority.get_suppressed_columns() connection = self.datastore.connection stream = CSVStream(csv_entities(entities, model, suppress_columns), STREAM_PER) with connection.cursor() as cursor: yield "Export data" commit = COMMIT_PER while stream.has_items(): stream.reset_count() cursor.copy_expert( sql=f"COPY {self.schema}.{self.tmp_collection_name} FROM STDIN DELIMITER ';' CSV HEADER;", file=stream, size=BUFFER_PER ) if stream.total_count >= commit: connection.commit() commit += COMMIT_PER yield f"\n{self.collection_name}: {stream.total_count:,}" else: # Let client know we're still working. yield "." yield f"\nExported {stream.total_count} rows\n" connection.commit()
def test_is_secured(self): testcases = [ ('any catalog', 'any collection', True), ('secure catalog', 'any collection', True), ('any catalog', 'some other collection', False), ('open catalog', 'collection', False), ] for cat, coll, result in testcases: authority = Authority(cat, coll) self.assertEqual(result, authority.is_secured())
def test_filter_row(self): authority = Authority('cat', 'col') authority.get_suppressed_columns = lambda: ['b', 'd'] row = {'a': 1, 'b': 2, 'c': 3} authority.filter_row(row) self.assertEqual(row, {'a': 1, 'b': None, 'c': 3}) authority.allows_access = lambda: False row = {'a': 1, 'b': 2, 'c': 3} authority.filter_row(row) self.assertEqual(row, {'a': None, 'b': None, 'c': None})
def test_get_secured_columns(self, mock_model): mock_model.return_value.get_collection.return_value = { 'fields': { 'secure column': 'any spec' } } authority = Authority('secure catalog', 'any col') secure_columns = authority.get_secured_columns() self.assertEqual( secure_columns, { 'secure column': { 'gob_type': gob_secure_types.SecureString, 'spec': 'any spec' } })
def __init__(self, catalog_name: str, collection_name: str, config: dict): self.catalog_name = catalog_name self.collection_name = collection_name self.tmp_collection_name = f"tmp_{collection_name}" self.datastore = DatastoreFactory.get_datastore(config['db']) self.datastore.connect() self.schema = self._get_dst_schema(config, catalog_name, collection_name) _, self.model = get_table_and_model(catalog_name, collection_name) # Set attributes for create_table self.model['catalog'] = catalog_name self.model['collection'] = collection_name self.suppressed_columns = Authority(self.catalog_name, self.collection_name).get_suppressed_columns()
def sql(self): self._reset() # Relation without parent is main relation base_collection = [ k for k, v in self.relation_parents.items() if v is None ][0] self._collect_relation_info(base_collection, base_collection) base_info = self._get_relation_info(base_collection) select_fields = [ self._select_expression(base_info, field) for field in [FIELD.GOBID] + self.selects[base_collection]['fields'] ] self.select_expressions.extend(select_fields) # Add catalog and collection to allow for value resolution self.select_expressions.extend([ f"'{base_info['catalog_name']}' AS {CATALOG_NAME}", f"'{base_info['collection_name']}' AS {COLLECTION_NAME}", ]) authority = Authority(base_info['catalog_name'], base_info['collection_name']) if not authority.allows_access(): raise NoAccessException arguments = self._get_arguments_with_defaults( self.selects[base_collection]['arguments']) self.joins.append( self._build_from_table(arguments, base_info['tablename'], base_info['alias'])) del self.selects[base_collection] self._join_relations(self.selects) select = ',\n'.join(self.select_expressions) table_select = '\n'.join(self.joins) order_by = f"ORDER BY {base_info['alias']}.{FIELD.GOBID}" query = f"SELECT\n{select}\n{table_select}\n{order_by}" return query
def test_allows_access(self): authority = Authority('secure catalog', 'any col') authority.get_roles = lambda: [] self.assertFalse(authority.allows_access()) authority.get_roles = lambda: [role_b] self.assertTrue(authority.allows_access()) authority._catalog = "secure catalog collection" authority._collection = "secure collection" authority.get_roles = lambda: [] self.assertFalse(authority.allows_access()) authority.get_roles = lambda: [role_b] self.assertTrue(authority.allows_access()) authority._collection = "any collection" authority.get_roles = lambda: [] self.assertTrue(authority.allows_access())
def resolve_row(self, row, result): """ Resolve all values in the row Update the resolved value in the result :param row: :param result: :return: """ catalog_name = row.get(CATALOG_NAME) collection_name = row.get(COLLECTION_NAME) self._init_catalog_collection(catalog_name, collection_name) # Filter row and result for columns that do not match with the roles of the current request authority = Authority(catalog_name, collection_name) authority.filter_row(row, mapping=self._attributes[catalog_name][collection_name]) authority.filter_row(result, mapping=self._attributes[catalog_name][collection_name]) for attr in [name for name in [CATALOG_NAME, COLLECTION_NAME] if name in row]: # Once a row has been resolved, don't resolve it twice del row[attr]
def test_get_secured_json_columns(self, mock_model): mock_model.return_value.get_collection.return_value = { 'fields': { 'secure column': { 'type': 'GOB.JSON', 'attributes': { 'attr1': { 'type': 'GOB.SecureString' }, 'attr2': { 'type': 'GOB.String' } } } } } authority = Authority('secure catalog', 'any col') secure_columns = authority.get_secured_columns() expect = { 'secure column': { 'gob_type': gob_types.JSON, 'spec': { 'type': 'GOB.JSON', 'gob_type': gob_types.JSON, 'attributes': { 'attr1': { 'type': 'GOB.SecureString', 'gob_type': gob_secure_types.SecureString }, 'attr2': { 'type': 'GOB.String', 'gob_type': gob_types.String } }, } } } print(secure_columns) self.assertEqual(secure_columns, expect)
def test_handle_secured_columns(self): authority = Authority('secure catalog', 'any col') authority.get_secured_columns = lambda: {'col': 'any info'} authority.exposed_value = lambda value, info: 'exposed value' row = {} authority._handle_secured_columns({}, row) self.assertEqual(row, {}) row = {'col': 'any value', 'any other col': 'any value'} authority._handle_secured_columns({}, row) self.assertEqual(row, { 'col': 'exposed value', 'any other col': 'any value' }) row = {'mapped col': 'any value', 'any other col': 'any value'} mapping = {'col': 'mapped col'} authority._handle_secured_columns(mapping, row) self.assertEqual(row, { 'mapped col': 'exposed value', 'any other col': 'any value' })
def test_create(self): authority = Authority('cat', 'col') self.assertEqual(authority._catalog, 'cat') self.assertEqual(authority._collection, 'col') self.assertEqual(authority._auth_scheme, GOB_AUTH_SCHEME)