def get_items(self, params, context=None): """Get items in model. Args: params: A dict-like object containing parameters from the request query string and body. context: Key-values providing frame of reference of request Returns: A dict containing at least a 'results' key whose value is a list of items in the model. Additional keys set in the dict will also be rendered for the user. """ LOG.info('get_items has context %s', context) # Note(thread-safety): blocking call caller, source_id = api_utils.get_id_from_context(context) # FIXME(threod-safety): in DSE2, the returned caller can be a # datasource name. But the datasource name may now refer to a new, # unrelated datasource. Causing the rest of this code to operate on # an unintended datasource. # Fix: check UUID of datasource before operating. Abort if mismatch try: # Note(thread-safety): blocking call tablenames = self.invoke_rpc(caller, 'get_tablenames', {'source_id': source_id}) except exception.CongressException as e: LOG.exception("Exception occurred while retrieving tables" "from datasource %s", source_id) raise webservice.DataModelException.create(e) # when the source_id doesn't have any table, 'tablenames' is set([]) if isinstance(tablenames, set) or isinstance(tablenames, list): return {'results': [{'id': x} for x in tablenames]}
def get_items(self, params, context=None): """Get items in model. :param: params: A dict-like object containing parameters from the request query string and body. :param: context: Key-values providing frame of reference of request :returns: A dict containing at least a 'results' key whose value is a list of items in the model. Additional keys set in the dict will also be rendered for the user. """ LOG.info('get_items has context %s', context) # Note(thread-safety): blocking call caller, source_id = api_utils.get_id_from_context(context) # FIXME(threod-safety): in DSE2, the returned caller can be a # datasource name. But the datasource name may now refer to a new, # unrelated datasource. Causing the rest of this code to operate on # an unintended datasource. # Fix: check UUID of datasource before operating. Abort if mismatch try: # Note(thread-safety): blocking call tablenames = self.invoke_rpc(caller, 'get_tablenames', {'source_id': source_id}) except exception.CongressException as e: LOG.debug( "Exception occurred while retrieving tables" "from datasource %s", source_id) raise webservice.DataModelException.create(e) # when the source_id doesn't have any table, 'tablenames' is set([]) if isinstance(tablenames, set) or isinstance(tablenames, list): return {'results': [{'id': x} for x in tablenames]}
def get_item(self, id_, params, context=None): """Retrieve item with id id_ from model. Args: id_: The ID of the item to retrieve params: A dict-like object containing parameters from the request query string and body. context: Key-values providing frame of reference of request Returns: The matching item or None if item with id_ does not exist. """ caller, source_id = api_utils.get_id_from_context( context, self.datasource_mgr, self.engine) args = {'source_id': source_id, 'table_id': id_} try: tablename = self.invoke_rpc(caller, 'get_tablename', args) except exception.CongressException as e: LOG.exception("Exception occurred while retrieving table %s" "from datasource %s", id_, source_id) raise webservice.DataModelException.create(e) if tablename: return {'id': tablename} LOG.info('table id %s is not found in datasource %s', id_, source_id)
def get_item(self, id_, params, context=None): """Retrieve item with id id_ from model. Args: id_: The ID of the item to retrieve params: A dict-like object containing parameters from the request query string and body. context: Key-values providing frame of reference of request Returns: The matching item or None if item with id_ does not exist. """ caller, source_id = api_utils.get_id_from_context(context, self.datasource_mgr) table = context.get('table_id') args = {'source_id': source_id} try: schema = self.invoke_rpc(caller, 'get_datasource_schema', args) except exception.CongressException as e: raise webservice.DataModelException(e.code, str(e), http_status_code=e.code) # request to see the schema for one table if table: if table not in schema: raise webservice.DataModelException( 404, ("Table '{}' for datasource '{}' has no " "schema ".format(id_, source_id)), http_status_code=404) return api_utils.create_table_dict(table, schema) tables = [api_utils.create_table_dict(table_, schema) for table_ in schema] return {'tables': tables}
def add_item(self, item, params, id_=None, context=None): """POST webhook notification. :param item: The webhook payload :param params: A dict-like object containing parameters from the request query string and body. :param id_: not used in this case; should be None :param context: Key-values providing frame of reference of request """ caller, source_id = api_utils.get_id_from_context(context) table_name = context.get('table_name') try: if table_name: # json ingester case args = {'table_name': table_name, 'body': item} # Note(thread-safety): blocking call self.invoke_rpc(base.JSON_DS_SERVICE_PREFIX + caller, 'json_ingester_webhook_handler', args) else: args = {'payload': item} # Note(thread-safety): blocking call self.invoke_rpc(caller, 'process_webhook_notification', args) except exception.CongressException as e: raise webservice.DataModelException.create(e)
def get_items(self, params, context=None): """Get items in model. Args: params: A dict-like object containing parameters from the request query string and body. context: Key-values providing frame of reference of request Returns: A dict containing at least a 'results' key whose value is a list of items in the model. Additional keys set in the dict will also be rendered for the user. """ LOG.info('get_items has context %s', context) caller, source_id = api_utils.get_id_from_context( context, self.datasource_mgr, self.engine) try: tablenames = self.invoke_rpc(caller, 'get_tablenames', {'source_id': source_id}) except exception.CongressException as e: LOG.exception("Exception occurred while retrieving tables" "from datasource %s", source_id) raise webservice.DataModelException.create(e) # when the source_id doesn't have any table, 'tablenames' is set([]) if isinstance(tablenames, set) or isinstance(tablenames, list): return {'results': [{'id': x} for x in tablenames]}
def get_item(self, id_, params, context=None): """Retrieve item with id id_ from model. Args: id_: The ID of the item to retrieve params: A dict-like object containing parameters from the request query string and body. context: Key-values providing frame of reference of request Returns: The matching item or None if item with id_ does not exist. """ # Note(thread-safety): blocking call caller, source_id = api_utils.get_id_from_context(context) # FIXME(threod-safety): in DSE2, the returned caller can be a # datasource name. But the datasource name may now refer to a new, # unrelated datasource. Causing the rest of this code to operate on # an unintended datasource. # Fix: check UUID of datasource before operating. Abort if mismatch args = {'source_id': source_id, 'table_id': id_} try: # Note(thread-safety): blocking call tablename = self.invoke_rpc(caller, 'get_tablename', args) except exception.CongressException as e: LOG.exception("Exception occurred while retrieving table %s" "from datasource %s", id_, source_id) raise webservice.DataModelException.create(e) if tablename: return {'id': tablename} LOG.info('table id %s is not found in datasource %s', id_, source_id)
def get_item(self, id_, params, context=None): """Retrieve item with id id\_ from model. :param: id\_: The ID of the item to retrieve :param: params: A dict-like object containing parameters from the request query string and body. :param: context: Key-values providing frame of reference of request :returns: The matching item or None if item with id\_ does not exist. """ # Note(thread-safety): blocking call caller, source_id = api_utils.get_id_from_context(context) # FIXME(threod-safety): in DSE2, the returned caller can be a # datasource name. But the datasource name may now refer to a new, # unrelated datasource. Causing the rest of this code to operate on # an unintended datasource. # Fix: check UUID of datasource before operating. Abort if mismatch args = {'source_id': source_id, 'table_id': id_} try: # Note(thread-safety): blocking call tablename = self.invoke_rpc(caller, 'get_tablename', args) except exception.CongressException as e: LOG.debug( "Exception occurred while retrieving table %s" "from datasource %s", id_, source_id) raise webservice.DataModelException.create(e) if tablename: return {'id': tablename} LOG.info('table id %s is not found in datasource %s', id_, source_id)
def update_items(self, items, params, context=None): """Updates all data in a table. Args: id_: A table id for updating all row items: A data for new rows params: A dict-like object containing parameters from request query context: Key-values providing frame of reference of request Returns: None Raises: KeyError: table id doesn't exist DataModelException: any error occurs during replacing rows. """ LOG.info("update_items(context=%s)" % context) caller, source_id = api_utils.get_id_from_context(context, self.datasource_mgr, self.engine) table_id = context['table_id'] try: args = {'table_id': table_id, 'source_id': source_id, 'objs': items} self.invoke_rpc(caller, 'update_entire_data', args) except exception.CongressException as e: m = ("Error occurred while processing updating rows for " "source_id '%s' and table_id '%s'" % (source_id, table_id)) LOG.exception(m) raise webservice.DataModelException.create(e) LOG.info("finish update_items(context=%s)" % context) LOG.debug("updated table %s with row items: %s" % (table_id, str(items)))
def get_item(self, id_, params, context=None): """Retrieve item with id id_ from model. Args: id_: The ID of the item to retrieve params: A dict-like object containing parameters from the request query string and body. context: Key-values providing frame of reference of request Returns: The matching item or None if item with id_ does not exist. """ # Note(thread-safety): blocking call caller, source_id = api_utils.get_id_from_context(context) # FIXME(threod-safety): in DSE2, the returned caller can be a # datasource name. But the datasource name may now refer to a new, # unrelated datasource. Causing the rest of this code to operate on # an unintended datasource. # Fix: check UUID of datasource before operating. Abort if mismatch try: rpc_args = {'params': context, 'source_id': source_id} # Note(thread-safety): blocking call status = self.invoke_rpc(caller, 'get_status', rpc_args) except exception.CongressException as e: raise webservice.DataModelException( exception.NotFound.code, str(e), http_status_code=exception.NotFound.code) return status
def test_get_id_from_context_policy_id(self): context = {'policy_id': 'policy id'} expected = ('policy-engine', 'policy id') result = api_utils.get_id_from_context(context, 'datasource-mgr', 'policy-engine') self.assertEqual(expected, result)
def get_items(self, params, context=None): """Get items in model. Args: params: A dict-like object containing parameters from the request query string and body. context: Key-values providing frame of reference of request Returns: A dict containing at least a 'results' key whose value is a list of items in the model. Additional keys set in the dict will also be rendered for the user. """ LOG.info('get_items has context %s', context) caller, source_id = api_utils.get_id_from_context(context, self.datasource_mgr, self.engine) tablenames = self.rpc(caller, 'get_tablenames', source_id) # when the source_id doesn't have any table, 'tablenames' is set([]) # when the source_id doesn't exist 'tablenames' is None if isinstance(tablenames, set): return {'results': [{'id': x} for x in tablenames]} LOG.info('source id %s not found', source_id)
def request_refresh_action(self, params, context=None, request=None): caller, source_id = api_utils.get_id_from_context(context, self.datasource_mgr) try: args = {"source_id": source_id} self.invoke_rpc(caller, "request_refresh", args) except exception.CongressException as e: LOG.exception(e) raise webservice.DataModelException.create(e)
def request_refresh_action(self, params, context=None, request=None): caller, source_id = api_utils.get_id_from_context(context) try: args = {'source_id': source_id} # Note(thread-safety): blocking call self.invoke_rpc(caller, 'request_refresh', args) except exception.CongressException as e: LOG.debug(e) raise webservice.DataModelException.create(e)
def request_refresh_action(self, params, context=None, request=None): caller, source_id = api_utils.get_id_from_context(context) try: args = {'source_id': source_id} # Note(thread-safety): blocking call self.invoke_rpc(caller, 'request_refresh', args) except exception.CongressException as e: LOG.exception(e) raise webservice.DataModelException.create(e)
def get_items(self, params, context=None): """Get items in model. Args: params: A dict-like object containing parameters from the request query string and body. context: Key-values providing frame of reference of request Returns: A dict containing at least a 'results' key whose value is a list of items in the model. Additional keys set in the dict will also be rendered for the user. """ LOG.info("get_items(context=%s)", context) gen_trace = False if 'trace' in params and params['trace'].lower() == 'true': gen_trace = True # Get the caller, it should be either policy or datasource # Note(thread-safety): blocking call caller, source_id = api_utils.get_id_from_context(context) # FIXME(threod-safety): in DSE2, the returned caller can be a # datasource name. But the datasource name may now refer to a new, # unrelated datasource. Causing the rest of this code to operate on # an unintended datasource. # It would have saved us if table_id was an UUID rather than a name, # but it appears that table_id is just another word for tablename. # Fix: check UUID of datasource before operating. Abort if mismatch table_id = context['table_id'] try: args = { 'table_id': table_id, 'source_id': source_id, 'trace': gen_trace } if caller is base.ENGINE_SERVICE_ID: # allow extra time for row policy engine query # Note(thread-safety): blocking call result = self.invoke_rpc(caller, 'get_row_data', args, timeout=self.dse_long_timeout) else: # Note(thread-safety): blocking call result = self.invoke_rpc(caller, 'get_row_data', args) except exception.CongressException as e: m = ("Error occurred while processing source_id '%s' for row " "data of the table '%s'" % (source_id, table_id)) LOG.exception(m) raise webservice.DataModelException.create(e) if gen_trace and caller is base.ENGINE_SERVICE_ID: # DSE2 returns lists instead of tuples, so correct that. results = [{'data': tuple(x['data'])} for x in result[0]] return {'results': results, 'trace': result[1] or "Not available"} else: result = [{'data': tuple(x['data'])} for x in result] return {'results': result}
def get_items(self, params, context=None): """Get items in model. Args: params: A dict-like object containing parameters from the request query string and body. context: Key-values providing frame of reference of request Returns: A dict containing at least a 'results' key whose value is a list of items in the model. Additional keys set in the dict will also be rendered for the user. """ LOG.info("get_items(context=%s)", context) gen_trace = False if 'trace' in params and params['trace'].lower() == 'true': gen_trace = True # Get the caller, it should be either policy or datasource # Note(thread-safety): blocking call caller, source_id = api_utils.get_id_from_context(context) # FIXME(threod-safety): in DSE2, the returned caller can be a # datasource name. But the datasource name may now refer to a new, # unrelated datasource. Causing the rest of this code to operate on # an unintended datasource. # It would have saved us if table_id was an UUID rather than a name, # but it appears that table_id is just another word for tablename. # Fix: check UUID of datasource before operating. Abort if mismatch table_id = context['table_id'] try: args = {'table_id': table_id, 'source_id': source_id, 'trace': gen_trace} if caller is base.ENGINE_SERVICE_ID: # allow extra time for row policy engine query # Note(thread-safety): blocking call result = self.invoke_rpc( caller, 'get_row_data', args, timeout=self.dse_long_timeout) else: # Note(thread-safety): blocking call result = self.invoke_rpc(caller, 'get_row_data', args) except exception.CongressException as e: m = ("Error occurred while processing source_id '%s' for row " "data of the table '%s'" % (source_id, table_id)) LOG.exception(m) raise webservice.DataModelException.create(e) if gen_trace and caller is base.ENGINE_SERVICE_ID: # DSE2 returns lists instead of tuples, so correct that. results = [{'data': tuple(x['data'])} for x in result[0]] return {'results': results, 'trace': result[1] or "Not available"} else: result = [{'data': tuple(x['data'])} for x in result] return {'results': result}
def add_item(self, item, id_=None, context=None): """POST webhook notification. :param item: The webhook payload :param id_: not used in this case; should be None :param context: Key-values providing frame of reference of request """ caller, source_id = api_utils.get_id_from_context(context) try: args = {'payload': item} # Note(thread-safety): blocking call self.invoke_rpc(caller, 'process_webhook_notification', args) except exception.CongressException as e: raise webservice.DataModelException.create(e)
def get_item(self, id_, params, context=None): """Retrieve item with id id_ from model. Args: id_: The ID of the item to retrieve params: A dict-like object containing parameters from the request query string and body. context: Key-values providing frame of reference of request Returns: The matching item or None if item with id_ does not exist. """ # Note(thread-safety): blocking call caller, source_id = api_utils.get_id_from_context(context) # FIXME(threod-safety): in DSE2, the returned caller can be a # datasource name. But the datasource name may now refer to a new, # unrelated datasource. Causing the rest of this code to operate on # an unintended datasource. # Fix: check UUID of datasource before operating. Abort if mismatch table = context.get('table_id') args = {'source_id': source_id} try: # Note(thread-safety): blocking call schema = self.invoke_rpc(caller, 'get_datasource_schema', args) except exception.CongressException as e: raise webservice.DataModelException(e.code, str(e), http_status_code=e.code) # request to see the schema for one table if table: if table not in schema: raise webservice.DataModelException( 404, ("Table '{}' for datasource '{}' has no " "schema ".format(id_, source_id)), http_status_code=404) return api_utils.create_table_dict(table, schema) tables = [ api_utils.create_table_dict(table_, schema) for table_ in schema ] return {'tables': tables}
def replace_items(self, items, params, context=None): """Replaces all data in a table. :param: id\_: A table id for replacing all row :param: items: A data for new rows :param: params: A dict-like object containing parameters from request query :param: context: Key-values providing frame of reference of request :returns: None :raises KeyError: table id doesn't exist :raises DataModelException: any error occurs during replacing rows. """ LOG.info("replace_items(context=%s)", context) # Note(thread-safety): blocking call caller, source_id = api_utils.get_id_from_context(context) # FIXME(thread-safety): in DSE2, the returned caller can be a # datasource name. But the datasource name may now refer to a new, # unrelated datasource. Causing the rest of this code to operate on # an unintended datasource. # It would have saved us if table_id was an UUID rather than a name, # but it appears that table_id is just another word for tablename. # Fix: check UUID of datasource before operating. Abort if mismatch table_id = context['table_id'] try: args = { 'table_id': table_id, 'source_id': source_id, 'objs': items } # Note(thread-safety): blocking call self.invoke_rpc(caller, 'replace_entire_table_data', args) except exception.CongressException as e: LOG.debug( "Error occurred while processing updating rows " "for source_id '%s' and table_id '%s'", source_id, table_id, exc_info=True) raise webservice.DataModelException.create(e) LOG.info("finish replace_items(context=%s)", context) LOG.debug("replaced table %s with row items: %s", table_id, str(items))
def get_items(self, params, context=None): """Retrieve items from this model. :param: params: A dict-like object containing parameters from the request query string and body. :param: context: Key-values providing frame of reference of request :returns: A dict containing at least a 'actions' key whose value is a list of items in this model. """ # Note: blocking call caller, source_id = api_utils.get_id_from_context(context) try: rpc_args = {'source_id': source_id} # Note(dse2): blocking call return self.invoke_rpc(caller, 'get_actions', rpc_args) except exception.CongressException as e: raise webservice.DataModelException( exception.NotFound.code, str(e), http_status_code=exception.NotFound.code)
def get_items(self, params, context=None): """Get items in model. Args: params: A dict-like object containing parameters from the request query string and body. context: Key-values providing frame of reference of request Returns: A dict containing at least a 'results' key whose value is a list of items in the model. Additional keys set in the dict will also be rendered for the user. """ LOG.info("get_items(context=%s)", context) gen_trace = False if 'trace' in params and params['trace'].lower() == 'true': gen_trace = True # Get the caller, it should be either policy or datasource caller, source_id = api_utils.get_id_from_context( context, self.datasource_mgr, self.engine) table_id = context['table_id'] try: args = {'table_id': table_id, 'source_id': source_id, 'trace': gen_trace} result = self.invoke_rpc(caller, 'get_row_data', args) except exception.CongressException as e: m = ("Error occurred while processing source_id '%s' for row " "data of the table '%s'" % (source_id, table_id)) LOG.exception(m) raise webservice.DataModelException.create(e) if gen_trace and caller is self.engine: # DSE2 returns lists instead of tuples, so correct that. result[0] = [{'data': tuple(x['data'])} for x in result[0]] return {'results': result[0], 'trace': result[1] or "Not available"} else: result = [{'data': tuple(x['data'])} for x in result] return {'results': result}
def update_items(self, items, params, context=None): """Updates all data in a table. Args: id_: A table id for updating all row items: A data for new rows params: A dict-like object containing parameters from request query context: Key-values providing frame of reference of request Returns: None Raises: KeyError: table id doesn't exist DataModelException: any error occurs during replacing rows. """ LOG.info("update_items(context=%s)", context) # Note(thread-safety): blocking call caller, source_id = api_utils.get_id_from_context(context) # FIXME(threod-safety): in DSE2, the returned caller can be a # datasource name. But the datasource name may now refer to a new, # unrelated datasource. Causing the rest of this code to operate on # an unintended datasource. # It would have saved us if table_id was an UUID rather than a name, # but it appears that table_id is just another word for tablename. # Fix: check UUID of datasource before operating. Abort if mismatch table_id = context['table_id'] try: args = {'table_id': table_id, 'source_id': source_id, 'objs': items} # Note(thread-safety): blocking call self.invoke_rpc(caller, 'update_entire_data', args) except exception.CongressException as e: LOG.exception("Error occurred while processing updating rows " "for source_id '%s' and table_id '%s'", source_id, table_id) raise webservice.DataModelException.create(e) LOG.info("finish update_items(context=%s)", context) LOG.debug("updated table %s with row items: %s", table_id, str(items))
def get_item(self, id_, params, context=None): """Retrieve item with id id_ from model. Args: id_: The ID of the item to retrieve params: A dict-like object containing parameters from the request query string and body. context: Key-values providing frame of reference of request Returns: The matching item or None if item with id_ does not exist. """ caller, source_id = api_utils.get_id_from_context(context, self.datasource_mgr, self.engine) tablename = self.rpc(caller, 'get_tablename', source_id, id_) if tablename: return {'id': tablename} LOG.info('source id %s or table id %s is not found', source_id, id_)
def get_item(self, id_, params, context=None): """Retrieve item with id id_ from model. Args: id_: The ID of the item to retrieve params: A dict-like object containing parameters from the request query string and body. context: Key-values providing frame of reference of request Returns: The matching item or None if item with id_ does not exist. """ caller, source_id = api_utils.get_id_from_context( context, self.datasource_mgr, self.engine) try: rpc_args = {'params': context, 'source_id': source_id} status = self.invoke_rpc(caller, 'get_status', rpc_args) except exception.CongressException as e: raise webservice.DataModelException( exception.NotFound.code, str(e), http_status_code=exception.NotFound.code) return status
def test_get_id_from_context_ds_id(self): context = {'ds_id': 'datasource id'} expected = ('datasource id', 'datasource id') result = api_utils.get_id_from_context(context) self.assertEqual(expected, result)
def test_get_id_from_context_policy_id(self): context = {'policy_id': 'policy id'} expected = ('__engine', 'policy id') result = api_utils.get_id_from_context(context) self.assertEqual(expected, result)