def run_executable_insert(did, app_did, target_did, target_app_did, executable_body, params): r, msg = can_access_vault(target_did, VAULT_ACCESS_WR) if not r: return None, msg executable_body_document = executable_body.get('document', {}) populate_params_insert_update(did, app_did, executable_body_document, params) created = "created" in executable_body_document.keys() options = populate_options_insert_one(executable_body) col = get_collection(target_did, target_app_did, executable_body.get('collection')) data, err_message = query_insert_one(col, executable_body, options, created=created) if err_message: return None, err_message db_size = get_mongo_database_size(target_did, target_app_did) update_vault_db_use_storage_byte(did, db_size) return data, None
def delete_many(self): did, app_id, content, err = post_json_param_pre_proc( self.response, "collection", "filter", access_vault=VAULT_ACCESS_DEL) if err: return err col = get_collection(did, app_id, content["collection"]) if not col: return self.response.response_err(NOT_FOUND, "collection not exist") try: ret = col.delete_many(convert_oid(content["filter"])) data = { "acknowledged": ret.acknowledged, "deleted_count": ret.deleted_count, } db_size = get_mongo_database_size(did, app_id) update_vault_db_use_storage_byte(did, db_size) return self.response.response_ok(data) except Exception as e: return self.response.response_err(INTERNAL_SERVER_ERROR, "Exception:" + str(e))
def run_executable_update(did, app_did, target_did, target_app_did, executable_body, params): r, msg = can_access_vault(target_did, VAULT_ACCESS_WR) if r != SUCCESS: return None, msg executable_body_filter = executable_body.get('filter', {}) err_message = populate_with_params_values(did, app_did, executable_body_filter, params) if err_message: return None, err_message executable_body_update = executable_body.get('update').get('$set') err_message = populate_with_params_values(did, app_did, executable_body_update, params) if err_message: return None, err_message options = populate_options_update_one(executable_body) err_message = populate_with_params_values(did, app_did, options, params) if err_message: return None, err_message col = get_collection(target_did, target_app_did, executable_body.get('collection')) data, err_message = query_update_one(col, executable_body, options) if err_message: return None, err_message db_size = get_mongo_database_size(target_did, target_app_did) update_vault_db_use_storage_byte(did, db_size) return data, None
def insert_many(self): did, app_id, content, err = post_json_param_pre_proc(self.response, "collection", "document", access_vault=VAULT_ACCESS_WR) if err: return err col = get_collection(did, app_id, content["collection"]) if not col: return self.response.response_err(NOT_FOUND, "collection not exist") options = options_filter(content, ("bypass_document_validation", "ordered")) try: new_document = [] for document in content["document"]: document["created"] = datetime.utcnow() document["modified"] = datetime.utcnow() new_document.append(convert_oid(document)) ret = col.insert_many(new_document, **options) db_size = get_mongo_database_size(did, app_id) update_vault_db_use_storage_byte(did, db_size) data = { "acknowledged": ret.acknowledged, "inserted_ids": [str(_id) for _id in ret.inserted_ids] } return self.response.response_ok(data) except Exception as e: return self.response.response_err(INTERNAL_SERVER_ERROR, "Exception:" + str(e))
def run_executable_find(did, app_did, target_did, target_app_did, executable_body, params): r, msg = can_access_vault(target_did, VAULT_ACCESS_R) if r != SUCCESS: return None, msg executable_body_filter = executable_body.get('filter', {}) err_message = populate_with_params_values(did, app_did, executable_body_filter, params) if err_message: return None, err_message options = populate_options_find_many(executable_body) err_message = populate_with_params_values(did, app_did, options, params) if err_message: return None, err_message col = get_collection(target_did, target_app_did, executable_body.get('collection')) if not col: return None, f'Can not find the collection {executable_body.get("collection")}' data, err_message = query_find_many(col, executable_body, options) if err_message: return None, err_message return data, None
def find_one(self): did, app_id, content, err = post_json_param_pre_proc(self.response, "collection", access_vault=VAULT_ACCESS_R) if err: return err col = get_collection(did, app_id, content["collection"]) if not col: return self.response.response_err(NOT_FOUND, "collection not exist") options = options_filter(content, ("projection", "skip", "sort", "allow_partial_results", "return_key", "show_record_id", "batch_size")) if "sort" in options: sorts = gene_sort(options["sort"]) options["sort"] = sorts try: if "filter" in content: result = col.find_one(convert_oid(content["filter"]), **options) else: result = col.find_one(**options) data = {"items": json.loads(json_util.dumps(result))} return self.response.response_ok(data) except Exception as e: return self.response.response_err(INTERNAL_SERVER_ERROR, "Exception:" + str(e))
def update_many(self): did, app_id, content, err = post_json_param_pre_proc(self.response, "collection", "filter", "update", access_vault=VAULT_ACCESS_WR) if err: return err col = get_collection(did, app_id, content["collection"]) if not col: return self.response.response_err(NOT_FOUND, "collection not exist") options = options_filter(content, ("upsert", "bypass_document_validation")) try: update_set_on_insert = content.get('update').get('$setOnInsert', None) if update_set_on_insert: content["update"]["$setOnInsert"]['created'] = datetime.utcnow() else: content["update"]["$setOnInsert"] = { "created": datetime.utcnow() } if "$set" in content["update"]: content["update"]["$set"]["modified"] = datetime.utcnow() ret = col.update_many(convert_oid(content["filter"]), convert_oid(content["update"], update=True), **options) data = { "acknowledged": ret.acknowledged, "matched_count": ret.matched_count, "modified_count": ret.modified_count, "upserted_id": str(ret.upserted_id) } db_size = get_mongo_database_size(did, app_id) update_vault_db_use_storage_byte(did, db_size) return self.response.response_ok(data) except Exception as e: return self.response.response_err(INTERNAL_SERVER_ERROR, "Exception:" + str(e))
def update_one(self): did, app_id, content, err = post_json_param_pre_proc( self.response, "collection", "filter", "update", access_vault=VAULT_ACCESS_WR) if err: return err options = populate_options_update_one(content) col = get_collection(did, app_id, content["collection"]) if not col: return self.response.response_err(NOT_FOUND, "collection not exist") data, err_message = query_update_one(col, content, options) if err_message: return self.response.response_err(INTERNAL_SERVER_ERROR, err_message) db_size = get_mongo_database_size(did, app_id) update_vault_db_use_storage_byte(did, db_size) return self.response.response_ok(data)
def run_executable_file_upload(did, app_did, target_did, target_app_did, executable_body, params): r, msg = can_access_vault(target_did, VAULT_ACCESS_WR) if r != SUCCESS: return None, msg executable_body_path = executable_body.get("path", "") if executable_body_path.startswith(f"{SCRIPTING_EXECUTABLE_PARAMS}."): v = executable_body_path.replace(f"{SCRIPTING_EXECUTABLE_PARAMS}.", "") try: v = params[v] except Exception as e: return None, f"Exception: {str(e)}" executable_body_path = v if not executable_body_path: return None, f"Path cannot be empty" full_path_name, err = query_upload_get_filepath(target_did, target_app_did, executable_body_path) if err: return None, f"Exception: Could not upload file. Status={err['status_code']} Error='{err['description']}'" content = { "document": { "file_name": executable_body_path, "fileapi_type": "upload" } } col = get_collection(target_did, target_app_did, SCRIPTING_SCRIPT_TEMP_TX_COLLECTION) if not col: return None, f"collection {SCRIPTING_SCRIPT_TEMP_TX_COLLECTION} does not exist" data, err_message = query_insert_one(col, content, {}) if err_message: return None, f"Could not insert data into the database: Err: {err_message}" db_size = get_mongo_database_size(target_did, target_app_did) update_vault_db_use_storage_byte(target_did, db_size) row_id = data.get("inserted_id", None) if not row_id: return None, f"Could not retrieve the transaction ID. Please try again" data = { "transaction_id": jwt.encode( { "row_id": row_id, "target_did": target_did, "target_app_did": target_app_did }, hive_setting.DID_STOREPASS, algorithm='HS256') } return data, None
def run_script_fileapi_setup(self, transaction_id, fileapi_type): # Request script content first try: transaction_detail = jwt.decode(transaction_id, hive_setting.DID_STOREPASS, algorithms=['HS256']) row_id, target_did, target_app_did = transaction_detail.get('row_id', None), transaction_detail.get('target_did', None), \ transaction_detail.get('target_app_did', None) except Exception as e: err = [ INTERNAL_SERVER_ERROR, f"Error while executing file {fileapi_type} via scripting: Could not unpack details " f"from transaction_id jwt token. Exception: {str(e)}" ] return None, None, None, None, err r, m = can_access_vault(target_did, VAULT_ACCESS_R) if r != SUCCESS: err = [ r, f"Error while executing file {fileapi_type} via scripting: vault can not be accessed" ] return None, None, None, None, err # Find the temporary tx in the database try: col = get_collection(target_did, target_app_did, SCRIPTING_SCRIPT_TEMP_TX_COLLECTION) content_filter = {"_id": ObjectId(row_id)} script_temp_tx = col.find_one(content_filter) except Exception as e: err = [ NOT_FOUND, f"Error while executing file {fileapi_type} via scripting: Exception: {str(e)}" ] return None, None, None, None, err if not script_temp_tx: err = [ NOT_FOUND, f"Error while executing file {fileapi_type} via scripting: " f"Exception: Could not find the transaction ID '{transaction_id}' in the database" ] return None, None, None, None, err file_name = script_temp_tx.get('file_name', None) if not file_name: err = [ NOT_FOUND, f"Error while executing file {fileapi_type} via scripting: Could not find a file_name " f"'{file_name}' to be used to upload" ] return None, None, None, None, err return row_id, target_did, target_app_did, file_name, None
def run_executable_file_download(did, app_did, target_did, target_app_did, executable_body, params): r, msg = can_access_vault(target_did, VAULT_ACCESS_R) if not r: return None, msg executable_body_path = executable_body.get("path", "") if executable_body_path.startswith(f"{SCRIPTING_EXECUTABLE_PARAMS}."): v = executable_body_path.replace(f"{SCRIPTING_EXECUTABLE_PARAMS}.", "") if not (params and params.get(v, None)): return None, "Exception: Parameter is not set" executable_body_path = params[v] if not executable_body_path: return None, f"Path cannot be empty" content = { "document": { "file_name": executable_body_path, "fileapi_type": "download" } } col = get_collection(target_did, target_app_did, SCRIPTING_SCRIPT_TEMP_TX_COLLECTION) if not col: return None, f"collection {SCRIPTING_SCRIPT_TEMP_TX_COLLECTION} does not exist" data, err_message = query_insert_one(col, content, {}) if err_message: return None, f"Could not insert data into the database: Err: {err_message}" db_size = get_mongo_database_size(target_did, target_app_did) update_vault_db_use_storage_byte(target_did, db_size) row_id = data.get("inserted_id", None) if not row_id: return None, f"Could not retrieve the transaction ID. Please try again" data = { "transaction_id": jwt.encode( { "row_id": row_id, "target_did": target_did, "target_app_did": target_app_did }, hive_setting.DID_STOREPASS, algorithm='HS256') } return data, None
def find_many(self): did, app_id, content, err = post_json_param_pre_proc(self.response, "collection", access_vault=VAULT_ACCESS_R) if err: return err options = populate_options_find_many(content) col = get_collection(did, app_id, content.get('collection')) if not col: return self.response.response_err(NOT_FOUND, "collection not exist") data, err_message = query_find_many(col, content, options) if err_message: return self.response.response_err(INTERNAL_SERVER_ERROR, err_message) return self.response.response_ok(data)
def run_script_fileapi_teardown(self, row_id, target_did, target_app_did, fileapi_type): try: col = get_collection(target_did, target_app_did, SCRIPTING_SCRIPT_TEMP_TX_COLLECTION) content_filter = {"filter": {"_id": ObjectId(row_id)}} _, err_message = query_delete_one(col, content_filter) if err_message: err_message = f"Error while executing file {fileapi_type} via scripting: {err_message}" return err_message db_size = get_mongo_database_size(target_did, target_app_did) update_vault_db_use_storage_byte(target_did, db_size) except Exception as e: err_message = f"Error while executing file {fileapi_type} via scripting: Exception: {str(e)}" return err_message return None
def run_executable_find(did, app_did, target_did, target_app_did, executable_body, params): r, msg = can_access_vault(target_did, VAULT_ACCESS_R) if not r: return None, msg executable_body_filter = executable_body.get('filter', {}) populate_params_find_count_delete(did, app_did, executable_body_filter, params) options = populate_options_find_many(executable_body) col = get_collection(target_did, target_app_did, executable_body.get('collection')) data, err_message = query_find_many(col, executable_body, options) if err_message: return None, err_message return data, None
def run_executable_delete(did, app_did, target_did, target_app_did, executable_body, params): r, msg = can_access_vault(target_did, VAULT_ACCESS_R) if not r: return None, msg executable_body_filter = executable_body.get('filter', {}) populate_params_find_count_delete(did, app_did, executable_body_filter, params) col = get_collection(target_did, target_app_did, executable_body.get('collection')) data, err_message = query_delete_one(col, executable_body) if err_message: return None, err_message db_size = get_mongo_database_size(target_did, target_app_did) update_vault_db_use_storage_byte(did, db_size) return data, None
def __upsert_script_to_db(self, did, app_id, content): if hive_setting.MONGO_URI: uri = hive_setting.MONGO_URI connection = MongoClient(uri) else: connection = MongoClient(host=hive_setting.MONGO_HOST, port=hive_setting.MONGO_PORT) db_name = gene_mongo_db_name(did, app_id) db = connection[db_name] try: db.create_collection(SCRIPTING_SCRIPT_COLLECTION) except CollectionInvalid: pass except Exception as e: return None, f"Could not create collection. Please try again later. Exception : {str(e)}" try: db.create_collection(SCRIPTING_SCRIPT_TEMP_TX_COLLECTION) except CollectionInvalid: pass except Exception as e: return None, f"Could not create collection. Please try again later. Exception : {str(e)}" col = get_collection(did, app_id, SCRIPTING_SCRIPT_COLLECTION) query = {"name": content.get("name")} options = {"upsert": True, "bypass_document_validation": False} try: ret = col.replace_one(query, convert_oid(content), **options) data = { "acknowledged": ret.acknowledged, "matched_count": ret.matched_count, "modified_count": ret.modified_count, "upserted_id": str(ret.upserted_id), } except Exception as e: return None, f"Exception: method: '__upsert_script_to_db', Err: {str(e)}" db_size = get_mongo_database_size(did, app_id) update_vault_db_use_storage_byte(did, db_size) return data, None
def run_condition(did, app_did, target_did, target_app_did, condition_body, params): condition_body_filter = condition_body.get('filter', {}) populate_params_find_count_delete(did, app_did, condition_body_filter, params, condition=True) options = populate_options_count_documents(condition_body) col = get_collection(target_did, target_app_did, condition_body.get('collection')) data, err_message = query_count_documents(col, condition_body, options) if err_message: return False if data.get('count') == 0: return False return True
def run_condition(did, app_did, target_did, target_app_did, condition_body, params): condition_body_filter = condition_body.get('filter', {}) err_message = populate_with_params_values(did, app_did, condition_body_filter, params) if err_message: return False options = populate_options_count_documents(condition_body) err_message = populate_with_params_values(did, app_did, options, params) if err_message: return False col = get_collection(target_did, target_app_did, condition_body.get('collection')) data, err_message = query_count_documents(col, condition_body, options) if err_message: return False if data.get('count') == 0: return False return True
def __run_script(self, script_name, caller_did, caller_app_did, target_did, target_app_did, params): r, msg = can_access_vault(target_did, VAULT_ACCESS_R) if r != SUCCESS: logging.debug( f"Error while executing script named '{script_name}': vault can not be accessed" ) return self.response.response_err(r, msg) # Find the script in the database col = get_collection(target_did, target_app_did, SCRIPTING_SCRIPT_COLLECTION) content_filter = {"name": script_name} err_message = f"could not find script '{script_name}' in the database. Please register the script " \ f"first with set_script' API endpoint" try: script = col.find_one(content_filter) except Exception as e: err_message = f"{err_message}. Exception: {str(e)}" logging.debug( f"Error while executing script named '{script_name}': {err_message}" ) return self.response.response_err(INTERNAL_SERVER_ERROR, err_message) if not script: logging.debug( f"Error while executing script named '{script_name}': {err_message}" ) return self.response.response_err(NOT_FOUND, err_message) # Validate anonymity options allow_anonymous_user = script.get('allowAnonymousUser', False) allow_anonymous_app = script.get('allowAnonymousApp', False) if (allow_anonymous_user is True) and (allow_anonymous_app is False): err_message = "Error while validating anonymity options: Cannot set allowAnonymousUser to be True but " \ "allowAnonymousApp to be False as we cannot request an auth to prove an app identity without " \ "proving the user identity" logging.debug(err_message) return self.response.response_err(BAD_REQUEST, err_message) if allow_anonymous_user is True: caller_did = None else: if not caller_did: logging.debug( f"Error while executing script named '{script_name}': Auth failed. caller_did " f"not set") return self.response.response_err( UNAUTHORIZED, "Auth failed. caller_did not set") if allow_anonymous_app is True: caller_app_did = None else: if not caller_app_did: logging.debug( f"Error while executing script named '{script_name}': Auth failed. " f"caller_app_did not set") return self.response.response_err( UNAUTHORIZED, "Auth failed. caller_app_did not set") logging.debug( f"Executing a script named '{script_name}' with params: " f"Caller DID: '{caller_did}', Caller App DID: '{caller_app_did}', " f"Target DID: '{target_did}', Target App DID: '{target_app_did}', " f"Anonymous User Access: {allow_anonymous_user}, Anonymous App Access: {allow_anonymous_app}" ) condition = script.get('condition', None) if condition: # Currently, there's only one kind of condition("count" db query) r, msg = can_access_vault(target_did, VAULT_ACCESS_R) if r != SUCCESS: logging.debug( f"Error while executing script named '{script_name}': vault can not be accessed" ) return self.response.response_err(r, msg) passed = self.__condition_execution(caller_did, caller_app_did, target_did, target_app_did, condition, params) if not passed: err_message = f"the conditions were not met to execute this script" logging.debug( f"Error while executing script named '{script_name}': {err_message}" ) return self.response.response_err(FORBIDDEN, err_message) executable = script.get("executable") unmassage_keys_with_dollar_signs(executable) output = {} data = self.__executable_execution(caller_did, caller_app_did, target_did, target_app_did, executable, params, output=output, output_key=executable.get( 'name', "output0")) return data