def count_db_storage_size(did): did_info_list = get_all_did_info_by_did(did) total_size = 0.0 for did_info in did_info_list: total_size += get_mongo_database_size(did_info[USER_DID], did_info[APP_ID]) return total_size
def update_document(self, collection_name, json_body, is_update_one): user_did, app_did, col = self.__get_collection(collection_name, VAULT_ACCESS_WR) update = json_body["update"] if "$set" in update: update["$set"]["modified"] = datetime.utcnow() if is_update_one: ret = col.update_one( convert_oid(json_body["filter"]), convert_oid(update, update=True), **options_filter(json_body, ("upsert", "bypass_document_validation"))) else: ret = col.update_many( convert_oid(json_body["filter"]), convert_oid(update, update=True), **options_filter(json_body, ("upsert", "bypass_document_validation"))) update_used_storage_for_mongodb_data( user_did, get_mongo_database_size(user_did, app_did)) return { "acknowledged": ret.acknowledged, "matched_count": ret.matched_count, "modified_count": ret.modified_count, "upserted_id": str(ret.upserted_id) if ret.upserted_id else None }
def run_executable_update(did, app_did, target_did, target_app_did, executable_body, params): r, msg = can_access_vault(target_did, VAULT_ACCESS_WR) if r != SUCCESS: return None, msg executable_body_filter = executable_body.get('filter', {}) err_message = populate_with_params_values(did, app_did, executable_body_filter, params) if err_message: return None, err_message executable_body_update = executable_body.get('update').get('$set') err_message = populate_with_params_values(did, app_did, executable_body_update, params) if err_message: return None, err_message options = populate_options_update_one(executable_body) err_message = populate_with_params_values(did, app_did, options, params) if err_message: return None, err_message col = get_collection(target_did, target_app_did, executable_body.get('collection')) data, err_message = query_update_one(col, executable_body, options) if err_message: return None, err_message db_size = get_mongo_database_size(target_did, target_app_did) update_vault_db_use_storage_byte(did, db_size) return data, None
def run_executable_insert(did, app_did, target_did, target_app_did, executable_body, params): r, msg = can_access_vault(target_did, VAULT_ACCESS_WR) if r != SUCCESS: return None, msg executable_body_document = executable_body.get('document', {}) err_message = populate_with_params_values(did, app_did, executable_body_document, params) if err_message: return None, err_message created = "created" in executable_body_document.keys() options = populate_options_insert_one(executable_body) err_message = populate_with_params_values(did, app_did, options, params) if err_message: return None, err_message col = get_collection(target_did, target_app_did, executable_body.get('collection')) data, err_message = query_insert_one(col, executable_body, options, created=created) if err_message: return None, err_message db_size = get_mongo_database_size(target_did, target_app_did) update_vault_db_use_storage_byte(did, db_size) return data, None
def delete_collection(self, collection_name): user_did, app_did = check_auth_and_vault(VAULT_ACCESS_DEL) cli.delete_collection(user_did, app_did, collection_name, is_check_exist=False) update_used_storage_for_mongodb_data( user_did, get_mongo_database_size(user_did, app_did))
def delete_document(self, collection_name, col_filter, is_delete_one): user_did, app_did, col = self.__get_collection(collection_name, VAULT_ACCESS_WR) if is_delete_one: col.delete_one(convert_oid(col_filter)) else: col.delete_many(convert_oid(col_filter)) update_used_storage_for_mongodb_data( user_did, get_mongo_database_size(user_did, app_did))
def _create_transaction(self, permission, action_type): cli.check_vault_access(self.get_target_did(), permission) body = self.get_populated_body() anonymous_url = '' if self.is_ipfs: if action_type == 'download': metadata = self.ipfs_files.get_file_metadata( self.get_target_did(), self.get_target_app_did(), body['path']) anonymous_url = self.ipfs_files.get_ipfs_file_access_url( metadata) else: _, err = query_upload_get_filepath(self.get_target_did(), self.get_target_app_did(), body['path']) if err: raise BadRequestException( msg='Cannot get file full path with error message: ' + str(err)) # INFO: Do not consider run script twice. data = cli.insert_one( self.get_target_did(), self.get_target_app_did(), SCRIPTING_SCRIPT_TEMP_TX_COLLECTION, { "document": { "file_name": body['path'], "fileapi_type": action_type }, 'anonymous': self.script.anonymous_app and self.script.anonymous_user }, create_on_absence=True) if not data.get('inserted_id', None): raise BadRequestException('Cannot retrieve the transaction ID.') update_used_storage_for_mongodb_data( self.get_target_did(), get_mongo_database_size(self.get_target_did(), self.get_target_app_did())) result = { "transaction_id": jwt.encode( { "row_id": data.get('inserted_id', None), "target_did": self.get_target_did(), "target_app_did": self.get_target_app_did() }, hive_setting.DID_STOREPASS, algorithm='HS256') } if action_type == 'download' and self.is_ipfs and self.script.anonymous_app and self.script.anonymous_user: result['anonymous_url'] = anonymous_url return result
def run_executable_file_upload(did, app_did, target_did, target_app_did, executable_body, params): r, msg = can_access_vault(target_did, VAULT_ACCESS_WR) if r != SUCCESS: return None, msg executable_body_path = executable_body.get("path", "") if executable_body_path.startswith(f"{SCRIPTING_EXECUTABLE_PARAMS}."): v = executable_body_path.replace(f"{SCRIPTING_EXECUTABLE_PARAMS}.", "") try: v = params[v] except Exception as e: return None, f"Exception: {str(e)}" executable_body_path = v if not executable_body_path: return None, f"Path cannot be empty" full_path_name, err = query_upload_get_filepath(target_did, target_app_did, executable_body_path) if err: return None, f"Exception: Could not upload file. Status={err['status_code']} Error='{err['description']}'" content = { "document": { "file_name": executable_body_path, "fileapi_type": "upload" } } col = get_collection(target_did, target_app_did, SCRIPTING_SCRIPT_TEMP_TX_COLLECTION) if not col: return None, f"collection {SCRIPTING_SCRIPT_TEMP_TX_COLLECTION} does not exist" data, err_message = query_insert_one(col, content, {}) if err_message: return None, f"Could not insert data into the database: Err: {err_message}" db_size = get_mongo_database_size(target_did, target_app_did) update_vault_db_use_storage_byte(target_did, db_size) row_id = data.get("inserted_id", None) if not row_id: return None, f"Could not retrieve the transaction ID. Please try again" data = { "transaction_id": jwt.encode( { "row_id": row_id, "target_did": target_did, "target_app_did": target_app_did }, hive_setting.DID_STOREPASS, algorithm='HS256') } return data, None
def set_script(self, script_name): user_did, app_did = check_auth_and_vault(VAULT_ACCESS_WR) json_data = request.get_json(force=True, silent=True) Script.validate_script_data(json_data) result = self.__upsert_script_to_database(script_name, json_data, user_did, app_did) update_used_storage_for_mongodb_data( user_did, get_mongo_database_size(user_did, app_did)) return result
def delete_script(self, script_name): user_did, app_did = check_auth_and_vault(VAULT_ACCESS_DEL) col = cli.get_user_collection(user_did, app_did, SCRIPTING_SCRIPT_COLLECTION) if not col: raise NotFoundException(NotFoundException.SCRIPT_NOT_FOUND, 'The script collection does not exist.') ret = col.delete_many({'name': script_name}) if ret.deleted_count > 0: update_used_storage_for_mongodb_data( user_did, get_mongo_database_size(user_did, app_did))
def execute(self): cli.check_vault_access(self.get_target_did(), VAULT_ACCESS_DEL) data = cli.delete_one(self.get_target_did(), self.get_target_app_did(), self.get_collection_name(), self.get_populated_filter()) update_used_storage_for_mongodb_data( self.get_did(), get_mongo_database_size(self.get_target_did(), self.get_target_app_did())) return self.get_output_data(data)
def insert_document(self, collection_name, json_body): user_did, app_did, col = self.__get_collection(collection_name, VAULT_ACCESS_WR) documents = [] for document in json_body["document"]: document["created"] = datetime.utcnow() document["modified"] = datetime.utcnow() documents.append(convert_oid(document)) ret = col.insert_many( documents, **options_filter(json_body, ("bypass_document_validation", "ordered"))) update_used_storage_for_mongodb_data( user_did, get_mongo_database_size(user_did, app_did)) return { "acknowledged": ret.acknowledged, "inserted_ids": [str(_id) for _id in ret.inserted_ids] }
def handle_transaction(self, transaction_id, is_download=False): check_auth_and_vault( VAULT_ACCESS_R if is_download else VAULT_ACCESS_WR) # check by transaction id row_id, target_did, target_app_did = self.parse_transaction_id( transaction_id) col_filter = {"_id": ObjectId(row_id)} trans = cli.find_one(target_did, target_app_did, SCRIPTING_SCRIPT_TEMP_TX_COLLECTION, col_filter) if not trans: raise BadRequestException("Cannot find the transaction by id.") # executing uploading or downloading data = None logging.info( f'handle transaction by id: is_ipfs={self.is_ipfs}, ' f'is_download={is_download}, file_name={trans["document"]["file_name"]}' ) if self.is_ipfs: if is_download: data = self.ipfs_files.download_file_with_path( target_did, target_app_did, trans['document']['file_name']) else: self.ipfs_files.upload_file_with_path( target_did, target_app_did, trans['document']['file_name']) else: if is_download: data = self.get_files().download_file_by_did( target_did, target_app_did, trans['document']['file_name']) else: self.get_files().upload_file_by_did( target_did, target_app_did, trans['document']['file_name']) # recalculate the storage usage of the database cli.delete_one(target_did, target_app_did, SCRIPTING_SCRIPT_TEMP_TX_COLLECTION, col_filter) update_used_storage_for_mongodb_data( target_did, get_mongo_database_size(target_did, target_app_did)) # return the content of the file return data
def execute(self): cli.check_vault_access(self.get_target_did(), VAULT_ACCESS_WR) document = self.get_document() msg = populate_with_params_values(self.get_did(), self.get_app_id(), document, self.get_params()) if msg: raise BadRequestException( msg='Cannot get parameter value for the executable document: ' + msg) data = cli.insert_one(self.get_target_did(), self.get_target_app_did(), self.get_collection_name(), document, populate_options_insert_one(self.body)) update_used_storage_for_mongodb_data( self.get_did(), get_mongo_database_size(self.get_target_did(), self.get_target_app_did())) return self.get_output_data(data)