def write_to_mongo(self, data): try: results = self.mongo.insert(data) if results and results.count() > 0: return True else: raise PyMongoError("Could Not Insert Document " + str(data)) except PyMongoError as pyme: raise PyMongoError(pyme)
def borrar_evento(self, id_evento): try: return self.bd_mongo.eventos.remove({"_id": ObjectId(id_evento)}) except PyMongoError as pyerr: raise PyMongoError( u'Error borrando evento de BD (pymongo). {0}'.format( pyerr.message))
def _get_relatives(citizen_id: int, import_id: int, db: Database, session: ClientSession) -> Set[int]: """ Возвращает сет родственников указанного жителя в указанной поставке. :param int citizen_id: Уникальный идентификатор жителя, чьих родственников необходимо вернуть :param int import_id: Уникальный индетификатор поставки, в которой ищется житель :param Database db: объект базы данных, в которую записываются наборы данных о жителях :param ClientSession session: сессия соединения с базой данных, через которую производятся все запросы :raises: :class:`PyMongoError`: Объект с указанным уникальным идентификатором не был найден в базе данных :return: сет родственников указанного жителя в указанной поставке :rtype: Set[int] """ db_response: dict = db['imports'].find_one( { 'import_id': import_id, 'citizens.citizen_id': citizen_id }, {'citizens': { '$elemMatch': { 'citizen_id': citizen_id } }}, session=session) if db_response is None: raise PyMongoError('Import or citizen with specified id not found') relatives = set(db_response['citizens'][0]['relatives']) return relatives
def registerVehicle(path, postBody): response = {} try: fleet_id = path.split("/")[1] if not(validFleet(ObjectId(fleet_id))): raise PyMongoError(f"Fleet with ID: {fleet_id} does not exist") vehicleClass = Vehicle(postBody['vehicle_model'], postBody['license_plate'], postBody['vehicle_status'], ObjectId(fleet_id) ) vehicleObj = vehicleClass.get_register_data() # 2. Open a new database connection client = mongoConnect() # 3. write data from the database db = client.team12_supply # check vehicle = db.vehicle if (vehicle.count_documents({'license_plate': vehicleObj['license_plate']}) == 0): vehicleID = vehicle.insert_one(vehicleObj).inserted_id response = {'status': 'OK', 'data': {'fleet_id': vehicleObj['fleet_id'], 'vehicle_model': vehicleObj['vehicle_model'], 'license_plate': vehicleObj[ 'license_plate'], 'vehicle_status': vehicleObj['vehicle_status'], '_id': vehicleID, "current_location": vehicleObj["current_location"]}} else: response = {'status': 'CONFLICT', 'data': {'msg': 'Vehicle already registered'}} except PyMongoError as err: response = {'status': 'CONFLICT', 'data': { 'msg': err}} except ValueError as err: response = {'status': 'CONFLICT', 'data': { 'msg': err}} except Exception as err: logging.error(err) response = {'status': 'INTERNAL_SERVER_ERROR', 'data': { 'msg': 'Server stopped working, please try again later'}} return response
def _check_all_citizens_exist(citizens_ids: Set[int], import_id: int, db: Database, session: ClientSession): """ Проверяет наличие всех жителей, указанных в relatives_ids в поставке с идентификатором import_id. :param Set[int] citizens_ids: Уникальные идентификаторы жителей :param int import_id: Уникальный идентификатор поставки :param Database db: объект базы данных, в которую записываются наборы данных о жителях :param ClientSession session: сессия соединения с базой данных, через которую производятся все запросы :raises: :class:`PyMongoError`: Объект с указанным уникальным идентификатором не был найден в базе данных """ if not citizens_ids: return count = db['imports'].count_documents( { 'import_id': import_id, 'citizens.citizen_id': { '$all': list(citizens_ids) } }, session=session, limit=1) if count == 0: raise PyMongoError('Citizens with specified id not found')
def test_get_db_error(self): totest = UserCars() totest.db.find = Mock(side_effect=PyMongoError("server error")) result = UserCars().get("nomeacaso") self.assertEqual({"executed": False}, result)
def insertar_evento(self, evento): try: return self.bd_mongo.eventos.insert_one(evento.to_json_obj()) except PyMongoError as pyerr: raise PyMongoError( u'Error insertando evento en BD (pymongo). {0}'.format( pyerr.message))
def actualizar_evento(self, evento): try: return self.bd_mongo.eventos.update_one( {"_id": ObjectId(evento['_id'])}, {"$set": evento}) except PyMongoError as pyerr: raise PyMongoError( u'Error actualizando evento en BD (pymongo). {0}'.format( pyerr.message))
def test_get_contract_context_error(self, get_collection_mock): uid = "123" task = Mock(retry=Retry) get_collection_mock.return_value.find_one.side_effect = PyMongoError( "Connection error") with self.assertRaises(Retry): get_contract_context(task, uid)
def delete_document(self, select_cond, **kwargs): if not isinstance(select_cond, dict): select_cond = {} if self.collection: result = self.collection.remove(select_cond, j=True, **kwargs) return result else: raise PyMongoError("No collection specified")
def test_update_organisations_error(self, get_collection_mock): get_collection_mock.return_value.bulk_write.side_effect = PyMongoError( "Connection error") task = Mock(retry=Retry) records = [] with self.assertRaises(Retry): update_organisations(task, records)
def test_save_contract_context_error(self, get_collection_mock): uid = "123" task = Mock(retry=Retry) data = ["ja ja"] get_collection_mock.return_value.update_one.side_effect = PyMongoError( "Connection error") with self.assertRaises(Retry): save_contract_context(task, uid, data)
def remove_case(case_id, owner, mongo_db='variantDatabase', username=None, password=None, port=27017, host='localhost', logger=None): """Delete variants and users from the mongo database.""" # get root path of the Flask app # project_root = '/'.join(app.root_path.split('/')[0:-1]) if not logger: logger = logging.getLogger(__name__) logger.info('Trying to access collection {0}'.format(mongo_db)) connection = connect(mongo_db, host=host, port=port, username=username, password=password) logger.debug('Connection successful') case_mongo_id = '_'.join([owner, case_id]) databases = connection.database_names() logger.info("Databases found: {0}".format(', '.join(databases))) if mongo_db in databases: logger.info("Accsessing {0}".format(mongo_db)) db = connection[mongo_db] logger.debug("Access succesful") logger.info( "Trying to find case {0} in database".format(case_mongo_id)) case = Case.objects(case_id=case_mongo_id) if case: logger.info("Case found! Deleting case") case.delete() logger.debug("Case deleted") else: logger.info("Case {0} not found in database".format(case_mongo_id)) nr_of_variants = 0 for variant in Variant.objects(case_id=case_mongo_id): logger.debug("Found variant {0}".format(variant)) variant.delete() logger.debug("Variant deleted") nr_of_variants += 1 if nr_of_variants == 0: logger.info("No variants that belong to case found in database") else: logger.info("Deleted {0} variants".format(nr_of_variants)) else: raise PyMongoError('{0} database does not exist'.format(mongo_db))
def test_init_organisations_index(self, get_collection_mock): get_collection_mock.return_value.create_index.side_effect = PyMongoError( "Connection error") with self.assertRaises(Retry): init_organisations_index() get_collection_mock.assert_called_once_with(TREASURY_ORG_COLLECTION) get_collection_mock.return_value.create_index.assert_called_once_with( ORG_UNIQUE_FIELD)
def getFleetId(service_type): client = mongoConnect() fleet = client.team12_supply.fleet doc = fleet.find_one({"service_type": service_type}) if doc != None: fleetId = doc['_id'] return fleetId else: raise PyMongoError( f"Fleet with service type, {service_type}, does not exist")
def insertar_medida_boya_en_evento(self, id_evento, medida): try: return self.bd_mongo.eventos.update_one( {"_id": ObjectId(id_evento)}, {"$push": { "datosPuertos": medida.to_json_obj() }}) except PyMongoError as pyerr: raise PyMongoError( u'Error insertando medida de boya en evento(pymongo). {0}'. format(pyerr.message))
def delete_fields(self, select_cond, delete_condition, **kwargs): if not isinstance(select_cond, dict): select_cond = {} if self.collection: result = self.collection.update(select_cond, {'$unset': update_args}, j=True, **kwargs) return result else: raise PyMongoError("No collection specified")
def inner(*args, **kwargs) -> Callable: try: func(*args, **kwargs) except PyMongoError as e: error(e.__str__()) raise PyMongoError('Erro ao executar a operação.') except TypeError as e: error(e.__str__()) raise finally: from .connectionDatabase import ConnectionDatabase ConnectionDatabase().get_connection.close() return func(*args, **kwargs)
def insertar_ejecucion(cliente_mongo, ejecucion_ok, nombre_operacional): try: fecha_actual = util.fecha_actual(cfg.FORMATO_FECHA) ejecucion = { 'fecha': fecha_actual, 'estado': ejecucion_ok, 'tipo': nombre_operacional } return cliente_mongo.ejecuciones.insert_one(ejecucion) except PyMongoError as pyerr: raise PyMongoError( u'Error insertando datos de la ejecución en BD (pymongo). {0}'. format(pyerr.message))
def exec_func(*args, **kwargs): try: temp_func = func(*args, **kwargs) except PyMongoError as pme: raise PyMongoError(pme) except Exception as e: raise Exception(e) finally: try: self.close() except: pass return temp_func
def _write_relatives_update(db_requests: List[UpdateMany], db: Database, session: ClientSession): """ При наличии запросов в db_request производит их запись в базу данных. :param List[UpdateMany] db_requests: Список запросов к базе данных на обновление множества документов :param Database db: объект базы данных, в которую записываются наборы данных о жителях :param ClientSession session: сессия соединения с базой данных, через которую производятся все запросы :raises: :class:`PyMongoError`: Объект с указанным уникальным идентификатором не был найден в базе данных """ if db_requests: bulk_response: BulkWriteResult = db['imports'].bulk_write( db_requests, session=session) if bulk_response.modified_count != len(db_requests): raise PyMongoError('Import with specified id not found')
def keep_alive(self, token): #result= self.read_from_mongo({"token": token, "allocation_time" : {"$gt" : time.time()-timeout}}) try: result = self.mongo.update({'token': token}, { 'allocation_time': time.time(), 'validation_time': time.time() }) print result if result and result['nModified'] > 0 and result['updatedExisting']: return {"status": "Success", "status_code": status.HTTP_200_OK} return { "status": "No token found!!", "status_code": status.HTTP_404_NOT_FOUND } except PyMongoError as pme: raise PyMongoError(pme)
def insertar_tweets_en_evento(self, id_evento, tweets): try: return self.bd_mongo.eventos.update_one( {'_id': ObjectId(id_evento)}, { "$push": { 'datosTwitter': { '$each': tweets, '$sort': { 'id': -1 } } } }) except PyMongoError as pyerr: raise PyMongoError( u'Error insertando tweets en evento(pymongo). {0}'.format( pyerr.message))
def _write_to_db(import_data: dict, db: Database) -> Tuple[dict, int]: """ Производит запись набора данных о жителях в базу данных. :param import_data: валидированный набор данных о жителях :param db: объект базы данных, в которую записываются наборы данных о жителях :raises: :class:`PyMongoError`: Операция записи в базу данных не была разрешена :returns: В случае успеха возвращается пару из ответа с идентификатором импорта и http кода 201 :rtype: Tuple[dict, int] """ db_response: InsertOneResult = db['imports'].insert_one(import_data) if db_response.acknowledged: response = {'data': {'import_id': import_data['import_id']}} return response, 201 else: raise PyMongoError('Operation was not acknowledged')
def get_citizens(import_id: int, db: Database, projection: dict = None) -> List[dict]: """ Возвращает список жителей в указанной поставке, выбранный с указанной проекцией. :param int import_id: уникальный идентификатор поставки :param Database db: объект базы данных, в которую записываются наборы данных о жителях :param dict projection: словарь проекции выборки :raises :class:`PyMongoError`: Поставка с указанным уникальным идентификатором остутствует в базе данных :return: Список жителей :rtype: List[dict] """ import_data = db['imports'].find_one({'import_id': import_id}, projection) if import_data is None: raise PyMongoError('Import with specified id not found') return import_data['citizens']
def delete_tok(self, tokens): locked = None try: locked = self.lock.acquire() deleted = self.mongo.delete_document({"token": {"$in": tokens}}) print deleted if deleted and deleted['n'] == 1: return {"status": "Success", "status_code": status.HTTP_200_OK} else: return { "status": "No token found", "status_code": status.HTTP_404_NOT_FOUND } except PyMongoError as pme: raise PyMongoError(pme) finally: if locked: self.lock.release()
def update_token(self, query_cond, update_cond): locked = None try: locked = self.lock.acquire() result = self.mongo.update(query_cond, update_cond) if result and result['nModified'] > 0 and result['updatedExisting']: return True return False except PyMongoError as pme: raise PyMongoError(pme) except Exception as e: raise Exception(e) finally: try: if locked: self.lock.release() except: pass
def _write_citizen_update(citizen_id: int, import_id: int, patch_data: dict, db: Database, session: ClientSession) -> dict: """ Записывает обновление информации о жителе в базу данных. :param int citizen_id: Уникальный идентификатор модифицируемого жителя :param int import_id: Уникальный идентификатор поставки :param dict patch_data: Новая информация о жителе :param Database db: объект базы данных, в которую записываются наборы данных о жителях :param ClientSession session: сессия соединения с базой данных, через которую производятся все запросы :raises: :class:`PyMongoError`: Объект с указанным уникальным идентификатором не был найден в базе данных :return: Обновленная информация о жителе :rtype: dict """ update_data = { '$set': {f'citizens.$.{key}': val for key, val in patch_data.items()} } projection = { '_id': 0, 'import_id': 0, 'citizens': { '$elemMatch': { 'citizen_id': citizen_id } } } db_response: dict = db['imports'].find_one_and_update( filter={ 'import_id': import_id, 'citizens.citizen_id': citizen_id }, update=update_data, projection=projection, return_document=ReturnDocument.AFTER, session=session) if db_response is None: raise PyMongoError('Import or citizen with specified id not found') return db_response
class MongoDBInstanceTestCase(TestCase): """ Test cases for MongoDBInstanceMixin and OpenEdXDatabaseMixin """ def setUp(self): super().setUp() self.instance = None def tearDown(self): if self.instance: with patch( 'instance.tests.models.factories.openedx_instance.OpenEdXInstance._write_metadata_to_consul', return_value=(1, True) ): self.instance.deprovision_mongo() super().tearDown() def check_mongo(self): """ Check that the instance mongo user has access to the external mongo database """ mongo = pymongo.MongoClient(settings.DEFAULT_INSTANCE_MONGO_URL) for database in self.instance.mongo_database_names: self.assertTrue(mongo[database].authenticate(self.instance.mongo_user, self.instance.mongo_pass)) def check_mongo_vars_not_set(self, appserver): """ Check that the given OpenEdXAppServer does not point to a mongo database """ for var in ('EDXAPP_MONGO_USER', 'EDXAPP_MONGO_PASSWORD' 'EDXAPP_MONGO_HOSTS', 'EDXAPP_MONGO_PORT', 'EDXAPP_MONGO_DB_NAME', 'FORUM_MONGO_USER', 'FORUM_MONGO_PASSWORD', 'FORUM_MONGO_HOSTS', 'FORUM_MONGO_PORT', 'FORUM_MONGO_DATABASE'): self.assertNotIn(var, appserver.configuration_settings) def check_mongo_vars_set(self, appserver, expected_hosts, expected_replica_set=None): """ Check that the given OpenEdXAppServer is using the expected mongo settings. """ ansible_vars = appserver.configuration_settings ansible_vars_data = yaml.safe_load(ansible_vars) self.assertIn('EDXAPP_MONGO_USER: {0}'.format(self.instance.mongo_user), ansible_vars) self.assertIn('EDXAPP_MONGO_PASSWORD: {0}'.format(self.instance.mongo_pass), ansible_vars) self.assertIn('EDXAPP_MONGO_PORT: {0}'.format(MONGODB_SERVER_DEFAULT_PORT), ansible_vars) self.assertIn('EDXAPP_MONGO_DB_NAME: {0}'.format(self.instance.mongo_database_name), ansible_vars) if isinstance(ansible_vars_data['EDXAPP_MONGO_HOSTS'], list): rendered_edxapp_mongo_hosts = ansible_vars_data['EDXAPP_MONGO_HOSTS'] else: rendered_edxapp_mongo_hosts = ansible_vars_data['EDXAPP_MONGO_HOSTS'].split(',') self.assertSetEqual(set(expected_hosts), set(rendered_edxapp_mongo_hosts)) if expected_replica_set: self.assertIn('EDXAPP_MONGO_REPLICA_SET: {0}'.format(expected_replica_set), ansible_vars) self.assertIn('FORUM_MONGO_REPLICA_SET: {0}'.format(expected_replica_set), ansible_vars) self.assertIn('FORUM_MONGO_URL', ansible_vars_data) self.assertSetEqual(set(ansible_vars_data['FORUM_MONGO_HOSTS']), set(expected_hosts)) else: self.assertNotIn('EDXAPP_MONGO_REPLICA_SET', ansible_vars) self.assertNotIn('FORUM_MONGO_REPLICA_SET', ansible_vars) self.assertEqual(ansible_vars_data['FORUM_MONGO_HOSTS'], expected_hosts) self.assertIn('FORUM_MONGO_USER: {0}'.format(self.instance.mongo_user), ansible_vars) self.assertIn('FORUM_MONGO_PASSWORD: {0}'.format(self.instance.mongo_pass), ansible_vars) self.assertIn('FORUM_MONGO_PORT: {0}'.format(MONGODB_SERVER_DEFAULT_PORT), ansible_vars) self.assertIn('FORUM_MONGO_DATABASE: {0}'.format(self.instance.forum_database_name), ansible_vars) def test_provision_mongo(self, mock_consul): """ Provision mongo databases """ self.instance = OpenEdXInstanceFactory() self.instance.provision_mongo() self.check_mongo() def test_provision_mongo_again(self, mock_consul): """ Only create the databases once """ self.instance = OpenEdXInstanceFactory() self.instance.provision_mongo() self.assertIs(self.instance.mongo_provisioned, True) mongo_user = self.instance.mongo_user mongo_pass = self.instance.mongo_pass self.instance.provision_mongo() self.assertEqual(self.instance.mongo_user, mongo_user) self.assertEqual(self.instance.mongo_pass, mongo_pass) self.check_mongo() def test_provision_mongo_no_mongodb_server(self, mock_consul): """ Don't provision a mongo database if instance has no MongoDB server """ mongo = pymongo.MongoClient(settings.DEFAULT_INSTANCE_MONGO_URL) self.instance = OpenEdXInstanceFactory() self.instance.mongodb_server = None self.instance.save() self.instance.provision_mongo() databases = mongo.database_names() for database in self.instance.mongo_database_names: self.assertNotIn(database, databases) @override_settings(DEFAULT_INSTANCE_MONGO_URL='mongodb://*****:*****@mongo.opencraft.com') def test_ansible_settings_mongo(self, mock_consul): """ Add mongo ansible vars if instance has a MongoDB server """ # Delete MongoDBServer object created during the migrations to allow the settings override # to take effect. MongoDBServer.objects.all().delete() self.instance = OpenEdXInstanceFactory() appserver = make_test_appserver(self.instance) self.check_mongo_vars_set(appserver, expected_hosts=['mongo.opencraft.com']) @override_settings( DEFAULT_INSTANCE_MONGO_URL=None, DEFAULT_MONGO_REPLICA_SET_NAME="test_name", DEFAULT_MONGO_REPLICA_SET_USER="******", DEFAULT_MONGO_REPLICA_SET_PASSWORD="******", DEFAULT_MONGO_REPLICA_SET_PRIMARY="test.opencraft.hosting", DEFAULT_MONGO_REPLICA_SET_HOSTS="test.opencraft.hosting,test1.opencraft.hosting,test2.opencraft.hosting" ) @ddt.data( ('open-release/ficus', 'open-release/ficus'), ('open-release/ficus', 'opencraft-release/ficus'), ('open-release/ginkgo', 'open-release/ginkgo'), ) @ddt.unpack def test_ansible_settings_no_replica_set(self, openedx_release, configuration_version, mock_consul): """ Prior to Hawthorn, edx configuration does not support MongoDB replica sets, and the mongo hosts must be a single host, provided as a list of strings. """ # Delete MongoDBServer object created during the migrations to allow the settings override # to take effect. MongoDBServer.objects.all().delete() self.instance = OpenEdXInstanceFactory(openedx_release=openedx_release, configuration_version=configuration_version) appserver = make_test_appserver(self.instance) self.check_mongo_vars_set(appserver, expected_hosts=["test.opencraft.hosting"]) @override_settings( DEFAULT_INSTANCE_MONGO_URL=None, DEFAULT_MONGO_REPLICA_SET_NAME="test_name", DEFAULT_MONGO_REPLICA_SET_USER="******", DEFAULT_MONGO_REPLICA_SET_PASSWORD="******", DEFAULT_MONGO_REPLICA_SET_PRIMARY="test.opencraft.hosting", DEFAULT_MONGO_REPLICA_SET_HOSTS="test.opencraft.hosting,test1.opencraft.hosting,test2.opencraft.hosting" ) @ddt.data( ('open-release/ginkgo', 'opencraft-release/ginkgo'), (settings.OPENEDX_RELEASE_STABLE_REF, settings.STABLE_CONFIGURATION_VERSION), (settings.DEFAULT_OPENEDX_RELEASE, settings.DEFAULT_CONFIGURATION_VERSION), ) @ddt.unpack def test_ansible_settings_use_replica_set(self, openedx_release, configuration_version, mock_consul): """ Add mongo ansible vars if instance has a MongoDB replica set Also, the mongo hosts are provied as a comma-separated string. """ # Delete MongoDBServer object created during the migrations to allow the settings override # to take effect. MongoDBServer.objects.all().delete() self.instance = OpenEdXInstanceFactory(openedx_release=openedx_release, configuration_version=configuration_version) appserver = make_test_appserver(self.instance) self.check_mongo_vars_set(appserver, expected_hosts=['test.opencraft.hosting', 'test1.opencraft.hosting', 'test2.opencraft.hosting'], expected_replica_set='test_name') def test_ansible_settings_no_mongo_server(self, mock_consul): """ Don't add mongo ansible vars if instance has no MongoDB server """ self.instance = OpenEdXInstanceFactory() self.instance.mongodb_server = None self.instance.save() appserver = make_test_appserver(self.instance) self.check_mongo_vars_not_set(appserver) @override_settings( DEFAULT_INSTANCE_MONGO_URL=None, DEFAULT_MONGO_REPLICA_SET_NAME="test_name", DEFAULT_MONGO_REPLICA_SET_USER="******", DEFAULT_MONGO_REPLICA_SET_PASSWORD="******", DEFAULT_MONGO_REPLICA_SET_PRIMARY="test.opencraft.hosting", DEFAULT_MONGO_REPLICA_SET_HOSTS="test.opencraft.hosting,test1.opencraft.hosting,test2.opencraft.hosting" ) def test__get_main_database_url(self, mock_consul): """ Main database url should be extracted from primary replica set MongoDBServer """ self.instance = OpenEdXInstanceFactory() self.assertEqual( self.instance._get_main_database_url(), "mongodb://*****:*****@test.opencraft.hosting" ) @patch('instance.models.mixins.database.MongoDBInstanceMixin._get_main_database_url') @patch('instance.models.mixins.database.pymongo.MongoClient', autospec=True) def test_deprovision_mongo(self, mock_mongo_client_cls, mock_get_main_db_url, mock_consul): """ Test deprovision_mongo calls drop_database. """ self.instance = OpenEdXInstanceFactory() self.instance.mongo_provisioned = True self.instance.deprovision_mongo() for database in self.instance.mongo_database_names: mock_mongo_client_cls().drop_database.assert_any_call(database) @patch('instance.models.mixins.database.pymongo.MongoClient', autospec=True) @patch('instance.models.mixins.database.MongoDBInstanceMixin._get_main_database_url') @patch('instance.models.mixins.database.pymongo.MongoClient.drop_database', side_effect=PyMongoError()) def test_ignore_errors_deprovision_mongo(self, mock_mongo_client_cls, *mock_methods): """ Test mongo is set as deprovision when ignoring errors. """ self.instance = OpenEdXInstanceFactory() self.instance.mongo_provisioned = True self.instance.deprovision_mongo(ignore_errors=True) self.assertFalse(self.instance.mongo_provisioned)
def test_get_db_error_creation(self): to_mock = UserStats() to_mock.db.insert = Mock(side_effect=PyMongoError("db error")) to_mock.db.find_one = Mock(return_value=None) response = UserStats().get("notpippo") self.assertEqual({"executed": False}, response)