def _iterate_all_records(mongo_db: pymongo.database.Database, mongo_fields: Iterable[_MongoField], field_type: str) -> Iterator[MongoReference]: db_collections = set(mongo_db.list_collection_names()) for collection, field_name in mongo_fields: if collection not in db_collections: logging.error('The collection "%s" does not exist.', collection) continue records = mongo_db.get_collection(collection).\ find({field_name: {'$exists': True}}, {field_name: 1}) has_at_least_one_field = False has_at_least_one_value = False for record in records: field_value = record[field_name] has_at_least_one_field = True if not field_value: continue has_at_least_one_value = True yield MongoReference(collection, field_name, field_value, record['_id']) if not has_at_least_one_field: logging.error('The collection "%s" has no field "%s".', collection, field_name) continue if not has_at_least_one_value: logging.error('The collection "%s" has no %ss in its field "%s"', collection, field_type, field_name)
def _iterate_all_records( mongo_db: pymongo.database.Database, mongo_fields: Iterable[_MongoField], field_type: str) \ -> Iterator[Tuple[str, Dict[str, Any], str]]: db_collections = set(mongo_db.list_collection_names()) for collection, field_name in mongo_fields: if collection not in db_collections: logging.error('The collection "%s" does not exist.', collection) continue records = mongo_db.get_collection(collection).\ find({field_name: {'$exists': True}}, {field_name: 1}) if not records.count(): logging.error('The collection "%s" has no field "%s".', collection, field_name) continue has_at_least_one_field = False for record in records: field_value = record[field_name] if not field_value: continue has_at_least_one_field = True yield field_value, record, collection if not has_at_least_one_field: logging.error( 'The collection "%s" has no %ss in its field "%s"', collection, field_type, field_name)
def ensure_indices(mongo_db: pymongo.database.Database) -> None: """Ensure that indices exist on relevant collections.""" db_collections = set(mongo_db.list_collection_names()) for collection, field in _INDICES: if collection not in db_collections: logging.error('The collection "%s" does not exist.', collection) continue mongo_db.get_collection(collection).create_index({field: 1})
def create_collection_if_not_exists(mongo_database: pymongo.database.Database, collection_name: str): """ Create a mongodb collection if it doesn't exist """ if collection_name in mongo_database.collection_names(): print('Collection "{collection}" already created'.format(collection=collection_name)) else: mongo_database.create_collection(collection_name) print('Created collection "{c}"'.format(c=collection_name))
def remove_alarm_from_jobs(db: pymongo.database.Database, context: CallbackContext, alarm_id: str, modification: str) -> Tuple[bool, str]: jobs = context.job_queue.get_jobs_by_name(alarm_id) if len(jobs) == 0: return False, f"no alarm with alarm_id {alarm_id}" for job in jobs: db.update_alarm(alarm_id, modification) job.schedule_removal() return True, f"removed alarm with alarm_id {alarm_id}"
def remove_all_alarms_from_jobs(db: pymongo.database.Database, context: CallbackContext, modification: str) -> Tuple[bool, str]: jobs = context.job_queue.jobs() if len(jobs) == 0: return False, f"no alarm to unset" for job in jobs: alarm_id = job.name db.update_alarm(alarm_id, modification) job.schedule_removal() return True, f"all alarms unset"
def _check(base: pymongo.database.Database) -> (str, dict): """ Return Health checks for this Mongo database connection. :param base: database object as returned by the _load method (Mandatory). :return: A tuple with a string providing the status (pass, warn, fail), and the checks. """ try: response = base.command("ping") return ( "pass", { f"{base.name}:ping": { "componentType": "datastore", "observedValue": response, "status": "pass", "time": datetime.datetime.utcnow().isoformat(), } }, ) except Exception as e: return ( "fail", { f"{base.name}:ping": { "componentType": "datastore", "status": "fail", "time": datetime.datetime.utcnow().isoformat(), "output": str(e), } }, )
def compute_collections_diff( importers: Dict[str, Importer], db_client: pymongo.database.Database) -> CollectionsDiff: """Determine which collections have been imported and which are missing.""" collection_names = { name for name in db_client.list_collection_names() if name not in _MAINTENANCE_COLLECTIONS and not _is_archive(name) } is_personal = is_personal_database(collection_names) personal_safe_importers = { key: importer for key, importer in importers.items() if importer.has_pii == is_personal } importers_to_import = { key for key, importer in personal_safe_importers.items() if importer.is_imported } return CollectionsDiff( collection_missing=importers_to_import - collection_names, importer_missing=collection_names - personal_safe_importers.keys(), imported=collection_names & personal_safe_importers.keys(), )
def insert_many_to_db(docs: list, db_collection: pymongo.database.Database): ''' inserts many documents to DB and skip those already in collection.''' logging.debug('trying to insert {} documents to db'.format(len(docs))) try: db_collection.insert_many(docs, ordered=False) except pymongo.errors.BulkWriteError as bwe: for error in bwe.details['writeErrors']: if error['code'] == 11000: logging.debug('Document {} already in collection'.format( error['op']['id'])) else: logging.exception( "unknown error while inserting to db, error code {}". format(error['code'])) sys.exit() logging.debug('inserted {} documents to db'.format( bwe.details['nInserted']))
def populate_attack_mitigations(database: pymongo.database.Database, cti_repo: Path): database.create_collection(COLLECTION_NAME) attack_data_path = cti_repo / "enterprise-attack" stix2_mitigations = get_all_mitigations(attack_data_path) mongo_mitigations = AttackMitigations.dict_from_stix2_attack_patterns( get_all_attack_techniques(attack_data_path)) mitigation_technique_relationships = get_technique_and_mitigation_relationships( attack_data_path) for relationship in mitigation_technique_relationships: mongo_mitigations[relationship["target_ref"]].add_mitigation( stix2_mitigations[relationship["source_ref"]]) for relationship in mitigation_technique_relationships: mongo_mitigations[relationship["target_ref"]].add_no_mitigations_info( stix2_mitigations[relationship["source_ref"]]) for key, mongo_object in mongo_mitigations.items(): mongo_object.save()
def _reset(base: pymongo.database.Database) -> None: """ If the database was already created, then drop all tables and recreate them all. :param base: database object as returned by the _load method (Mandatory). """ if base: for collection in base.list_collection_names(): _reset_collection(base, collection)
def _iterate_translations_templates(mongo_db: pymongo.database.Database) \ -> Iterator[MongoReference]: if 'translations' not in set(mongo_db.list_collection_names()): logging.error( 'The database doe not contains any "translations" collection.') return for record in mongo_db.translations.find({}): record.pop('_id') string = record.pop('string') for lang, translation in record.items(): if isinstance(translation, str) and '%' in translation: yield MongoReference('translations', lang, translation, string)
def _revert_collection(collection_name: str, database: pymongo.database.Database) -> None: archived_collections = sorted(( name for name in database.list_collection_names() if _is_archive(name) and name.startswith(collection_name)), reverse=True) if not archived_collections: logging.error( 'No archived version of collection "%s" found, cannot revert.', collection_name) return name_length = len(collection_name) archive = archived_collections[0] archive_date = archive[name_length + 1:name_length + 11] logging.info('Reverting collection "%s" to version from %s…', collection_name, archive_date) database[archive].rename(collection_name, dropTarget=True)
def send_mail( self, campaign_id: str, user: _UserProto, *, database: pymongo.database.Database, users_database: pymongo.database.Database, now: datetime.datetime, action: 'Action' = 'dry-run', dry_run_email: Optional[str] = None, mongo_user_update: Optional[Dict[str, Any]] = None) -> bool: """Send an email for this campaign.""" template_vars = self._get_vars( user, database=database, users_database=users_database, now=now) if not template_vars: return False collection = self._users_collection if action == 'list': user_id = collection.get_id(user) logging.info('%s: %s %s', campaign_id, user_id, collection.get_profile(user).email) return True if action == 'dry-run': collection.get_profile(user).email = dry_run_email or '*****@*****.**' if action == 'ghost': email_sent = user.emails_sent.add() email_sent.sent_at.FromDatetime(now) email_sent.sent_at.nanos = 0 email_sent.subject = get_campaign_subject(self._mailjet_template) or '' else: res = mail.send_template( self._mailjet_template, collection.get_profile(user), template_vars, sender_email=self._sender_email, sender_name=self._sender_name, campaign_id=campaign_id) logging.info('Email sent to %s', collection.get_profile(user).email) res.raise_for_status() maybe_email_sent = mail.create_email_sent_proto(res) if not maybe_email_sent: logging.warning('Impossible to retrieve the sent email ID:\n%s', res.json()) return False if action == 'dry-run': return True email_sent = maybe_email_sent email_sent.mailjet_template = self._mailjet_template email_sent.campaign_id = campaign_id if mongo_user_update and '$push' in mongo_user_update: # pragma: no-cover raise ValueError( f'$push operations are not allowed in mongo_user_update:\n{mongo_user_update}') user_id = collection.get_id(user) if user_id and action != 'ghost': users_database.get_collection(collection.mongo_collection).update_one( {'_id': objectid.ObjectId(user_id)}, dict(mongo_user_update or {}, **{'$push': { 'emailsSent': json_format.MessageToDict(email_sent), }})) # TODO(pascal): Clean that up or make it work in ghost mode. if self._on_email_sent: self._on_email_sent( user, email_sent=email_sent, template_vars=template_vars, database=database, user_database=users_database) return True
def collection_exists(database: pymongo.database.Database, collection_name: str) -> bool: return collection_name in database.list_collection_names()
def clean_collection(database: pymongo.database.Database): if collection_exists(database, COLLECTION_NAME): database.drop_collection(COLLECTION_NAME)
def get_data_from_database( database: pymongo.database.Database) -> pymongo.cursor.Cursor: collection = database.get_collection(COLLECTION_NAME) collection_contents = collection.find() return collection_contents