def _iterate_all_records(mongo_db: pymongo.database.Database,
mongo_fields: Iterable[_MongoField],
field_type: str) -> Iterator[MongoReference]:
db_collections = set(mongo_db.list_collection_names())
for collection, field_name in mongo_fields:
if collection not in db_collections:
logging.error('The collection "%s" does not exist.', collection)
continue
records = mongo_db.get_collection(collection).\
find({field_name: {'$exists': True}}, {field_name: 1})
has_at_least_one_field = False
has_at_least_one_value = False
for record in records:
field_value = record[field_name]
has_at_least_one_field = True
if not field_value:
continue
has_at_least_one_value = True
yield MongoReference(collection, field_name, field_value,
record['_id'])
if not has_at_least_one_field:
logging.error('The collection "%s" has no field "%s".', collection,
field_name)
continue
if not has_at_least_one_value:
logging.error('The collection "%s" has no %ss in its field "%s"',
collection, field_type, field_name)
def _iterate_all_records(
mongo_db: pymongo.database.Database,
mongo_fields: Iterable[_MongoField], field_type: str) \
-> Iterator[Tuple[str, Dict[str, Any], str]]:
db_collections = set(mongo_db.list_collection_names())
for collection, field_name in mongo_fields:
if collection not in db_collections:
logging.error('The collection "%s" does not exist.', collection)
continue
records = mongo_db.get_collection(collection).\
find({field_name: {'$exists': True}}, {field_name: 1})
if not records.count():
logging.error('The collection "%s" has no field "%s".', collection, field_name)
continue
has_at_least_one_field = False
for record in records:
field_value = record[field_name]
if not field_value:
continue
has_at_least_one_field = True
yield field_value, record, collection
if not has_at_least_one_field:
logging.error(
'The collection "%s" has no %ss in its field "%s"',
collection, field_type, field_name)
def ensure_indices(mongo_db: pymongo.database.Database) -> None:
"""Ensure that indices exist on relevant collections."""
db_collections = set(mongo_db.list_collection_names())
for collection, field in _INDICES:
if collection not in db_collections:
logging.error('The collection "%s" does not exist.', collection)
continue
mongo_db.get_collection(collection).create_index({field: 1})
def create_collection_if_not_exists(mongo_database: pymongo.database.Database, collection_name: str):
"""
Create a mongodb collection if it doesn't exist
"""
if collection_name in mongo_database.collection_names():
print('Collection "{collection}" already created'.format(collection=collection_name))
else:
mongo_database.create_collection(collection_name)
print('Created collection "{c}"'.format(c=collection_name))
def remove_alarm_from_jobs(db: pymongo.database.Database,
context: CallbackContext, alarm_id: str,
modification: str) -> Tuple[bool, str]:
jobs = context.job_queue.get_jobs_by_name(alarm_id)
if len(jobs) == 0:
return False, f"no alarm with alarm_id {alarm_id}"
for job in jobs:
db.update_alarm(alarm_id, modification)
job.schedule_removal()
return True, f"removed alarm with alarm_id {alarm_id}"
def remove_all_alarms_from_jobs(db: pymongo.database.Database,
context: CallbackContext,
modification: str) -> Tuple[bool, str]:
jobs = context.job_queue.jobs()
if len(jobs) == 0:
return False, f"no alarm to unset"
for job in jobs:
alarm_id = job.name
db.update_alarm(alarm_id, modification)
job.schedule_removal()
return True, f"all alarms unset"
def _check(base: pymongo.database.Database) -> (str, dict):
"""
Return Health checks for this Mongo database connection.
:param base: database object as returned by the _load method (Mandatory).
:return: A tuple with a string providing the status (pass, warn, fail), and the checks.
"""
try:
response = base.command("ping")
return (
"pass",
{
f"{base.name}:ping": {
"componentType": "datastore",
"observedValue": response,
"status": "pass",
"time": datetime.datetime.utcnow().isoformat(),
}
},
)
except Exception as e:
return (
"fail",
{
f"{base.name}:ping": {
"componentType": "datastore",
"status": "fail",
"time": datetime.datetime.utcnow().isoformat(),
"output": str(e),
}
},
)
def compute_collections_diff(
importers: Dict[str, Importer],
db_client: pymongo.database.Database) -> CollectionsDiff:
"""Determine which collections have been imported and which are missing."""
collection_names = {
name
for name in db_client.list_collection_names()
if name not in _MAINTENANCE_COLLECTIONS and not _is_archive(name)
}
is_personal = is_personal_database(collection_names)
personal_safe_importers = {
key: importer
for key, importer in importers.items()
if importer.has_pii == is_personal
}
importers_to_import = {
key
for key, importer in personal_safe_importers.items()
if importer.is_imported
}
return CollectionsDiff(
collection_missing=importers_to_import - collection_names,
importer_missing=collection_names - personal_safe_importers.keys(),
imported=collection_names & personal_safe_importers.keys(),
)
def insert_many_to_db(docs: list, db_collection: pymongo.database.Database):
''' inserts many documents to DB and skip those already in collection.'''
logging.debug('trying to insert {} documents to db'.format(len(docs)))
try:
db_collection.insert_many(docs, ordered=False)
except pymongo.errors.BulkWriteError as bwe:
for error in bwe.details['writeErrors']:
if error['code'] == 11000:
logging.debug('Document {} already in collection'.format(
error['op']['id']))
else:
logging.exception(
"unknown error while inserting to db, error code {}".
format(error['code']))
sys.exit()
logging.debug('inserted {} documents to db'.format(
bwe.details['nInserted']))
def populate_attack_mitigations(database: pymongo.database.Database,
cti_repo: Path):
database.create_collection(COLLECTION_NAME)
attack_data_path = cti_repo / "enterprise-attack"
stix2_mitigations = get_all_mitigations(attack_data_path)
mongo_mitigations = AttackMitigations.dict_from_stix2_attack_patterns(
get_all_attack_techniques(attack_data_path))
mitigation_technique_relationships = get_technique_and_mitigation_relationships(
attack_data_path)
for relationship in mitigation_technique_relationships:
mongo_mitigations[relationship["target_ref"]].add_mitigation(
stix2_mitigations[relationship["source_ref"]])
for relationship in mitigation_technique_relationships:
mongo_mitigations[relationship["target_ref"]].add_no_mitigations_info(
stix2_mitigations[relationship["source_ref"]])
for key, mongo_object in mongo_mitigations.items():
mongo_object.save()
def _reset(base: pymongo.database.Database) -> None:
"""
If the database was already created, then drop all tables and recreate them all.
:param base: database object as returned by the _load method (Mandatory).
"""
if base:
for collection in base.list_collection_names():
_reset_collection(base, collection)
def _iterate_translations_templates(mongo_db: pymongo.database.Database) \
-> Iterator[MongoReference]:
if 'translations' not in set(mongo_db.list_collection_names()):
logging.error(
'The database doe not contains any "translations" collection.')
return
for record in mongo_db.translations.find({}):
record.pop('_id')
string = record.pop('string')
for lang, translation in record.items():
if isinstance(translation, str) and '%' in translation:
yield MongoReference('translations', lang, translation, string)
def _revert_collection(collection_name: str, database: pymongo.database.Database) -> None:
archived_collections = sorted((
name for name in database.list_collection_names()
if _is_archive(name) and name.startswith(collection_name)), reverse=True)
if not archived_collections:
logging.error(
'No archived version of collection "%s" found, cannot revert.', collection_name)
return
name_length = len(collection_name)
archive = archived_collections[0]
archive_date = archive[name_length + 1:name_length + 11]
logging.info('Reverting collection "%s" to version from %s…', collection_name, archive_date)
database[archive].rename(collection_name, dropTarget=True)
def send_mail(
self, campaign_id: str, user: _UserProto, *, database: pymongo.database.Database,
users_database: pymongo.database.Database, now: datetime.datetime,
action: 'Action' = 'dry-run',
dry_run_email: Optional[str] = None,
mongo_user_update: Optional[Dict[str, Any]] = None) -> bool:
"""Send an email for this campaign."""
template_vars = self._get_vars(
user, database=database, users_database=users_database, now=now)
if not template_vars:
return False
collection = self._users_collection
if action == 'list':
user_id = collection.get_id(user)
logging.info('%s: %s %s', campaign_id, user_id, collection.get_profile(user).email)
return True
if action == 'dry-run':
collection.get_profile(user).email = dry_run_email or '*****@*****.**'
if action == 'ghost':
email_sent = user.emails_sent.add()
email_sent.sent_at.FromDatetime(now)
email_sent.sent_at.nanos = 0
email_sent.subject = get_campaign_subject(self._mailjet_template) or ''
else:
res = mail.send_template(
self._mailjet_template, collection.get_profile(user), template_vars,
sender_email=self._sender_email, sender_name=self._sender_name,
campaign_id=campaign_id)
logging.info('Email sent to %s', collection.get_profile(user).email)
res.raise_for_status()
maybe_email_sent = mail.create_email_sent_proto(res)
if not maybe_email_sent:
logging.warning('Impossible to retrieve the sent email ID:\n%s', res.json())
return False
if action == 'dry-run':
return True
email_sent = maybe_email_sent
email_sent.mailjet_template = self._mailjet_template
email_sent.campaign_id = campaign_id
if mongo_user_update and '$push' in mongo_user_update: # pragma: no-cover
raise ValueError(
f'$push operations are not allowed in mongo_user_update:\n{mongo_user_update}')
user_id = collection.get_id(user)
if user_id and action != 'ghost':
users_database.get_collection(collection.mongo_collection).update_one(
{'_id': objectid.ObjectId(user_id)},
dict(mongo_user_update or {}, **{'$push': {
'emailsSent': json_format.MessageToDict(email_sent),
}}))
# TODO(pascal): Clean that up or make it work in ghost mode.
if self._on_email_sent:
self._on_email_sent(
user, email_sent=email_sent, template_vars=template_vars,
database=database, user_database=users_database)
return True
def collection_exists(database: pymongo.database.Database,
collection_name: str) -> bool:
return collection_name in database.list_collection_names()
def clean_collection(database: pymongo.database.Database):
if collection_exists(database, COLLECTION_NAME):
database.drop_collection(COLLECTION_NAME)
def get_data_from_database(
database: pymongo.database.Database) -> pymongo.cursor.Cursor:
collection = database.get_collection(COLLECTION_NAME)
collection_contents = collection.find()
return collection_contents