def prep_comments():
"""
Migrate comments.
"""
print "Adjusting comment url_keys..."
col = settings.COL_COMMENTS
query = {'url_key': {'$type': 7}}
comments = mongo_find(col, query)
total = 0
for comment in comments:
_id = comment['_id']
url_key = str(comment['url_key'])
mongo_update(col, {'_id': ObjectId(_id)},
{'$set': {
'url_key': url_key
}})
total += 1
print "Fixed %s comments, correcting ObjectId url_key!\n" % total
query = {'obj_type': "Campaign", "url_key": {'$exists': 0}}
comments = mongo_find(col, query)
total = 0
for comment in comments:
_id = comment['_id']
obj = mongo_find_one(settings.COL_CAMPAIGNS,
{"_id": comment['obj_id']})
if obj:
url_key = obj['name']
mongo_update(col, {'_id': ObjectId(_id)},
{'$set': {
'url_key': url_key
}})
total += 1
print "Fixed %s comments, correcting url_key based on obj_id!\n" % total
def prep_comments():
"""
Migrate comments.
"""
print "Adjusting comment url_keys..."
col = settings.COL_COMMENTS
query = {'url_key': {'$type': 7}}
comments = mongo_find(col, query)
total = 0
for comment in comments:
_id = comment['_id']
url_key = str(comment['url_key'])
mongo_update(col,
{'_id': ObjectId(_id)},
{'$set': {'url_key': url_key}})
total += 1
print "Fixed %s comments, correcting ObjectId url_key!\n" % total
query = {'obj_type': "Campaign", "url_key": {'$exists': 0} }
comments = mongo_find(col, query)
total = 0
for comment in comments:
_id = comment['_id']
obj = mongo_find_one(settings.COL_CAMPAIGNS, {"_id": comment['obj_id']})
if obj:
url_key = obj['name']
mongo_update(col,
{'_id': ObjectId(_id)},
{'$set': {'url_key': url_key}})
total += 1
print "Fixed %s comments, correcting url_key based on obj_id!\n" % total
def prep_exploits():
"""
Migrate exploits.
"""
# fix exploits not having schema_versions
sv = Exploit._meta['latest_schema_version']
mongo_update(settings.COL_EXPLOIT_DETAILS,
{'schema_version': {'$exists': 0}},
{'$set': {'schema_version': sv}})
print "Fixed exploits without a schema!\n"
def prep_notifications():
"""
Update notifications.
"""
a1 = {"$unset": {"notifications": 1}}
a2 = {"$unset": {"unsupported_attrs.notifications": 1}}
mongo_update(settings.COL_USERS, {}, a1)
mongo_update(settings.COL_USERS, {}, a2)
query = {"type": "notification"}
mongo_remove(settings.COL_COMMENTS, query)
def prep_user_roles():
"""
Migrate user roles.
"""
# fix user_roles not having schema_versions
sv = UserRole._meta['latest_schema_version']
mongo_update(settings.COL_USER_ROLES,
{'schema_version': {'$exists': 0}},
{'$set': {'schema_version': sv}})
print "Fixed user roles without a schema!\n"
def prep_notifications():
"""
Update notifications.
"""
a1 = {"$unset": {"notifications": 1}}
a2 = {"$unset": {"unsupported_attrs.notifications": 1}}
mongo_update(settings.COL_USERS, {}, a1)
mongo_update(settings.COL_USERS, {}, a2)
query = {"type": "notification"}
mongo_remove(settings.COL_COMMENTS, query)
def prep_backdoors():
"""
Migrate backdoors.
"""
# fix backdoors not having schema_versions
sv = Backdoor._meta['latest_schema_version']
mongo_update(settings.COL_BACKDOOR_DETAILS,
{'schema_version': {'$exists': 0}},
{'$set': {'schema_version': sv}})
print "Fixed backdoors without a schema!\n"
def prep_yarahits():
"""
Migrate yara hits.
"""
# fix yarahits not having schema_versions
sv = YaraHit._meta['latest_schema_version']
mongo_update(settings.COL_YARAHITS,
{'schema_version': {'$exists': 0}},
{'$set': {'schema_version': sv}})
print "Fixed yara hits without a schema!\n"
def prep_indicator_actions():
"""
Migrate indicator actions.
"""
# fix indicator actions not having schema_versions
sv = IndicatorAction._meta['latest_schema_version']
mongo_update(settings.COL_IDB_ACTIONS,
{'schema_version': {'$exists': 0}},
{'$set': {'schema_version': sv}})
print "Fixed Indicator Actions without a schema!\n"
def prep_sources():
"""
Migrate sources.
"""
# fix source_access not having schema_versions
sv = SourceAccess._meta['latest_schema_version']
mongo_update(settings.COL_SOURCE_ACCESS,
{'schema_version': {'$exists': 0}},
{'$set': {'schema_version': sv}})
print "Fixed sources without a schema!\n"
def prep_divisions():
"""
Migrate divisions.
"""
# fix divisions not having schema_versions
sv = Division._meta['latest_schema_version']
mongo_update(settings.COL_DIVISION_DATA,
{'schema_version': {'$exists': 0}},
{'$set': {'schema_version': sv}})
print "Fixed divisions without a schema!\n"
def prep_indicator_actions():
"""
Migrate indicator actions.
"""
# fix indicator actions not having schema_versions
sv = IndicatorAction._meta['latest_schema_version']
mongo_update(settings.COL_IDB_ACTIONS, {'schema_version': {
'$exists': 0
}}, {'$set': {
'schema_version': sv
}})
print "Fixed Indicator Actions without a schema!\n"
def prep_yarahits():
"""
Migrate yara hits.
"""
# fix yarahits not having schema_versions
sv = YaraHit._meta['latest_schema_version']
mongo_update(settings.COL_YARAHITS, {'schema_version': {
'$exists': 0
}}, {'$set': {
'schema_version': sv
}})
print "Fixed yara hits without a schema!\n"
def prep_user_roles():
"""
Migrate user roles.
"""
# fix user_roles not having schema_versions
sv = UserRole._meta['latest_schema_version']
mongo_update(settings.COL_USER_ROLES, {'schema_version': {
'$exists': 0
}}, {'$set': {
'schema_version': sv
}})
print "Fixed user roles without a schema!\n"
def prep_divisions():
"""
Migrate divisions.
"""
# fix divisions not having schema_versions
sv = Division._meta['latest_schema_version']
mongo_update(settings.COL_DIVISION_DATA,
{'schema_version': {
'$exists': 0
}}, {'$set': {
'schema_version': sv
}})
print "Fixed divisions without a schema!\n"
def prep_exploits():
"""
Migrate exploits.
"""
# fix exploits not having schema_versions
sv = Exploit._meta['latest_schema_version']
mongo_update(settings.COL_EXPLOIT_DETAILS,
{'schema_version': {
'$exists': 0
}}, {'$set': {
'schema_version': sv
}})
print "Fixed exploits without a schema!\n"
def prep_sources():
"""
Migrate sources.
"""
# fix source_access not having schema_versions
sv = SourceAccess._meta['latest_schema_version']
mongo_update(settings.COL_SOURCE_ACCESS,
{'schema_version': {
'$exists': 0
}}, {'$set': {
'schema_version': sv
}})
print "Fixed sources without a schema!\n"
def prep_backdoors():
"""
Migrate backdoors.
"""
# fix backdoors not having schema_versions
sv = Backdoor._meta['latest_schema_version']
mongo_update(settings.COL_BACKDOOR_DETAILS,
{'schema_version': {
'$exists': 0
}}, {'$set': {
'schema_version': sv
}})
print "Fixed backdoors without a schema!\n"
def prep_audit_log():
"""
Migrate the audit log.
"""
print "Adjusting Audit Log Dates..."
col = settings.COL_AUDIT_LOG
schema_version = AuditLog._meta['latest_schema_version']
entries = mongo_find(col, {})
total = 0
for entry in entries:
changes = {}
changes['schema_version'] = schema_version
if 'date' in entry:
if not isinstance(entry['date'], datetime.datetime):
changes['date'] = parse(entry['date'], fuzzy=True)
total += 1
else:
changes['date'] = entry['date']
mongo_update(col, {'_id': ObjectId(entry['_id'])}, {'$set': changes})
print "Fixed %s audit log entries!\n" % total
def prep_audit_log():
"""
Migrate the audit log.
"""
print "Adjusting Audit Log Dates..."
col = settings.COL_AUDIT_LOG
schema_version = AuditLog._meta['latest_schema_version']
entries = mongo_find(col, {})
total = 0
for entry in entries:
changes = {}
changes['schema_version'] = schema_version
if 'date' in entry:
if not isinstance(entry['date'], datetime.datetime):
changes['date'] = parse(entry['date'], fuzzy=True)
total += 1
else:
changes['date'] = entry['date']
mongo_update(col,
{'_id': ObjectId(entry['_id'])},
{'$set': changes})
print "Fixed %s audit log entries!\n" % total