def prep_comments():
"""
Migrate comments.
"""
print "Adjusting comment url_keys..."
col = settings.COL_COMMENTS
query = {'url_key': {'$type': 7}}
comments = mongo_find(col, query)
total = 0
for comment in comments:
_id = comment['_id']
url_key = str(comment['url_key'])
mongo_update(col,
{'_id': ObjectId(_id)},
{'$set': {'url_key': url_key}})
total += 1
print "Fixed %s comments, correcting ObjectId url_key!\n" % total
query = {'obj_type': "Campaign", "url_key": {'$exists': 0} }
comments = mongo_find(col, query)
total = 0
for comment in comments:
_id = comment['_id']
obj = mongo_find_one(settings.COL_CAMPAIGNS, {"_id": comment['obj_id']})
if obj:
url_key = obj['name']
mongo_update(col,
{'_id': ObjectId(_id)},
{'$set': {'url_key': url_key}})
total += 1
print "Fixed %s comments, correcting url_key based on obj_id!\n" % total
def prep_comments():
"""
Migrate comments.
"""
print "Adjusting comment url_keys..."
col = settings.COL_COMMENTS
query = {'url_key': {'$type': 7}}
comments = mongo_find(col, query)
total = 0
for comment in comments:
_id = comment['_id']
url_key = str(comment['url_key'])
mongo_update(col, {'_id': ObjectId(_id)},
{'$set': {
'url_key': url_key
}})
total += 1
print "Fixed %s comments, correcting ObjectId url_key!\n" % total
query = {'obj_type': "Campaign", "url_key": {'$exists': 0}}
comments = mongo_find(col, query)
total = 0
for comment in comments:
_id = comment['_id']
obj = mongo_find_one(settings.COL_CAMPAIGNS,
{"_id": comment['obj_id']})
if obj:
url_key = obj['name']
mongo_update(col, {'_id': ObjectId(_id)},
{'$set': {
'url_key': url_key
}})
total += 1
print "Fixed %s comments, correcting url_key based on obj_id!\n" % total
def migrate_collection(class_obj, sort_ids):
"""
Migrate a collection by opening each document. This will, by nature of the
core functionality in `crits.core.crits_mongoengine` check the
schema_version and migrate it if it is not the latest version.
:param class_obj: The class to migrate documents for.
:type class_obj: class that inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param sort_ids: If we should sort by ids ascending.
:type sort_ids: boolean
"""
# find all documents that don't have the latest schema version
# and migrate those.
version = class_obj._meta['latest_schema_version']
print "\nMigrating %ss" % class_obj._meta['crits_type']
if sort_ids:
docs = (
class_obj.objects(schema_version__lt=version)
.order_by('+id')
.timeout(False)
)
total = docs.count()
else:
docs = class_obj.objects(schema_version__lt=version).timeout(False)
total = docs.count()
if not total:
print "\tNo %ss to migrate!" % class_obj._meta['crits_type']
return
print "\tMigrated 0 of %d" % total,
count = 0
doc = None
try:
for doc in docs:
if 'migrated' in doc._meta and doc._meta['migrated']:
count += 1
print "\r\tMigrated %d of %d" % (count, total),
print ""
except Exception as e:
# Provide some basic info so admin can query their db and figure out
# what bad data is blowing up the migration.
print "\n\n\tAn error occurred during migration!"
print "\tMigrated: %d" % count
formatted_lines = traceback.format_exc().splitlines()
print "\tError: %s" % formatted_lines[-1]
if hasattr(e, 'tlo'):
print "\tDocument ID: %s" % e.tlo
else:
doc_id = mongo_find_one(class_obj._meta.get('collection'),
{'schema_version': {'$lt': version}}, '_id')
print "\tDocument ID: %s" % doc_id.get('_id')
if doc:
print "\tLast ID: %s" % doc.id
sys.exit(1)
def migrate_collection(class_obj, sort_ids):
"""
Migrate a collection by opening each document. This will, by nature of the
core functionality in `crits.core.crits_mongoengine` check the
schema_version and migrate it if it is not the latest version.
:param class_obj: The class to migrate documents for.
:type class_obj: class that inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param sort_ids: If we should sort by ids ascending.
:type sort_ids: boolean
"""
# find all documents that don't have the latest schema version
# and migrate those.
version = class_obj._meta['latest_schema_version']
print "\nMigrating %ss" % class_obj._meta['crits_type']
if sort_ids:
docs = (class_obj.objects(
schema_version__lt=version).order_by('+id').timeout(False))
else:
docs = class_obj.objects(schema_version__lt=version).timeout(False)
total = docs.count()
if not total:
print "\tNo %ss to migrate!" % class_obj._meta['crits_type']
return
print "\tMigrated 0 of %d" % total,
count = 0
doc = None
try:
for doc in docs:
if 'migrated' in doc._meta and doc._meta['migrated']:
count += 1
print "\r\tMigrated %d of %d" % (count, total),
print ""
except Exception as e:
# Provide some basic info so admin can query their db and figure out
# what bad data is blowing up the migration.
print "\n\n\tAn error occurred during migration!"
print "\tMigrated: %d" % count
formatted_lines = traceback.format_exc().splitlines()
print "\tError: %s" % formatted_lines[-1]
if hasattr(e, 'tlo'):
print "\tDocument ID: %s" % e.tlo
else:
doc_id = mongo_find_one(class_obj._meta.get('collection'),
{'schema_version': {
'$lt': version
}}, '_id')
print "\tDocument ID: %s" % doc_id.get('_id')
if doc:
print "\tLast ID: %s" % doc.id
sys.exit(1)
