def prep_comments(): """ Migrate comments. """ print "Adjusting comment url_keys..." col = settings.COL_COMMENTS query = {'url_key': {'$type': 7}} comments = mongo_find(col, query) total = 0 for comment in comments: _id = comment['_id'] url_key = str(comment['url_key']) mongo_update(col, {'_id': ObjectId(_id)}, {'$set': {'url_key': url_key}}) total += 1 print "Fixed %s comments, correcting ObjectId url_key!\n" % total query = {'obj_type': "Campaign", "url_key": {'$exists': 0} } comments = mongo_find(col, query) total = 0 for comment in comments: _id = comment['_id'] obj = mongo_find_one(settings.COL_CAMPAIGNS, {"_id": comment['obj_id']}) if obj: url_key = obj['name'] mongo_update(col, {'_id': ObjectId(_id)}, {'$set': {'url_key': url_key}}) total += 1 print "Fixed %s comments, correcting url_key based on obj_id!\n" % total
def prep_comments(): """ Migrate comments. """ print "Adjusting comment url_keys..." col = settings.COL_COMMENTS query = {'url_key': {'$type': 7}} comments = mongo_find(col, query) total = 0 for comment in comments: _id = comment['_id'] url_key = str(comment['url_key']) mongo_update(col, {'_id': ObjectId(_id)}, {'$set': { 'url_key': url_key }}) total += 1 print "Fixed %s comments, correcting ObjectId url_key!\n" % total query = {'obj_type': "Campaign", "url_key": {'$exists': 0}} comments = mongo_find(col, query) total = 0 for comment in comments: _id = comment['_id'] obj = mongo_find_one(settings.COL_CAMPAIGNS, {"_id": comment['obj_id']}) if obj: url_key = obj['name'] mongo_update(col, {'_id': ObjectId(_id)}, {'$set': { 'url_key': url_key }}) total += 1 print "Fixed %s comments, correcting url_key based on obj_id!\n" % total
def migrate_collection(class_obj, sort_ids): """ Migrate a collection by opening each document. This will, by nature of the core functionality in `crits.core.crits_mongoengine` check the schema_version and migrate it if it is not the latest version. :param class_obj: The class to migrate documents for. :type class_obj: class that inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :param sort_ids: If we should sort by ids ascending. :type sort_ids: boolean """ # find all documents that don't have the latest schema version # and migrate those. version = class_obj._meta['latest_schema_version'] print "\nMigrating %ss" % class_obj._meta['crits_type'] if sort_ids: docs = ( class_obj.objects(schema_version__lt=version) .order_by('+id') .timeout(False) ) total = docs.count() else: docs = class_obj.objects(schema_version__lt=version).timeout(False) total = docs.count() if not total: print "\tNo %ss to migrate!" % class_obj._meta['crits_type'] return print "\tMigrated 0 of %d" % total, count = 0 doc = None try: for doc in docs: if 'migrated' in doc._meta and doc._meta['migrated']: count += 1 print "\r\tMigrated %d of %d" % (count, total), print "" except Exception as e: # Provide some basic info so admin can query their db and figure out # what bad data is blowing up the migration. print "\n\n\tAn error occurred during migration!" print "\tMigrated: %d" % count formatted_lines = traceback.format_exc().splitlines() print "\tError: %s" % formatted_lines[-1] if hasattr(e, 'tlo'): print "\tDocument ID: %s" % e.tlo else: doc_id = mongo_find_one(class_obj._meta.get('collection'), {'schema_version': {'$lt': version}}, '_id') print "\tDocument ID: %s" % doc_id.get('_id') if doc: print "\tLast ID: %s" % doc.id sys.exit(1)
def migrate_collection(class_obj, sort_ids): """ Migrate a collection by opening each document. This will, by nature of the core functionality in `crits.core.crits_mongoengine` check the schema_version and migrate it if it is not the latest version. :param class_obj: The class to migrate documents for. :type class_obj: class that inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :param sort_ids: If we should sort by ids ascending. :type sort_ids: boolean """ # find all documents that don't have the latest schema version # and migrate those. version = class_obj._meta['latest_schema_version'] print "\nMigrating %ss" % class_obj._meta['crits_type'] if sort_ids: docs = (class_obj.objects( schema_version__lt=version).order_by('+id').timeout(False)) else: docs = class_obj.objects(schema_version__lt=version).timeout(False) total = docs.count() if not total: print "\tNo %ss to migrate!" % class_obj._meta['crits_type'] return print "\tMigrated 0 of %d" % total, count = 0 doc = None try: for doc in docs: if 'migrated' in doc._meta and doc._meta['migrated']: count += 1 print "\r\tMigrated %d of %d" % (count, total), print "" except Exception as e: # Provide some basic info so admin can query their db and figure out # what bad data is blowing up the migration. print "\n\n\tAn error occurred during migration!" print "\tMigrated: %d" % count formatted_lines = traceback.format_exc().splitlines() print "\tError: %s" % formatted_lines[-1] if hasattr(e, 'tlo'): print "\tDocument ID: %s" % e.tlo else: doc_id = mongo_find_one(class_obj._meta.get('collection'), {'schema_version': { '$lt': version }}, '_id') print "\tDocument ID: %s" % doc_id.get('_id') if doc: print "\tLast ID: %s" % doc.id sys.exit(1)