def main(dry_run=True): pending_embargoes = models.Embargo.find(Q('state', 'eq', models.Embargo.UNAPPROVED)) for embargo in pending_embargoes: if should_be_embargoed(embargo): if dry_run: logger.warn('Dry run mode') parent_registration = models.Node.find_one(Q('embargo', 'eq', embargo)) logger.warn( 'Embargo {0} approved. Activating embargo for registration {1}' .format(embargo._id, parent_registration._id) ) if not dry_run: with TokuTransaction(): try: embargo.state = models.Embargo.ACTIVE parent_registration.registered_from.add_log( action=NodeLog.EMBARGO_APPROVED, params={ 'node': parent_registration._id, 'embargo_id': embargo._id, }, auth=None, ) embargo.save() except Exception as err: logger.error( 'Unexpected error raised when activating embargo for ' 'registration {}. Continuing...'.format(parent_registration)) logger.exception(err) active_embargoes = models.Embargo.find(Q('state', 'eq', models.Embargo.ACTIVE)) for embargo in active_embargoes: if embargo.end_date < datetime.datetime.utcnow(): if dry_run: logger.warn('Dry run mode') parent_registration = models.Node.find_one(Q('embargo', 'eq', embargo)) logger.warn( 'Embargo {0} complete. Making registration {1} public' .format(embargo._id, parent_registration._id) ) if not dry_run: with TokuTransaction(): try: parent_registration.set_privacy('public') embargo.state = models.Embargo.COMPLETED parent_registration.registered_from.add_log( action=NodeLog.EMBARGO_COMPLETED, params={ 'node': parent_registration._id, 'embargo_id': embargo._id, }, auth=None, ) embargo.save() except Exception as err: logger.error( 'Unexpected error raised when completing embargo for ' 'registration {}. Continuing...'.format(parent_registration)) logger.exception(err)
def do_migration(records, dry=False): for node in records: logs = list(NodeLog.find(Q('was_connected_to', 'contains', node))) existing_logs = node.logs for log in logs: if not log.node__logged: continue log_node = log.node__logged[0] # if the log_node is not contained in the node parent list then it doesn't belong to this node if log_node not in get_all_parents(node): logger.info( 'Excluding log {} from list because it is not associated with node {}' .format(log, node)) logs.remove(log) with TokuTransaction(): node.logs = logs + existing_logs node.system_tags.append(SYSTEM_TAG) node_type = 'registration' if node.is_registration else 'fork' logger.info('Adding {} logs to {} {}'.format( len(logs), node_type, node)) if not dry: try: node.save() except Exception as err: logger.error( 'Could not update logs for node {} due to error'. format(node._id)) logger.exception(err) logger.error('Skipping...')
def main(nworkers, worker_id, dry=True): if not dry: scripts_utils.add_file_logger(logger, __file__) logger.info('Not running in dry mode, changes WILL be made') else: logger.info('Running in dry mode, changes NOT will be made') to_migrate = model.OsfStorageNodeSettings.find(Q('_migrated_from_old_models', 'ne', True)) if to_migrate.count() == 0: logger.info('No nodes to migrate; exiting...') return failed = 0 logger.info('Found {} nodes to migrate'.format(to_migrate.count())) for node_settings in to_migrate: if hash(node_settings._id) % nworkers != worker_id: continue try: with TokuTransaction(): migrate_node_settings(node_settings, dry=dry) migrate_children(node_settings, dry=dry) if not dry: node_settings.reload() node_settings._migrated_from_old_models = True node_settings.save() except Exception as error: logger.error('Could not migrate file tree from {}'.format(node_settings.owner._id)) logger.exception(error) failed += 1 if failed > 0: logger.error('Failed to migrate {} nodes'.format(failed))
def main(): init_app(routes=False) dry_run = '--dry' in sys.argv with TokuTransaction(): lowercase_nids() if dry_run: raise Exception('Dry run')
def modify_user_dates_in_mongo(new_date): with TokuTransaction(): for user in database.user.find(): database['user'].find_and_modify( {'_id': user['_id']}, {'$set': { 'date_registered': new_date }})
def get_users_emails(send_type): """Get all emails that need to be sent. :param send_type: from NOTIFICATION_TYPES :return: [{ 'user_id': 'se8ea', 'info': [{ 'message': { 'message': 'Freddie commented on your project Open Science', 'timestamp': datetime object }, 'node_lineage': ['parent._id', 'node._id'], '_id': NotificationDigest._id }, ... }] }, { 'user_id': ... }] """ with TokuTransaction(): emails = db['notificationdigest'].group( key={'user_id': 1}, condition={'send_type': send_type}, initial={'info': []}, reduce=Code(""" function(curr, result) { result.info.push({ 'message': curr.message, 'node_lineage': curr.node_lineage, '_id': curr._id }); }; """)) return emails
def main(): dry_run = '--dry' in sys.argv if not dry_run: script_utils.add_file_logger(logger, __file__) init_app(set_backends=True, routes=False) with TokuTransaction(): migrate(dry_run)
def main(dry=True): init_app(set_backends=True, routes=False) # Sets the storage backends on all models with TokuTransaction(): do_migration() if dry: raise Exception('Abort Transaction - Dry Run')
def main(dry_run=True): #find all emails to be sent, pops the top one for each user(to obey the once #a week requirement), checks to see if one has been sent this week, and if #not send the email, otherwise leave it in the queue user_queue = {} for email in find_queued_mails_ready_to_be_sent(): user_queue.setdefault(email.user._id, []).append(email) emails_to_be_sent = pop_and_verify_mails_for_each_user(user_queue) logger.info('Emails being sent at {0}'.format( datetime.utcnow().isoformat())) for mail in emails_to_be_sent: if not dry_run: with TokuTransaction(): try: sent_ = mail.send_mail() message = 'Email of type {0} sent to {1}'.format(mail.email_type, mail.to_addr) if sent_ else \ 'Email of type {0} failed to be sent to {1}'.format(mail.email_type, mail.to_addr) logger.info(message) except Exception as error: logger.error( 'Email of type {0} to be sent to {1} caused an ERROR'. format(mail.email_type, mail.to_addr)) logger.exception(error) pass
def main(): init_app(routes=False) # Sets the storage backends on all models dry = '--dry' in sys.argv if not dry: script_utils.add_file_logger(logger, __file__) with TokuTransaction(): log_duplicate_acount(dry)
def main(dry_run=True): pending_RegistrationApprovals = models.RegistrationApproval.find( Q('state', 'eq', models.RegistrationApproval.UNAPPROVED)) for registration_approval in pending_RegistrationApprovals: if should_be_approved(registration_approval): if dry_run: logger.warn('Dry run mode') pending_registration = models.Node.find_one( Q('registration_approval', 'eq', registration_approval)) logger.warn( 'RegistrationApproval {0} automatically approved by system. Making registration {1} public.' .format(registration_approval._id, pending_registration._id)) if not dry_run: if pending_registration.is_deleted: # Clean up any registration failures during archiving registration_approval.forcibly_reject() registration_approval.save() continue with TokuTransaction(): try: # Ensure no `User` is associated with the final approval registration_approval._on_complete(None) except Exception as err: logger.error( 'Unexpected error raised when approving registration for ' 'registration {}. Continuing...'.format( pending_registration)) logger.exception(err)
def main(): with TokuTransaction(): for file in BoxFile.find(): new_path = '/' + file.path.split('/')[1] logger.info(u'{} -> {}'.format(file.path, new_path)) file.path = new_path file.save()
def migrate(dry=True): migrated = 0 pointers_with_invalid_backrefs = [] pointers = database.pointer.find({'$where': 'this._id.length <= 5'}, {'_id': True}) total = pointers.count() for i, doc in enumerate(pointers): pointer = Pointer.load(doc['_id']) with TokuTransaction(): old_id = pointer._id logger.info('({}/{}) Preparing to migrate Pointer {}'.format( i + 1, total, old_id)) pointer._legacy_id = old_id pointer._id = str(ObjectId()) try: if not dry: pointer.save() except ValueError: logger.warn( 'Removing backref for orphaned pointer: {}'.format(old_id)) if not dry: remove_invalid_backref(pointer) pointers_with_invalid_backrefs.append(old_id) pointer.save() logger.info('Successfully migrated Pointer {} _id to {}'.format( old_id, pointer._id)) migrated += 1 logger.info('Successfully migrated {} pointers'.format(migrated)) logger.info('Removed invalid backrefs on {} pointers: {}'.format( len(pointers_with_invalid_backrefs), pointers_with_invalid_backrefs))
def main(): parser = argparse.ArgumentParser( description= 'Changes the guid of specified object and updates references in provided targets' ) parser.add_argument( '--dry', action='store_true', dest='dry_run', help='Run migration and roll back changes to db', ) parser.add_argument( '--targets', action='store', dest='targets', help="""Target JSON, of form { 'data': { 'node': <target_id>', # Currently only supports nodes as target objects for new guids 'collections': { '<collection>': [<_ids to update>] } } } """, ) pargs = parser.parse_args() if not pargs.dry_run: script_utils.add_file_logger(logger, __file__) init_app(set_backends=True, routes=False) with TokuTransaction(): migrate(targets=json.loads(pargs.targets)['data']) if pargs.dry_run: raise RuntimeError('Dry run, transaction rolled back.')
def run_main(dry_run=True): if not dry_run: scripts_utils.add_file_logger(logger, __file__) init_app(routes=False) with TokuTransaction(): main() if dry_run: raise RuntimeError("Dry run, rolling back transaction")
def main(dry_run=False, test=False): with TokuTransaction(): prereg_drafts = DraftRegistration.find( Q('registration_schema', 'eq', PREREG_SCHEMA)) for draft in prereg_drafts: migrate_draft_metadata(draft, test) if dry_run: raise RuntimeError("Dry run, rolling back transaction")
def main(): dry = '--dry' in sys.argv script_utils.add_file_logger(logger, __file__) init_app(set_backends=True, routes=False) with TokuTransaction(): migrate() if dry: raise RuntimeError('Dry run -- Transaction rolled back')
def main(): dry_run = '--dry' in sys.argv if dry_run: logger.warn('DRY RUN mode') else: utils.add_file_logger(logger, __file__) init_app(routes=False) with TokuTransaction(): migrate(dry_run)
def do_migration(records, dry=False): for user in records: logger.info('Deleting user - {}'.format(user._id)) if not dry: with TokuTransaction(): migrate_project_contributed(user) user.is_disabled = True user.save() logger.info('{}Deleted {} users'.format('[dry]'if dry else '', len(records)))
def main(): init_app(routes=False) # Sets the storage backends on all models dry = '--dry' in sys.argv if not dry: script_utils.add_file_logger(logger, __file__) with TokuTransaction(): do_migration(get_targets(), dry) if dry: raise RuntimeError('Dry run, transaction rolled back')
def main(): dry_run = '--dry' in sys.argv if not dry_run: script_utils.add_file_logger(logger, __file__) init_app(set_backends=True, routes=False) with TokuTransaction(): migrate(dry_run=dry_run) if dry_run: raise RuntimeError('Dry run, transaction rolled back.')
def main(): init_app(routes=False) dry_run = '--dry' in sys.argv if dry_run: logger.warn('Running a dry run') if not dry_run: script_utils.add_file_logger(logger, __file__) with TokuTransaction(): migrate(dry=dry_run)
def main(dry_run=True): legacy_objs = find_legacy_objs() logger.info('Migrating {0} `OsfGuidFile` objects'.format( legacy_objs.count())) if dry_run: return for legacy_obj in legacy_objs: with TokuTransaction(): migrate_legacy_obj(legacy_obj)
def modify_node_dates_in_mongo(new_date): with TokuTransaction(): for node in database.node.find(): database['node'].find_and_modify( {'_id': node['_id']}, {'$set': { 'date_created': new_date }} )
def main(): init_app(routes=False) guid = sys.argv[1] dry = '--dry' in sys.argv if not dry: script_utils.add_file_logger(logger, __file__) with TokuTransaction(): restore_file(guid) if dry: raise RuntimeError('Dry run - rolling back transaction')
def main(dry): if dry: logger.info('[DRY MODE]') init_app(routes=False) for _id in FAILED_ARCHIVE_JOBS: archive_job = ArchiveJob.load(_id) assert archive_job.status == ARCHIVER_INITIATED root_node = archive_job.dst_node.root with TokuTransaction(): clean(reg=root_node, dry=dry)
def _update_node(self, node_id, updated_fields=None): # Avoid circular imports from framework.transactions.context import TokuTransaction from website import models node = models.Node.load(node_id) try: with TokuTransaction(): piwik._update_node_object(node, updated_fields) except Exception as error: raise self.retry(exc=error)
def main(dry_run=True): nodes = get_nodes() logger.info('Migrating files on {0} `Node` records'.format(len(nodes))) for node in nodes: try: with TokuTransaction(): migrate_node(node, dry_run=dry_run) except Exception as error: logger.error('Could not migrate node {0}'.format(node._id)) logger.exception(error)
def main(): init_app(routes=False) dry = '--dry' in sys.argv if not dry: # If we're not running in dry mode log everything to a file script_utils.add_file_logger(logger, __file__) with TokuTransaction(): migrate() if dry: raise Exception('Abort Transaction - Dry Run')
def main(dry=True): init_app(set_backends=True, routes=False) # Sets the storage backends on all models # Start a transaction that will be rolled back if any exceptions are un with TokuTransaction(): do_migration() if dry: # When running in dry mode force the transaction to rollback raise Exception('Abort Transaction - Dry Run')