def main(dry_run=True):
pending_embargoes = models.Embargo.find(Q('state', 'eq', models.Embargo.UNAPPROVED))
for embargo in pending_embargoes:
if should_be_embargoed(embargo):
if dry_run:
logger.warn('Dry run mode')
parent_registration = models.Node.find_one(Q('embargo', 'eq', embargo))
logger.warn(
'Embargo {0} approved. Activating embargo for registration {1}'
.format(embargo._id, parent_registration._id)
)
if not dry_run:
with TokuTransaction():
try:
embargo.state = models.Embargo.ACTIVE
parent_registration.registered_from.add_log(
action=NodeLog.EMBARGO_APPROVED,
params={
'node': parent_registration._id,
'embargo_id': embargo._id,
},
auth=None,
)
embargo.save()
except Exception as err:
logger.error(
'Unexpected error raised when activating embargo for '
'registration {}. Continuing...'.format(parent_registration))
logger.exception(err)
active_embargoes = models.Embargo.find(Q('state', 'eq', models.Embargo.ACTIVE))
for embargo in active_embargoes:
if embargo.end_date < datetime.datetime.utcnow():
if dry_run:
logger.warn('Dry run mode')
parent_registration = models.Node.find_one(Q('embargo', 'eq', embargo))
logger.warn(
'Embargo {0} complete. Making registration {1} public'
.format(embargo._id, parent_registration._id)
)
if not dry_run:
with TokuTransaction():
try:
parent_registration.set_privacy('public')
embargo.state = models.Embargo.COMPLETED
parent_registration.registered_from.add_log(
action=NodeLog.EMBARGO_COMPLETED,
params={
'node': parent_registration._id,
'embargo_id': embargo._id,
},
auth=None,
)
embargo.save()
except Exception as err:
logger.error(
'Unexpected error raised when completing embargo for '
'registration {}. Continuing...'.format(parent_registration))
logger.exception(err)
def do_migration(records, dry=False):
for node in records:
logs = list(NodeLog.find(Q('was_connected_to', 'contains', node)))
existing_logs = node.logs
for log in logs:
if not log.node__logged:
continue
log_node = log.node__logged[0]
# if the log_node is not contained in the node parent list then it doesn't belong to this node
if log_node not in get_all_parents(node):
logger.info(
'Excluding log {} from list because it is not associated with node {}'
.format(log, node))
logs.remove(log)
with TokuTransaction():
node.logs = logs + existing_logs
node.system_tags.append(SYSTEM_TAG)
node_type = 'registration' if node.is_registration else 'fork'
logger.info('Adding {} logs to {} {}'.format(
len(logs), node_type, node))
if not dry:
try:
node.save()
except Exception as err:
logger.error(
'Could not update logs for node {} due to error'.
format(node._id))
logger.exception(err)
logger.error('Skipping...')
def main(nworkers, worker_id, dry=True):
if not dry:
scripts_utils.add_file_logger(logger, __file__)
logger.info('Not running in dry mode, changes WILL be made')
else:
logger.info('Running in dry mode, changes NOT will be made')
to_migrate = model.OsfStorageNodeSettings.find(Q('_migrated_from_old_models', 'ne', True))
if to_migrate.count() == 0:
logger.info('No nodes to migrate; exiting...')
return
failed = 0
logger.info('Found {} nodes to migrate'.format(to_migrate.count()))
for node_settings in to_migrate:
if hash(node_settings._id) % nworkers != worker_id:
continue
try:
with TokuTransaction():
migrate_node_settings(node_settings, dry=dry)
migrate_children(node_settings, dry=dry)
if not dry:
node_settings.reload()
node_settings._migrated_from_old_models = True
node_settings.save()
except Exception as error:
logger.error('Could not migrate file tree from {}'.format(node_settings.owner._id))
logger.exception(error)
failed += 1
if failed > 0:
logger.error('Failed to migrate {} nodes'.format(failed))
def main():
init_app(routes=False)
dry_run = '--dry' in sys.argv
with TokuTransaction():
lowercase_nids()
if dry_run:
raise Exception('Dry run')
def modify_user_dates_in_mongo(new_date):
with TokuTransaction():
for user in database.user.find():
database['user'].find_and_modify(
{'_id': user['_id']}, {'$set': {
'date_registered': new_date
}})
def get_users_emails(send_type):
"""Get all emails that need to be sent.
:param send_type: from NOTIFICATION_TYPES
:return: [{
'user_id': 'se8ea',
'info': [{
'message': {
'message': 'Freddie commented on your project Open Science',
'timestamp': datetime object
},
'node_lineage': ['parent._id', 'node._id'],
'_id': NotificationDigest._id
}, ...
}]
},
{
'user_id': ...
}]
"""
with TokuTransaction():
emails = db['notificationdigest'].group(
key={'user_id': 1},
condition={'send_type': send_type},
initial={'info': []},
reduce=Code("""
function(curr, result) {
result.info.push({
'message': curr.message,
'node_lineage': curr.node_lineage,
'_id': curr._id
});
};
"""))
return emails
def main():
dry_run = '--dry' in sys.argv
if not dry_run:
script_utils.add_file_logger(logger, __file__)
init_app(set_backends=True, routes=False)
with TokuTransaction():
migrate(dry_run)
def main(dry=True):
init_app(set_backends=True,
routes=False) # Sets the storage backends on all models
with TokuTransaction():
do_migration()
if dry:
raise Exception('Abort Transaction - Dry Run')
def main(dry_run=True):
#find all emails to be sent, pops the top one for each user(to obey the once
#a week requirement), checks to see if one has been sent this week, and if
#not send the email, otherwise leave it in the queue
user_queue = {}
for email in find_queued_mails_ready_to_be_sent():
user_queue.setdefault(email.user._id, []).append(email)
emails_to_be_sent = pop_and_verify_mails_for_each_user(user_queue)
logger.info('Emails being sent at {0}'.format(
datetime.utcnow().isoformat()))
for mail in emails_to_be_sent:
if not dry_run:
with TokuTransaction():
try:
sent_ = mail.send_mail()
message = 'Email of type {0} sent to {1}'.format(mail.email_type, mail.to_addr) if sent_ else \
'Email of type {0} failed to be sent to {1}'.format(mail.email_type, mail.to_addr)
logger.info(message)
except Exception as error:
logger.error(
'Email of type {0} to be sent to {1} caused an ERROR'.
format(mail.email_type, mail.to_addr))
logger.exception(error)
pass
def main():
init_app(routes=False) # Sets the storage backends on all models
dry = '--dry' in sys.argv
if not dry:
script_utils.add_file_logger(logger, __file__)
with TokuTransaction():
log_duplicate_acount(dry)
def main(dry_run=True):
pending_RegistrationApprovals = models.RegistrationApproval.find(
Q('state', 'eq', models.RegistrationApproval.UNAPPROVED))
for registration_approval in pending_RegistrationApprovals:
if should_be_approved(registration_approval):
if dry_run:
logger.warn('Dry run mode')
pending_registration = models.Node.find_one(
Q('registration_approval', 'eq', registration_approval))
logger.warn(
'RegistrationApproval {0} automatically approved by system. Making registration {1} public.'
.format(registration_approval._id, pending_registration._id))
if not dry_run:
if pending_registration.is_deleted:
# Clean up any registration failures during archiving
registration_approval.forcibly_reject()
registration_approval.save()
continue
with TokuTransaction():
try:
# Ensure no `User` is associated with the final approval
registration_approval._on_complete(None)
except Exception as err:
logger.error(
'Unexpected error raised when approving registration for '
'registration {}. Continuing...'.format(
pending_registration))
logger.exception(err)
def main():
with TokuTransaction():
for file in BoxFile.find():
new_path = '/' + file.path.split('/')[1]
logger.info(u'{} -> {}'.format(file.path, new_path))
file.path = new_path
file.save()
def migrate(dry=True):
migrated = 0
pointers_with_invalid_backrefs = []
pointers = database.pointer.find({'$where': 'this._id.length <= 5'},
{'_id': True})
total = pointers.count()
for i, doc in enumerate(pointers):
pointer = Pointer.load(doc['_id'])
with TokuTransaction():
old_id = pointer._id
logger.info('({}/{}) Preparing to migrate Pointer {}'.format(
i + 1, total, old_id))
pointer._legacy_id = old_id
pointer._id = str(ObjectId())
try:
if not dry:
pointer.save()
except ValueError:
logger.warn(
'Removing backref for orphaned pointer: {}'.format(old_id))
if not dry:
remove_invalid_backref(pointer)
pointers_with_invalid_backrefs.append(old_id)
pointer.save()
logger.info('Successfully migrated Pointer {} _id to {}'.format(
old_id, pointer._id))
migrated += 1
logger.info('Successfully migrated {} pointers'.format(migrated))
logger.info('Removed invalid backrefs on {} pointers: {}'.format(
len(pointers_with_invalid_backrefs), pointers_with_invalid_backrefs))
def main():
parser = argparse.ArgumentParser(
description=
'Changes the guid of specified object and updates references in provided targets'
)
parser.add_argument(
'--dry',
action='store_true',
dest='dry_run',
help='Run migration and roll back changes to db',
)
parser.add_argument(
'--targets',
action='store',
dest='targets',
help="""Target JSON, of form
{
'data': {
'node': <target_id>', # Currently only supports nodes as target objects for new guids
'collections': {
'<collection>': [<_ids to update>]
}
}
}
""",
)
pargs = parser.parse_args()
if not pargs.dry_run:
script_utils.add_file_logger(logger, __file__)
init_app(set_backends=True, routes=False)
with TokuTransaction():
migrate(targets=json.loads(pargs.targets)['data'])
if pargs.dry_run:
raise RuntimeError('Dry run, transaction rolled back.')
def run_main(dry_run=True):
if not dry_run:
scripts_utils.add_file_logger(logger, __file__)
init_app(routes=False)
with TokuTransaction():
main()
if dry_run:
raise RuntimeError("Dry run, rolling back transaction")
def main(dry_run=False, test=False):
with TokuTransaction():
prereg_drafts = DraftRegistration.find(
Q('registration_schema', 'eq', PREREG_SCHEMA))
for draft in prereg_drafts:
migrate_draft_metadata(draft, test)
if dry_run:
raise RuntimeError("Dry run, rolling back transaction")
def main():
dry = '--dry' in sys.argv
script_utils.add_file_logger(logger, __file__)
init_app(set_backends=True, routes=False)
with TokuTransaction():
migrate()
if dry:
raise RuntimeError('Dry run -- Transaction rolled back')
def main():
dry_run = '--dry' in sys.argv
if dry_run:
logger.warn('DRY RUN mode')
else:
utils.add_file_logger(logger, __file__)
init_app(routes=False)
with TokuTransaction():
migrate(dry_run)
def do_migration(records, dry=False):
for user in records:
logger.info('Deleting user - {}'.format(user._id))
if not dry:
with TokuTransaction():
migrate_project_contributed(user)
user.is_disabled = True
user.save()
logger.info('{}Deleted {} users'.format('[dry]'if dry else '', len(records)))
def main():
init_app(routes=False) # Sets the storage backends on all models
dry = '--dry' in sys.argv
if not dry:
script_utils.add_file_logger(logger, __file__)
with TokuTransaction():
do_migration(get_targets(), dry)
if dry:
raise RuntimeError('Dry run, transaction rolled back')
def main():
dry_run = '--dry' in sys.argv
if not dry_run:
script_utils.add_file_logger(logger, __file__)
init_app(set_backends=True, routes=False)
with TokuTransaction():
migrate(dry_run=dry_run)
if dry_run:
raise RuntimeError('Dry run, transaction rolled back.')
def main():
init_app(routes=False)
dry_run = '--dry' in sys.argv
if dry_run:
logger.warn('Running a dry run')
if not dry_run:
script_utils.add_file_logger(logger, __file__)
with TokuTransaction():
migrate(dry=dry_run)
def main(dry_run=True):
legacy_objs = find_legacy_objs()
logger.info('Migrating {0} `OsfGuidFile` objects'.format(
legacy_objs.count()))
if dry_run:
return
for legacy_obj in legacy_objs:
with TokuTransaction():
migrate_legacy_obj(legacy_obj)
def modify_node_dates_in_mongo(new_date):
with TokuTransaction():
for node in database.node.find():
database['node'].find_and_modify(
{'_id': node['_id']},
{'$set': {
'date_created': new_date
}}
)
def main():
init_app(routes=False)
guid = sys.argv[1]
dry = '--dry' in sys.argv
if not dry:
script_utils.add_file_logger(logger, __file__)
with TokuTransaction():
restore_file(guid)
if dry:
raise RuntimeError('Dry run - rolling back transaction')
def main(dry):
if dry:
logger.info('[DRY MODE]')
init_app(routes=False)
for _id in FAILED_ARCHIVE_JOBS:
archive_job = ArchiveJob.load(_id)
assert archive_job.status == ARCHIVER_INITIATED
root_node = archive_job.dst_node.root
with TokuTransaction():
clean(reg=root_node, dry=dry)
def _update_node(self, node_id, updated_fields=None):
# Avoid circular imports
from framework.transactions.context import TokuTransaction
from website import models
node = models.Node.load(node_id)
try:
with TokuTransaction():
piwik._update_node_object(node, updated_fields)
except Exception as error:
raise self.retry(exc=error)
def main(dry_run=True):
nodes = get_nodes()
logger.info('Migrating files on {0} `Node` records'.format(len(nodes)))
for node in nodes:
try:
with TokuTransaction():
migrate_node(node, dry_run=dry_run)
except Exception as error:
logger.error('Could not migrate node {0}'.format(node._id))
logger.exception(error)
def main():
init_app(routes=False)
dry = '--dry' in sys.argv
if not dry:
# If we're not running in dry mode log everything to a file
script_utils.add_file_logger(logger, __file__)
with TokuTransaction():
migrate()
if dry:
raise Exception('Abort Transaction - Dry Run')
def main(dry=True):
init_app(set_backends=True,
routes=False) # Sets the storage backends on all models
# Start a transaction that will be rolled back if any exceptions are un
with TokuTransaction():
do_migration()
if dry:
# When running in dry mode force the transaction to rollback
raise Exception('Abort Transaction - Dry Run')
