def check_registrations(registration_ids):
for r_id in registration_ids:
reg = Registration.load(r_id)
if not reg:
logger.warn('Registration {} not found'.format(r_id))
else:
check(reg)
def check_registrations(registration_ids):
for r_id in registration_ids:
reg = Registration.load(r_id)
if not reg:
logger.warn('Registration {} not found'.format(r_id))
else:
check(reg)
def create(self, validated_data):
inst = self.context['view'].get_object()['self']
user = self.context['request'].user
registration_dicts = validated_data['data']
changes_flag = False
for registration_dict in registration_dicts:
registration = Registration.load(registration_dict['_id'])
if not registration:
raise exceptions.NotFound(
detail='Registration with id "{}" was not found'.format(
registration_dict['_id']))
if not registration.has_permission(user, osf_permissions.WRITE):
raise exceptions.PermissionDenied(
detail='Write permission on registration {} required'.
format(registration_dict['_id']))
if not registration.is_affiliated_with_institution(inst):
registration.add_affiliated_institution(inst, user, save=True)
changes_flag = True
if not changes_flag:
raise RelationshipPostMakesNoChanges
return {
'data':
list(inst.nodes.filter(is_deleted=False, type='osf.registration')),
'self':
inst,
}
def test_sanction_embargo_termination_first(self):
embargo_termination_approval = factories.EmbargoTerminationApprovalFactory(
)
registration = Registration.find_one(
Q('embargo_termination_approval', 'eq',
embargo_termination_approval))
assert registration.sanction == embargo_termination_approval
def setUp(self):
self.maxDiff = None
super(TestRegistrationUpdate, self).setUp()
self.user = AuthUserFactory()
self.user_two = AuthUserFactory()
self.user_three = AuthUserFactory()
self.registration_approval = RegistrationApprovalFactory(state='unapproved', approve=False, user=self.user)
self.unapproved_registration = Registration.find_one(Q('registration_approval', 'eq', self.registration_approval))
self.unapproved_url = '/{}registrations/{}/'.format(API_BASE, self.unapproved_registration._id)
self.public_project = ProjectFactory(title="Project One", is_public=True, creator=self.user)
self.private_project = ProjectFactory(title="Project Two", is_public=False, creator=self.user)
self.public_registration = RegistrationFactory(project=self.public_project, creator=self.user, is_public=True)
self.private_registration = RegistrationFactory(project=self.private_project, creator=self.user)
self.public_url = '/{}registrations/{}/'.format(API_BASE, self.public_registration._id)
self.private_url = '/{}registrations/{}/'.format(API_BASE, self.private_registration._id)
self.private_registration.add_contributor(self.user_two, permissions=[permissions.READ])
self.private_registration.add_contributor(self.user_three, permissions=[permissions.WRITE])
self.private_registration.save()
self.payload = {
"data": {
"id": self.private_registration._id,
"type": "registrations",
"attributes": {
"public": True,
}
}
}
def sync_registration_creator_bibliographic_status(registration_guid):
registration = Registration.load(registration_guid)
creator = registration.creator
creator_contributor_reg = registration.contributor_set.get(user=creator)
creator_contributor_node = registration.registered_from.contributor_set.get(user=creator)
creator_contributor_reg.visible = creator_contributor_node.visible
creator_contributor_reg.save()
def analyze_failed_registration_nodes():
""" If we can just retry the archive, but we can only do that if the
ORIGINAL node hasn't changed.
"""
# Get the registrations that are messed up
failed_registration_nodes = find_failed_registrations()
# Build up a list of dictionaries with info about these failed nodes
failed_registration_info = []
for broken_registration in failed_registration_nodes:
unacceptable_node_logs_after_date = list(
broken_registration.registered_from.get_aggregate_logs_queryset(Auth(broken_registration.registered_from.creator))
.filter(date__gt=broken_registration.registered_date)
.exclude(action__in=fa.LOG_WHITELIST)
.exclude(action__in=fa.LOG_GREYLIST)
.values_list('action', flat=True)
)
# Does it have any addons?
addon_list = [
addon for addon in ADDONS_REQUESTED
if broken_registration.registered_from.has_addon(addon)
and addon not in {'osfstorage', 'wiki'}
]
has_addons = bool(addon_list)
# Any registrations succeeded after the stuck one?
# Not sure why broken_registration.registered_from.registrations was always 0 locally...
succeeded_registrations_after_failed = []
for other_reg in Registration.find(
Q('registered_from', 'eq', broken_registration.registered_from) &
Q('registered_date', 'gt', broken_registration.registered_date)
):
if other_reg.sanction:
if other_reg.sanction.is_approved:
succeeded_registrations_after_failed.append(other_reg._id)
else:
succeeded_registrations_after_failed.append(other_reg._id)
can_be_reset = fa.verify(broken_registration)
logger.info('Found broken registration {}'.format(broken_registration._id))
failed_registration_info.append(
{
'registration': broken_registration._id,
'registered_date': broken_registration.registered_date,
'original_node': broken_registration.registered_from._id,
'logs_on_original_after_registration_date': unacceptable_node_logs_after_date,
'has_addons': has_addons,
'addon_list': addon_list,
'succeeded_registrations_after_failed': succeeded_registrations_after_failed,
'can_be_reset': can_be_reset,
'registered_from_public': broken_registration.registered_from.is_public,
}
)
return failed_registration_info
def analyze_failed_registration_nodes():
""" If we can just retry the archive, but we can only do that if the
ORIGINAL node hasn't changed.
"""
# Get the registrations that are messed up
failed_registration_nodes = Registration.find_failed_registrations()
# Build up a list of dictionaries with info about these failed nodes
failed_registration_info = []
for broken_registration in failed_registration_nodes:
unacceptable_node_logs_after_date = list(
broken_registration.registered_from.get_logs_queryset(Auth(broken_registration.registered_from.creator))
.filter(date__gt=broken_registration.registered_date)
.exclude(action__in=fa.LOG_WHITELIST)
.exclude(action__in=fa.LOG_GREYLIST)
.values_list('action', flat=True)
)
# Does it have any addons?
addon_list = [
addon for addon in ADDONS_REQUESTED
if broken_registration.registered_from.has_addon(addon)
and addon not in {'osfstorage', 'wiki'}
]
has_addons = bool(addon_list)
# Any registrations succeeded after the stuck one?
# Not sure why broken_registration.registered_from.registrations was always 0 locally...
succeeded_registrations_after_failed = []
for other_reg in Registration.objects.filter(
registered_from=broken_registration.registered_from,
registered_date__gt=broken_registration.registered_date
):
if other_reg.sanction:
if other_reg.sanction.is_approved:
succeeded_registrations_after_failed.append(other_reg._id)
else:
succeeded_registrations_after_failed.append(other_reg._id)
can_be_reset = fa.verify(broken_registration)
logger.info('Found broken registration {}'.format(broken_registration._id))
failed_registration_info.append(
{
'registration': broken_registration._id,
'registered_date': broken_registration.registered_date,
'original_node': broken_registration.registered_from._id,
'logs_on_original_after_registration_date': unacceptable_node_logs_after_date,
'has_addons': has_addons,
'addon_list': addon_list,
'succeeded_registrations_after_failed': succeeded_registrations_after_failed,
'can_be_reset': can_be_reset,
'registered_from_public': broken_registration.registered_from.is_public,
}
)
return failed_registration_info
def verify_registrations(registration_ids):
for r_id in registration_ids:
reg = Registration.load(r_id)
if not reg:
logger.warn('Registration {} not found'.format(r_id))
else:
if verify(reg):
VERIFIED.append(reg)
else:
SKIPPED.append(reg)
def verify_registrations(registration_ids):
for r_id in registration_ids:
reg = Registration.load(r_id)
if not reg:
logger.warn('Registration {} not found'.format(r_id))
else:
if verify(reg):
VERIFIED.append(reg)
else:
SKIPPED.append(reg)
def test_register_draft_without_embargo_creates_registration_approval(self, mock_enqueue):
res = self.app.post(
self.project.api_url_for('register_draft_registration', draft_id=self.draft._id),
self.valid_make_public_payload,
content_type='application/json',
auth=self.user.auth
)
assert_equal(res.status_code, 202)
registration = Registration.find().order_by('-registered_date').first()
assert_not_equal(registration.registration_approval, None)
def test_register_draft_without_embargo_creates_registration_approval(self, mock_enqueue):
res = self.app.post(
self.project.api_url_for('register_draft_registration', draft_id=self.draft._id),
self.valid_make_public_payload,
content_type='application/json',
auth=self.user.auth
)
assert_equal(res.status_code, 202)
registration = Registration.find().order_by('-registered_date').first()
assert_not_equal(registration.registration_approval, None)
def post(self, request, *args, **kwargs):
stuck_reg = self.get_object()
if Registration.find_failed_registrations().filter(
id=stuck_reg.id).exists():
stuck_reg.delete_registration_tree(save=True)
messages.success(request, 'The registration has been deleted')
else:
messages.error(
request, 'This registration may not technically be stuck,'
' if the problem persists get a developer to fix it.')
return redirect(self.get_success_url())
def test_register_draft_registration_with_embargo_is_not_public(
self, mock_enqueue):
res = self.app.post_json(self.node.api_url_for(
'register_draft_registration', draft_id=self.draft._id),
self.embargo_payload,
auth=self.user.auth)
assert_equal(res.status_code, http.ACCEPTED)
registration = Registration.find().order_by('-registered_date').first()
assert_false(registration.is_public)
assert_true(registration.is_pending_embargo)
assert_is_not_none(registration.embargo)
def perform_destroy(self, instance):
data = self.request.data['data']
user = self.request.user
ids = [datum['id'] for datum in data]
registrations = []
for id_ in ids:
registration = Registration.load(id_)
if not registration.has_permission(user, osf_permissions.WRITE):
raise exceptions.PermissionDenied(detail='Write permission on registration {} required'.format(id_))
registrations.append(registration)
for registration in registrations:
registration.remove_affiliated_institution(inst=instance['self'], user=user)
registration.save()
def perform_destroy(self, instance):
data = self.request.data['data']
user = self.request.user
ids = [datum['id'] for datum in data]
registrations = []
for id_ in ids:
registration = Registration.load(id_)
if not registration.has_permission(user, osf_permissions.WRITE):
raise exceptions.PermissionDenied(detail='Write permission on registration {} required'.format(id_))
registrations.append(registration)
for registration in registrations:
registration.remove_affiliated_institution(inst=instance['self'], user=user)
registration.save()
def test_sync_different_registration_creator_bibliographic_status(
self, user, project, registration, project_contributor,
registration_creator_contrib):
# Assert out-of-sync bibliographic status
assert project_contributor.visible is False
assert registration_creator_contrib.visible is True
registration_guid = registration._id
sync_registration_creator_bibliographic_status(registration_guid)
updated_registration = Registration.load(registration_guid)
updated_registration_creator_contrib = updated_registration.contributor_set.get(
user=registration.creator)
assert updated_registration_creator_contrib.visible is False
assert updated_registration_creator_contrib.visible == project_contributor.visible
def test_register_draft_registration_with_embargo_is_not_public(self, mock_enqueue):
res = self.app.post_json(
self.node.api_url_for('register_draft_registration', draft_id=self.draft._id),
self.embargo_payload,
auth=self.user.auth
)
assert_equal(res.status_code, http.ACCEPTED)
registration = Registration.find().order_by('-registered_date').first()
assert_false(registration.is_public)
assert_true(registration.is_pending_embargo)
assert_is_not_none(registration.embargo)
def test_POST_register_embargo_is_not_public(self, mock_enqueue):
res = self.app.post(self.project.api_url_for(
'register_draft_registration', draft_id=self.draft._id),
self.valid_embargo_payload,
content_type='application/json',
auth=self.user.auth)
assert_equal(res.status_code, 202)
registration = Registration.find().sort('-registered_date').first()
assert_false(registration.is_public)
assert_true(registration.is_pending_embargo_for_existing_registration)
assert_is_not_none(registration.embargo)
def test_POST_register_embargo_is_not_public(self, mock_enqueue):
res = self.app.post(
self.project.api_url_for('register_draft_registration', draft_id=self.draft._id),
self.valid_embargo_payload,
content_type='application/json',
auth=self.user.auth
)
assert_equal(res.status_code, 202)
registration = Registration.find().order_by('-registered_date').first()
assert_false(registration.is_public)
assert_true(registration.is_pending_embargo_for_existing_registration)
assert_is_not_none(registration.embargo)
def test_sync_same_registration_creator_bibliographic_status(
self, user, project, registration, registration_creator_contrib):
# Assert aligned bibliographic status
project_contributor = registration.registered_from.contributor_set.get(
user=user)
assert project_contributor.visible is True
assert registration_creator_contrib.visible is True
registration_guid = registration._id
sync_registration_creator_bibliographic_status(registration_guid)
updated_registration = Registration.load(registration_guid)
updated_registration_creator_contrib = updated_registration.contributor_set.get(
user=registration.creator)
assert updated_registration_creator_contrib.visible is True
assert updated_registration_creator_contrib.visible == project_contributor.visible
def main():
args = parse_args()
dry = args.dry_run
if not dry:
# If we're not running in dry mode log everything to a file
script_utils.add_file_logger(logger, __file__)
else:
logger.info('Running in dry mode...')
checked_ok, checked_stuck = [], []
force_failed = []
for reg_id in args.registration_ids:
logger.info('Processing registration {}'.format(reg_id))
reg = Registration.load(reg_id)
is_stuck = not check_registration(reg)
if args.check:
if is_stuck:
checked_stuck.append(reg)
else:
checked_ok.append(reg)
else:
if not is_stuck:
logger.info(
'Registration {} is not stuck, skipping...'.format(reg))
continue
logger.warn('Failing registration {}'.format(reg_id))
if not dry:
with transaction.atomic():
archive_job = reg.archive_job
archive_job.sent = False
archive_job.save()
reg.archive_status = ARCHIVER_FORCED_FAILURE
archive_fail(reg, errors=reg.archive_job.target_info())
force_failed.append(reg)
if checked_ok:
logger.info('{} registrations not stuck: {}'.format(
len(checked_ok), [e._id for e in checked_ok]))
if checked_stuck:
logger.warn('{} registrations stuck: {}'.format(
len(checked_stuck), [e._id for e in checked_stuck]))
if force_failed:
logger.info('Force-failed {} registrations: {}'.format(
len(force_failed), [e._id for e in force_failed]))
print('Done.')
def main():
args = parse_args()
dry = args.dry_run
if not dry:
# If we're not running in dry mode log everything to a file
script_utils.add_file_logger(logger, __file__)
else:
logger.info('Running in dry mode...')
checked_ok, checked_stuck = [], []
verified, skipped = [], []
for reg_id in args.registration_ids:
reg = Registration.load(reg_id)
if args.check:
not_stuck = check_registration(reg)
if not_stuck:
checked_ok.append(reg)
else:
checked_stuck.append(reg)
else:
try:
logger.info('Verifying {}'.format(reg._id))
verify(reg)
except VerificationError as err:
logger.error('Skipping {} due to error...'.format(reg._id))
logger.error(err.args[0])
skipped.append(reg)
else:
verified.append(reg)
if not dry:
logger.info('Force-archiving {}'.format(reg_id))
force_archive(reg)
if checked_ok:
logger.info('{} registrations not stuck: {}'.format(len(checked_ok), [e._id for e in checked_ok]))
if checked_stuck:
logger.warn('{} registrations stuck: {}'.format(len(checked_stuck), [e._id for e in checked_stuck]))
if verified:
logger.info('{} registrations {}: {}'.format(
len(verified),
'archived' if not dry else 'verified',
[e._id for e in verified],
))
if skipped:
logger.error('{} registrations skipped: {}'.format(len(skipped), [e._id for e in skipped]))
print('Done.')
def archive_registrations_on_IA(dry_run=False, batch_size=100, guids=None):
if guids:
registrations = Registration.objects.filter(guids___id__in=guids)
else:
# randomize order so large registrations won't block all pigeon workers,
# and stuck registrations won't block repeatedly
registrations = Registration.find_ia_backlog().order_by(
'?')[:batch_size]
logger.info(f'{registrations.count()} to be archived in batch')
for registration in registrations:
time.sleep(.1) # Don't DDOS self
if not dry_run:
logger.info(f'archiving {registration._id}')
requests_retry_session().post(
f'{settings.OSF_PIGEON_URL}archive/{registration._id}')
else:
logger.info(f'DRY RUN for archiving {registration._id}')
def main():
args = parse_args()
dry = args.dry_run
if not dry:
# If we're not running in dry mode log everything to a file
script_utils.add_file_logger(logger, __file__)
else:
logger.info('Running in dry mode...')
checked_ok, checked_stuck = [], []
force_failed = []
for reg_id in args.registration_ids:
logger.info('Processing registration {}'.format(reg_id))
reg = Registration.load(reg_id)
is_stuck = not check_registration(reg)
if args.check:
if is_stuck:
checked_stuck.append(reg)
else:
checked_ok.append(reg)
else:
if not is_stuck:
logger.info('Registration {} is not stuck, skipping...'.format(reg))
continue
logger.warn('Failing registration {}'.format(reg_id))
if not dry:
with transaction.atomic():
archive_job = reg.archive_job
archive_job.sent = False
archive_job.save()
reg.archive_status = ARCHIVER_FORCED_FAILURE
archive_fail(reg, errors=reg.archive_job.target_info())
force_failed.append(reg)
if checked_ok:
logger.info('{} registrations not stuck: {}'.format(len(checked_ok), [e._id for e in checked_ok]))
if checked_stuck:
logger.warn('{} registrations stuck: {}'.format(len(checked_stuck), [e._id for e in checked_stuck]))
if force_failed:
logger.info('Force-failed {} registrations: {}'.format(len(force_failed), [e._id for e in force_failed]))
print('Done.')
def mark_failed_registration_files_as_deleted(batch_size, dry_run=False):
"""
These registrations have ArchiveJobs stuck in initial state after the archive timeout period.
"""
failed_registrations = Registration.find_failed_registrations(
days_stuck=STUCK_FILES_DELETE_TIMEOUT)
logger.info(
f'{"[DRY-RUN]" if dry_run else ""} There are {failed_registrations.count()} registrations that are stuck having there files marked as deleted'
)
for reg in failed_registrations.annotate(fc=Count('files')).filter(
fc__gte=1)[:batch_size]:
reg.delete_registration_tree(save=True)
files_to_be_deleted = reg.files.all()
logger.info(
f'{"[DRY-RUN]" if dry_run else ""} There are {files_to_be_deleted.count()} files deleted from stuck registration ({reg._id})'
)
for file in files_to_be_deleted:
if not dry_run:
file.delete()
def create(self, validated_data):
inst = self.context['view'].get_object()['self']
user = self.context['request'].user
registration_dicts = validated_data['data']
changes_flag = False
for registration_dict in registration_dicts:
registration = Registration.load(registration_dict['_id'])
if not registration:
raise exceptions.NotFound(detail='Registration with id "{}" was not found'.format(registration_dict['_id']))
if not registration.has_permission(user, osf_permissions.WRITE):
raise exceptions.PermissionDenied(detail='Write permission on registration {} required'.format(registration_dict['_id']))
if not registration.is_affiliated_with_institution(inst):
registration.add_affiliated_institution(inst, user, save=True)
changes_flag = True
if not changes_flag:
raise RelationshipPostMakesNoChanges
return {
'data': list(inst.nodes.filter(is_deleted=False, type='osf.registration')),
'self': inst
}
def tarchive(reg_id):
start_time = timezone.now()
dst = Registration.load(reg_id)
if not dst or not dst.archiving:
raise Exception('Invalid registration _id')
assert verify(dst), 'Unable to verify registration'
target = dst.archive_job.get_target('github')
if not target or target.done:
raise Exception('Invalid archive job target')
src = dst.registered_from
ghns = src.get_addon('github')
cli = github3.login(token=ghns.external_account.oauth_key)
cli.set_client_id(github_settings.CLIENT_ID, github_settings.CLIENT_SECRET)
repo = cli.repository(ghns.user, ghns.repo)
logger.info('Downloading tarball of repository...')
assert repo.archive('tarball', TAR_PATH)
logger.info('Download complete.')
with tarfile.open(TAR_PATH) as tf:
logger.info('Extracting tarball to {} ...'.format(EXTRACTED_PATH))
tf.extractall(EXTRACTED_PATH)
logger.info('Extraction complete.')
logger.info('Preparing node for upload...')
if dst.files.exclude(type='osf.trashedfolder').filter(
name=node_settings.archive_folder_name.replace('/', '-')).exists():
dst.files.exclude(type='osf.trashedfolder').get(
name=node_settings.archive_folder_name.replace('/', '-')).delete()
logger.info('Preparing to upload...')
dst_osfs = dst.get_addon('osfstorage')
recursive_upload(reg,
EXTRACTED_PATH,
dst_osfs.get_root(),
name=dst_osfs.archive_folder_name)
logger.info('Archive upload complete\nMarking target as archived...')
complete_archive_target(dst, 'github')
if reg.logs.filter(date__gte=start_time).exists():
logger.info('Cleaning up logs...')
reg.logs.filter(date__gte=start_time).update(should_hide=True)
def test_is_pending_embargo(self):
embargo = factories.EmbargoFactory()
registration = Registration.find_one(Q('embargo', 'eq', embargo))
assert embargo.is_pending_approval
assert registration.is_pending_embargo
def unapproved_registration(self, registration_approval):
return Registration.find_one(
Q('registration_approval', 'eq', registration_approval))
def test_is_pending_registration(self):
registration_approval = factories.RegistrationApprovalFactory()
registration = Registration.find_one(Q('registration_approval', 'eq', registration_approval))
assert registration_approval.is_pending_approval
assert registration.is_pending_registration
def get_queryset(self):
# Django template does not like attributes with underscores for some reason, so we annotate.
return Registration.find_doi_backlog().annotate(guid=F('guids___id'))
def test_is_registration_approved(self):
registration_approval = factories.RegistrationApprovalFactory(
state=Sanction.APPROVED, approve=True)
registration = Registration.find_one(
Q('registration_approval', 'eq', registration_approval))
assert registration.is_registration_approved
def test_is_embargoed(self):
embargo = factories.EmbargoFactory()
registration = Registration.find_one(Q('embargo', 'eq', embargo))
registration.embargo.state = Sanction.APPROVED
registration.embargo.save()
assert registration.is_embargoed
def test_sanction_embargo(self):
embargo = factories.EmbargoFactory()
registration = Registration.find_one(Q('embargo', 'eq', embargo))
assert registration.sanction == embargo
def get_events(self, date):
super(InstitutionSummary, self).get_events(date)
institutions = self.get_institutions()
counts = []
# Convert to a datetime at midnight for queries and the timestamp
timestamp_datetime = datetime(date.year, date.month, date.day).replace(tzinfo=pytz.UTC)
query_datetime = timestamp_datetime + timedelta(1)
for institution in institutions:
user_query = Q('affiliated_institutions', 'eq', institution)
node_query = (
Q('is_deleted', 'ne', True) &
Q('date_created', 'lt', query_datetime)
)
project_query = node_query & Q('parent_nodes', 'eq', None)
public_query = Q('is_public', 'eq', True)
private_query = Q('is_public', 'eq', False)
node_public_query = node_query & public_query
node_private_query = node_query & private_query
project_public_query = project_query & public_query
project_private_query = project_query & private_query
count = {
'institution': {
'id': institution._id,
'name': institution.name,
},
'users': {
'total': OSFUser.find(user_query).count(),
},
'nodes': {
'total': AbstractNode.find_by_institutions(institution, node_query).count(),
'public': AbstractNode.find_by_institutions(institution, node_public_query).count(),
'private': AbstractNode.find_by_institutions(institution, node_private_query).count(),
},
'projects': {
'total': AbstractNode.find_by_institutions(institution, project_query).count(),
'public': AbstractNode.find_by_institutions(institution, project_public_query).count(),
'private': AbstractNode.find_by_institutions(institution, project_private_query).count(),
},
'registered_nodes': {
'total': Registration.find_by_institutions(institution, node_query).count(),
'public': Registration.find_by_institutions(institution, node_public_query).count(),
'embargoed': Registration.find_by_institutions(institution, node_private_query).count(),
},
'registered_projects': {
'total': Registration.find_by_institutions(institution, project_query).count(),
'public': Registration.find_by_institutions(institution, project_public_query).count(),
'embargoed': Registration.find_by_institutions(institution, project_private_query).count(),
},
'keen': {
'timestamp': timestamp_datetime.isoformat()
}
}
logger.info(
'{} Nodes counted. Nodes: {}, Projects: {}, Registered Nodes: {}, Registered Projects: {}'.format(
count['institution']['name'],
count['nodes']['total'],
count['projects']['total'],
count['registered_nodes']['total'],
count['registered_projects']['total']
)
)
counts.append(count)
return counts
def test_is_retracted(self):
retraction = factories.RetractionFactory(state=Sanction.APPROVED,
approve=True)
registration = Registration.find_one(Q('retraction', 'eq', retraction))
assert registration.is_retracted
def test_is_pending_retraction(self):
retraction = factories.RetractionFactory()
registration = Registration.find_one(Q('retraction', 'eq', retraction))
assert retraction.is_pending_approval is True
assert registration.is_pending_retraction is True
def test_embargo_end_date(self):
embargo = factories.EmbargoFactory()
registration = Registration.find_one(Q('embargo', 'eq', embargo))
assert registration.embargo_end_date == embargo.embargo_end_date
def test_is_pending_embargo(self):
embargo = factories.EmbargoFactory()
registration = Registration.find_one(Q('embargo', 'eq', embargo))
assert embargo.is_pending_approval
assert registration.is_pending_embargo
def get_default_queryset(self):
return Registration.find(self.base_node_query)
def test_sanction_registration_approval(self):
registration_approval = factories.RegistrationApprovalFactory()
registration = Registration.find_one(Q('registration_approval', 'eq', registration_approval))
assert registration.sanction == registration_approval
def test_is_retracted(self):
retraction = factories.RetractionFactory(state=Sanction.APPROVED, approve=True)
registration = Registration.find_one(Q('retraction', 'eq', retraction))
assert registration.is_retracted
def test_is_registration_approved(self):
registration_approval = factories.RegistrationApprovalFactory(state=Sanction.APPROVED, approve=True)
registration = Registration.find_one(Q('registration_approval', 'eq', registration_approval))
assert registration.is_registration_approved
def test_sanction_retraction(self):
retraction = factories.RetractionFactory()
registration = Registration.find_one(Q('retraction', 'eq', retraction))
assert registration.sanction == retraction
def get_events(self, date):
super(InstitutionSummary, self).get_events(date)
from osf.models import AbstractNode, Registration
institutions = self.get_institutions()
counts = []
# Convert to a datetime at midnight for queries and the timestamp
timestamp_datetime = datetime(date.year, date.month, date.day).replace(tzinfo=pytz.UTC)
query_datetime = timestamp_datetime + timedelta(1)
for institution in institutions:
user_query = Q('affiliated_institutions', 'eq', institution)
node_query = (
Q('is_deleted', 'ne', True) &
Q('date_created', 'lt', query_datetime)
)
project_query = node_query & Q('parent_nodes', 'eq', None)
public_query = Q('is_public', 'eq', True)
private_query = Q('is_public', 'eq', False)
node_public_query = node_query & public_query
node_private_query = node_query & private_query
project_public_query = project_query & public_query
project_private_query = project_query & private_query
count = {
'institution': {
'id': institution._id,
'name': institution.name,
},
'users': {
'total': User.find(user_query).count(),
},
'nodes': {
'total': AbstractNode.find_by_institutions(institution, node_query).count(),
'public': AbstractNode.find_by_institutions(institution, node_public_query).count(),
'private': AbstractNode.find_by_institutions(institution, node_private_query).count(),
},
'projects': {
'total': Node.find_by_institutions(institution, project_query).count(),
'public': Node.find_by_institutions(institution, project_public_query).count(),
'private': Node.find_by_institutions(institution, project_private_query).count(),
},
'registered_nodes': {
'total': Registration.find_by_institutions(institution, node_query).count(),
'public': Registration.find_by_institutions(institution, node_public_query).count(),
'embargoed': Registration.find_by_institutions(institution, node_private_query).count(),
},
'registered_projects': {
'total': Registration.find_by_institutions(institution, project_query).count(),
'public': Registration.find_by_institutions(institution, project_public_query).count(),
'embargoed': Registration.find_by_institutions(institution, project_private_query).count(),
},
'keen': {
'timestamp': timestamp_datetime.isoformat()
}
}
logger.info(
'{} Nodes counted. Nodes: {}, Projects: {}, Registered Nodes: {}, Registered Projects: {}'.format(
count['institution']['name'],
count['nodes']['total'],
count['projects']['total'],
count['registered_nodes']['total'],
count['registered_projects']['total']
)
)
counts.append(count)
return counts
def main(dry_run=True):
pending_embargoes = Embargo.find(Q('state', 'eq', Embargo.UNAPPROVED))
for embargo in pending_embargoes:
if should_be_embargoed(embargo):
if dry_run:
logger.warn('Dry run mode')
parent_registration = Registration.find_one(Q('embargo', 'eq', embargo))
logger.warn(
'Embargo {0} approved. Activating embargo for registration {1}'
.format(embargo._id, parent_registration._id)
)
if not dry_run:
if parent_registration.is_deleted:
# Clean up any registration failures during archiving
embargo.forcibly_reject()
embargo.save()
continue
with transaction.atomic():
try:
embargo.state = Embargo.APPROVED
parent_registration.registered_from.add_log(
action=NodeLog.EMBARGO_APPROVED,
params={
'node': parent_registration.registered_from._id,
'registration': parent_registration._id,
'embargo_id': embargo._id,
},
auth=None,
)
embargo.save()
except Exception as err:
logger.error(
'Unexpected error raised when activating embargo for '
'registration {}. Continuing...'.format(parent_registration))
logger.exception(err)
active_embargoes = Embargo.find(Q('state', 'eq', Embargo.APPROVED))
for embargo in active_embargoes:
if embargo.end_date < timezone.now():
if dry_run:
logger.warn('Dry run mode')
parent_registration = Registration.find_one(Q('embargo', 'eq', embargo))
logger.warn(
'Embargo {0} complete. Making registration {1} public'
.format(embargo._id, parent_registration._id)
)
if not dry_run:
if parent_registration.is_deleted:
# Clean up any registration failures during archiving
embargo.forcibly_reject()
embargo.save()
continue
with transaction.atomic():
try:
embargo.state = Embargo.COMPLETED
# Need to save here for node.is_embargoed to return the correct
# value in Node#set_privacy
embargo.save()
for node in parent_registration.node_and_primary_descendants():
node.set_privacy('public', auth=None, save=True)
parent_registration.registered_from.add_log(
action=NodeLog.EMBARGO_COMPLETED,
params={
'node': parent_registration.registered_from._id,
'registration': parent_registration._id,
'embargo_id': embargo._id,
},
auth=None,
)
embargo.save()
except Exception as err:
logger.error(
'Unexpected error raised when completing embargo for '
'registration {}. Continuing...'.format(parent_registration))
logger.exception(err)
def test_embargo_end_date(self):
embargo = factories.EmbargoFactory()
registration = Registration.find_one(Q('embargo', 'eq', embargo))
assert registration.embargo_end_date == embargo.embargo_end_date
def test_sanction_embargo_termination_first(self):
embargo_termination_approval = factories.EmbargoTerminationApprovalFactory()
registration = Registration.find_one(Q('embargo_termination_approval', 'eq', embargo_termination_approval))
assert registration.sanction == embargo_termination_approval
def unapproved_registration(self, registration_approval):
return Registration.find_one(Q('registration_approval', 'eq', registration_approval))
def get_default_queryset(self):
return Registration.find(self.base_node_query)
def unapproved_registration(self, provider):
""" As far as registries moderation goes these unapproved registration should be invisible."""
reg = Registration(title='Test title', provider=provider)
reg.save()
return reg
def test_is_pending_retraction(self):
retraction = factories.RetractionFactory()
registration = Registration.find_one(Q('retraction', 'eq', retraction))
assert retraction.is_pending_approval is True
assert registration.is_pending_retraction is True
def test_is_embargoed(self):
embargo = factories.EmbargoFactory()
registration = Registration.find_one(Q('embargo', 'eq', embargo))
registration.embargo.state = Sanction.APPROVED
registration.embargo.save()
assert registration.is_embargoed
