def create_or_update_provider(self, provider_data):
provider = self.get_page_provider()
licenses = [
NodeLicense.objects.get(license_id=license_id)
for license_id in provider_data.pop('licenses_acceptable', [])
]
default_license = provider_data.pop('default_license', False)
subject_data = provider_data.pop('subjects', False)
if provider:
for key, val in provider_data.items():
setattr(provider, key, val)
else:
provider = PreprintProvider(**provider_data)
provider.save()
if licenses:
provider.licenses_acceptable.set(licenses)
if default_license:
provider.default_license = NodeLicense.objects.get(
license_id=default_license)
# Only adds the JSON taxonomy if there is no existing taxonomy data
if subject_data and not provider.subjects.count():
self.add_subjects(provider, subject_data)
return provider
def rebrand_provider(src_id, dst_id):
assert src_id != 'osf', 'Cannot rebrand OSF Preprints'
src_prov = PreprintProvider.load(src_id)
dst_prov = PreprintProvider.load(dst_id)
assert src_prov, 'Unable to find provider {}'.format(src_id)
assert dst_prov, 'Unable to find provider {}'.format(dst_id)
assert set(src_prov.subjects.values_list('text', flat=True)) == set(
dst_prov.subjects.values_list(
'text', flat=True)), 'Provider subjects do not match'
assert set(src_prov.licenses_acceptable.values_list(
'id', flat=True)) == set(
dst_prov.licenses_acceptable.values_list(
'id', flat=True)), 'Provider licenses do not match'
assert set(
src_prov.notification_subscriptions.values_list(
'event_name', flat=True)) == set(
dst_prov.notification_subscriptions.values_list(
'event_name',
flat=True)), 'Provider subscription events do not match'
assert dst_prov.asset_files.filter(
name='square_color_no_transparent').exists(
), 'Invalid assets on {}'.format(dst_id)
assert dst_prov.access_token, 'Destination Provider must have a SHARE access token'
logger.info('Updating {}\'s preprints to provider {}'.format(
src_id, dst_id))
src_prov.preprints.update(provider_id=dst_prov.id)
logger.info(
'Updating preprint subjects with {} equivalent subjects'.format(
dst_id))
target_preprints = dst_prov.preprints.all()
pbar = progressbar.ProgressBar(
maxval=target_preprints.count() or 1).start()
for i, pp in enumerate(target_preprints, 1):
pbar.update(i)
# M2M .set does not require .save
pp.subjects.set(
dst_prov.subjects.filter(
text__in=list(pp.subjects.values_list('text', flat=True))))
pbar.finish()
logger.info('Updating {} moderators'.format(dst_id))
dst_prov.get_group('admin').user_set.set(
src_prov.get_group('admin').user_set.all())
dst_prov.get_group('moderator').user_set.set(
src_prov.get_group('moderator').user_set.all())
logger.info('Updating {} notification subscriptions'.format(dst_id))
for src_sub in src_prov.notification_subscriptions.all():
dst_sub = dst_prov.notification_subscriptions.get(
event_name=src_sub.event_name)
dst_sub.email_transactional.set(src_sub.email_transactional.all())
dst_sub.email_digest.set(src_sub.email_digest.all())
dst_sub.none.set(src_sub.none.all())
logger.info('Updating {} notification digests'.format(dst_id))
src_prov.notificationdigest_set.update(provider_id=dst_prov.id)
def withdraw_all_preprints(provider_id, page_size, user_guid, comment=None):
""" A management command to withdraw all preprints from a specified provider
Created to withdraw all EarthRxiv preprints, but can be used for any preprint
provider
"""
provider = PreprintProvider.load(provider_id)
if not provider:
raise RuntimeError('Could not find provider. Check ID and try again')
user = OSFUser.load(user_guid)
if not user:
raise RuntimeError('Could not find user. Check GUID and try again')
saved_comment = PROVIDER_WITHDRAWAL_COMMENT.get(provider_id, None)
if not comment and saved_comment:
comment = saved_comment
if not comment and not saved_comment:
raise RuntimeError('Comment not provided!')
preprints = provider.preprints.filter(date_withdrawn=None,
is_published=True)[:page_size]
preprints_withdrawn = 0
for preprint in preprints:
preprint.run_withdraw(user, comment)
preprint.reload()
assert preprint.is_retracted
preprints_withdrawn += 1
logger.info(
f'{preprints_withdrawn} have been withdrawn from provider {provider.name}'
)
def reindex_share(dst_id, dry_run):
dst_prov = PreprintProvider.load(dst_id)
if dry_run:
logger.info('Would send {} preprints to SHARE...'.format(
dst_prov.preprints.count()))
else:
reindex_provider(dst_prov)
def form_valid(self, form):
osf_id = form.cleaned_data.get('osf_id')
osf_user = OSFUser.load(osf_id)
if not osf_user:
raise Http404('OSF user with id "{}" not found. Please double check.'.format(osf_id))
osf_user.is_staff = True
osf_user.save()
# create AdminProfile for this new user
profile, created = AdminProfile.objects.get_or_create(user=osf_user)
osf_user.groups.clear()
prereg_admin_group = Group.objects.get(name='prereg_admin')
for group in form.cleaned_data.get('group_perms'):
osf_user.groups.add(group)
split = group.name.split('_')
group_type = split[0]
if group_type == 'reviews':
provider_id = split[1]
provider = PreprintProvider.load(provider_id)
provider.notification_subscriptions.get(event_name='new_pending_submissions').add_user_to_subscription(osf_user, 'email_transactional')
if group == prereg_admin_group:
administer_permission = Permission.objects.get(codename='administer_prereg')
osf_user.user_permissions.add(administer_permission)
osf_user.save()
if created:
messages.success(self.request, 'Registration successful for OSF User {}!'.format(osf_user.username))
else:
messages.success(self.request, 'Permissions update successful for OSF User {}!'.format(osf_user.username))
return super(RegisterUser, self).form_valid(form)
def populate_collection_providers(add_data):
for data in COLLECTION_PROVIDERS:
_id = data.pop('_id')
default_license = data.pop('default_license', False)
licenses = [
get_license(name) for name in data.pop('licenses_acceptable', [])
]
custom_taxonomy = data.pop('custom_taxonomy', False)
primary_collection = data.pop('primary_collection', False)
provider, created = CollectionProvider.objects.update_or_create(
_id=_id, defaults=data)
if licenses:
provider.licenses_acceptable.set(licenses)
if default_license:
provider.default_license = get_license(default_license)
if custom_taxonomy and not provider.subjects.exists():
logger.info('Adding custom taxonomy for: {}'.format(_id))
call_command('populate_custom_taxonomies', '--provider', _id,
'--type', 'osf.collectionprovider', '--data',
json.dumps(custom_taxonomy))
provider_subjects = provider.subjects.all()
subjects = provider_subjects if len(
provider_subjects) else PreprintProvider.load(
'osf').subjects.all()
if primary_collection and not provider.primary_collection:
primary_collection['provider'] = provider
provider.primary_collection = factories.CollectionFactory(
**primary_collection)
provider.primary_collection.save()
provider.save()
if add_data and provider.primary_collection:
user = factories.AuthUserFactory()
user.save()
for _ in range(5):
node = factories.NodeFactory()
node.is_public = True
node.save()
status = random.choice(
provider.primary_collection.status_choices)
collected_type = random.choice(
provider.primary_collection.collected_type_choices)
cgm = provider.primary_collection.collect_object(
node, user, collected_type=collected_type, status=status)
rando_subjects = random.sample(subjects, min(len(subjects), 5))
cgm.subjects.add(*rando_subjects)
cgm.save()
logger.info('{} collection provider: {}'.format(
'Added' if created else 'Updated', _id))
def create_or_update_provider(self, provider_data):
provider = self.get_page_provider()
licenses = [NodeLicense.objects.get(license_id=license_id) for license_id in provider_data.pop('licenses_acceptable', [])]
default_license = provider_data.pop('default_license', False)
subject_data = provider_data.pop('subjects', False)
if provider:
for key, val in provider_data.items():
setattr(provider, key, val)
else:
provider = PreprintProvider(**provider_data)
provider.save()
if licenses:
provider.licenses_acceptable = licenses
if default_license:
provider.default_license = NodeLicense.objects.get(license_id=default_license)
# Only adds the JSON taxonomy if there is no existing taxonomy data
if subject_data and not provider.subjects.count():
self.add_subjects(provider, subject_data)
return provider
def populate_collection_providers(add_data):
for data in COLLECTION_PROVIDERS:
_id = data.pop('_id')
default_license = data.pop('default_license', False)
licenses = [get_license(name) for name in data.pop('licenses_acceptable', [])]
custom_taxonomy = data.pop('custom_taxonomy', False)
primary_collection = data.pop('primary_collection', False)
provider, created = CollectionProvider.objects.update_or_create(_id=_id, defaults=data)
if licenses:
provider.licenses_acceptable.add(*licenses)
if default_license:
provider.default_license = get_license(default_license)
if custom_taxonomy and not provider.subjects.exists():
logger.info('Adding custom taxonomy for: {}'.format(_id))
call_command('populate_custom_taxonomies', '--provider', _id, '--type', 'osf.collectionprovider', '--data', json.dumps(custom_taxonomy))
provider_subjects = provider.subjects.all()
subjects = provider_subjects if len(provider_subjects) else PreprintProvider.load('osf').subjects.all()
if primary_collection and not provider.primary_collection:
primary_collection['provider'] = provider
provider.primary_collection = factories.CollectionFactory(**primary_collection)
provider.primary_collection.save()
provider.save()
if add_data and provider.primary_collection:
user = factories.AuthUserFactory()
user.save()
for _ in range(5):
node = factories.NodeFactory()
node.is_public = True
node.save()
status = random.choice(provider.primary_collection.status_choices)
collected_type = random.choice(provider.primary_collection.collected_type_choices)
cgm = provider.primary_collection.collect_object(node, user, collected_type=collected_type, status=status)
rando_subjects = random.sample(subjects, min(len(subjects), 5))
cgm.subjects.add(*rando_subjects)
cgm.save()
logger.info('{} collection provider: {}'.format('Added' if created else 'Updated', _id))
def form_valid(self, form):
osf_id = form.cleaned_data.get('osf_id')
osf_user = OSFUser.load(osf_id)
if not osf_user:
raise Http404(
'OSF user with id "{}" not found. Please double check.'.format(
osf_id))
osf_user.is_staff = True
osf_user.save()
# create AdminProfile for this new user
profile, created = AdminProfile.objects.get_or_create(user=osf_user)
osf_user.groups.clear()
prereg_admin_group = Group.objects.get(name='prereg_admin')
for group in form.cleaned_data.get('group_perms'):
osf_user.groups.add(group)
split = group.name.split('_')
group_type = split[0]
if group_type == 'reviews':
provider_id = split[1]
provider = PreprintProvider.load(provider_id)
provider.notification_subscriptions.get(
event_name='new_pending_submissions'
).add_user_to_subscription(osf_user, 'email_transactional')
if group == prereg_admin_group:
administer_permission = Permission.objects.get(
codename='administer_prereg')
osf_user.user_permissions.add(administer_permission)
osf_user.save()
if created:
messages.success(
self.request,
'Registration successful for OSF User {}!'.format(
osf_user.username))
else:
messages.success(
self.request,
'Permissions update successful for OSF User {}!'.format(
osf_user.username))
return super(RegisterUser, self).form_valid(form)
def handle(self, *args, **options):
reverse = options.get('reverse', False)
dry_run = options.get('dry_run', False)
add_data = options.get('add_data', False)
if not dry_run:
script_utils.add_file_logger(logger, __file__)
with transaction.atomic():
if reverse:
remove_preprint_providers()
remove_registration_providers()
remove_collection_providers()
else:
if not PreprintProvider.load('osf'):
update_taxonomies('bepress_taxonomy.json')
populate_preprint_providers()
populate_registration_providers()
populate_collection_providers(add_data)
if dry_run:
raise RuntimeError('Dry run, transaction rolled back.')
def handle(self, *args, **options):
reverse = options.get('reverse', False)
dry_run = options.get('dry_run', False)
add_data = options.get('add_data', False)
if not dry_run:
script_utils.add_file_logger(logger, __file__)
with transaction.atomic():
if reverse:
remove_preprint_providers()
remove_registration_providers()
remove_collection_providers()
else:
if not PreprintProvider.load('osf'):
update_taxonomies('bepress_taxonomy.json')
populate_preprint_providers()
populate_registration_providers()
populate_collection_providers(add_data)
if dry_run:
raise RuntimeError('Dry run, transaction rolled back.')
def delete_old_provider(src_id):
src_prov = PreprintProvider.load(src_id)
assert src_prov.preprints.count(
) == 0, 'Provider {} still has preprints'.format(src_id)
assert src_prov.notificationdigest_set.count(
) == 0, 'Provider {} still has queued digest emails'
assert src_prov.subjects.annotate(pc=Count('preprints')).filter(
pc__gt=0).count() == 0, 'Provider {} still has used subjects'.format(
src_id)
# I don't trust CASCADE deletes to be set up correctly
logger.warn('Deleting Assets: {}'.format(
src_prov.asset_files.annotate(pc=Count('providers')).filter(
pc=1).delete()))
logger.warn('Deleting Groups: {}'.format(src_prov.group_objects.delete()))
logger.warn('Deleting Subjects: {}'.format(
src_prov.subjects.all().delete()))
logger.warn('Deleting Subscriptions: {}'.format(
src_prov.notification_subscriptions.all().delete()))
logger.warn('Deleting Provider: {}'.format(src_prov.delete()))
def main(dry_run=True):
psy = PreprintProvider.load('psyarxiv')
target_date = parse(CUSTOM_TAXONOMY_APPLIED_DATE)
target_preprints = psy.preprint_services.filter(created__lte=target_date)
i = 0
for pp in target_preprints:
i += 1
original_modified = pp.modified
logger.info('Preparing to migrate preprint {}'.format(pp._id))
bps = Subject.objects.filter(
id__in=pp.subjects.values_list('bepress_subject', flat=True))
correct_subjects = Subject.objects.filter(
provider=psy, text__in=[BP_TO_PSY_MAP[s.text] for s in bps])
logger.info('Replacing subjects {} with {} on {}'.format(
pp.subjects.values_list('text', flat=True),
correct_subjects.values_list('text', flat=True), pp._id))
if not dry_run:
pp.subjects = correct_subjects
logger.info('Successfully migrated {} preprints'.format(i))
def get_queryset(self):
return PreprintProvider.find(self.get_query_from_request())
def claim_user_form(auth, **kwargs):
"""
View for rendering the set password page for a claimed user.
Must have ``token`` as a querystring argument.
Renders the set password form, validates it, and sets the user's password.
HTTP Method: GET, POST
"""
uid, pid = kwargs['uid'], kwargs['pid']
token = request.form.get('token') or request.args.get('token')
user = OSFUser.load(uid)
# If unregistered user is not in database, or url bears an invalid token raise HTTP 400 error
if not user or not verify_claim_token(user, token, pid):
error_data = {
'message_short': 'Invalid url.',
'message_long': 'Claim user does not exists, the token in the URL is invalid or has expired.'
}
raise HTTPError(http.BAD_REQUEST, data=error_data)
# If user is logged in, redirect to 're-enter password' page
if auth.logged_in:
return redirect(web_url_for('claim_user_registered',
uid=uid, pid=pid, token=token))
unclaimed_record = user.unclaimed_records[pid]
user.fullname = unclaimed_record['name']
user.update_guessed_names()
# The email can be the original referrer email if no claimer email has been specified.
claimer_email = unclaimed_record.get('claimer_email') or unclaimed_record.get('email')
# If there is a registered user with this email, redirect to 're-enter password' page
try:
user_from_email = OSFUser.objects.get(emails__address=claimer_email.lower().strip()) if claimer_email else None
except OSFUser.DoesNotExist:
user_from_email = None
if user_from_email and user_from_email.is_registered:
return redirect(web_url_for('claim_user_registered', uid=uid, pid=pid, token=token))
form = SetEmailAndPasswordForm(request.form, token=token)
if request.method == 'POST':
if not form.validate():
forms.push_errors_to_status(form.errors)
elif settings.RECAPTCHA_SITE_KEY and not validate_recaptcha(request.form.get('g-recaptcha-response'), remote_ip=request.remote_addr):
status.push_status_message('Invalid captcha supplied.', kind='error')
else:
username, password = claimer_email, form.password.data
if not username:
raise HTTPError(http.BAD_REQUEST, data=dict(
message_long='No email associated with this account. Please claim this '
'account on the project to which you were invited.'
))
user.register(username=username, password=password, accepted_terms_of_service=form.accepted_terms_of_service.data)
# Clear unclaimed records
user.unclaimed_records = {}
user.verification_key = generate_verification_key()
user.save()
# Authenticate user and redirect to project page
status.push_status_message(language.CLAIMED_CONTRIBUTOR, kind='success', trust=True)
# Redirect to CAS and authenticate the user with a verification key.
provider = PreprintProvider.load(pid)
redirect_url = None
if provider:
redirect_url = web_url_for('auth_login', next=provider.landing_url, _absolute=True)
else:
redirect_url = web_url_for('resolve_guid', guid=pid, _absolute=True)
return redirect(cas.get_login_url(
redirect_url,
username=user.username,
verification_key=user.verification_key
))
return {
'firstname': user.given_name,
'email': claimer_email if claimer_email else '',
'fullname': user.fullname,
'form': forms.utils.jsonify(form) if is_json_request() else form,
'osf_contact_email': settings.OSF_CONTACT_EMAIL,
}
def get_object(self, node_id):
return PreprintProvider.load(node_id)
def get_queryset(self):
return PreprintProvider.find(self.get_query_from_request())
def claim_user_form(auth, **kwargs):
"""
View for rendering the set password page for a claimed user.
Must have ``token`` as a querystring argument.
Renders the set password form, validates it, and sets the user's password.
HTTP Method: GET, POST
"""
uid, pid = kwargs['uid'], kwargs['pid']
token = request.form.get('token') or request.args.get('token')
user = OSFUser.load(uid)
# If unregistered user is not in database, or url bears an invalid token raise HTTP 400 error
if not user or not verify_claim_token(user, token, pid):
error_data = {
'message_short':
'Invalid url.',
'message_long':
'Claim user does not exists, the token in the URL is invalid or has expired.'
}
raise HTTPError(http_status.HTTP_400_BAD_REQUEST, data=error_data)
# If user is logged in, redirect to 're-enter password' page
if auth.logged_in:
return redirect(
web_url_for('claim_user_registered', uid=uid, pid=pid,
token=token))
unclaimed_record = user.unclaimed_records[pid]
user.fullname = unclaimed_record['name']
user.update_guessed_names()
# The email can be the original referrer email if no claimer email has been specified.
claimer_email = unclaimed_record.get(
'claimer_email') or unclaimed_record.get('email')
# If there is a registered user with this email, redirect to 're-enter password' page
try:
user_from_email = OSFUser.objects.get(
emails__address=claimer_email.lower().strip(
)) if claimer_email else None
except OSFUser.DoesNotExist:
user_from_email = None
if user_from_email and user_from_email.is_registered:
return redirect(
web_url_for('claim_user_registered', uid=uid, pid=pid,
token=token))
form = SetEmailAndPasswordForm(request.form, token=token)
if request.method == 'POST':
if not form.validate():
forms.push_errors_to_status(form.errors)
elif settings.RECAPTCHA_SITE_KEY and not validate_recaptcha(
request.form.get('g-recaptcha-response'),
remote_ip=request.remote_addr):
status.push_status_message('Invalid captcha supplied.',
kind='error')
else:
username, password = claimer_email, form.password.data
if not username:
raise HTTPError(
http_status.HTTP_400_BAD_REQUEST,
data=dict(
message_long=
'No email associated with this account. Please claim this '
'account on the project to which you were invited.'))
user.register(
username=username,
password=password,
accepted_terms_of_service=form.accepted_terms_of_service.data)
# Clear unclaimed records
user.unclaimed_records = {}
user.verification_key = generate_verification_key()
user.save()
# Authenticate user and redirect to project page
status.push_status_message(language.CLAIMED_CONTRIBUTOR,
kind='success',
trust=True)
# Redirect to CAS and authenticate the user with a verification key.
provider = PreprintProvider.load(pid)
redirect_url = None
if provider:
redirect_url = web_url_for('auth_login',
next=provider.landing_url,
_absolute=True)
else:
redirect_url = web_url_for('resolve_guid',
guid=pid,
_absolute=True)
return redirect(
cas.get_login_url(redirect_url,
username=user.username,
verification_key=user.verification_key))
return {
'firstname': user.given_name,
'email': claimer_email if claimer_email else '',
'fullname': user.fullname,
'form': forms.utils.jsonify(form) if is_json_request() else form,
'osf_contact_email': settings.OSF_CONTACT_EMAIL,
}
def get_object(self, node_id):
return PreprintProvider.load(node_id)
