def save_bare_system_tags(page_size=10000):
print('Starting save_bare_system_tags...')
start = timezone.now()
things = list(MODMNode.find(MQ(
'system_tags', 'ne', [])).sort('-_id')) + list(
MODMUser.find(MQ('system_tags', 'ne', [])).sort('-_id'))
system_tag_ids = []
for thing in things:
for system_tag in thing.system_tags:
system_tag_ids.append(system_tag)
unique_system_tag_ids = set(system_tag_ids)
total = len(unique_system_tag_ids)
system_tags = []
for system_tag_id in unique_system_tag_ids:
system_tags.append(Tag(name=system_tag_id, system=True))
created_system_tags = Tag.objects.bulk_create(system_tags)
print('MODM System Tags: {}'.format(total))
print('django system tags: {}'.format(
Tag.objects.filter(system=True).count()))
print('Done with {} in {} seconds...'.format(
sys._getframe().f_code.co_name,
(timezone.now() - start).total_seconds()))
def merge_duplicate_users():
print('Starting {}...'.format(sys._getframe().f_code.co_name))
start = timezone.now()
from framework.mongo.handlers import database
duplicates = database.user.aggregate([{
"$group": {
"_id": "$username",
"ids": {
"$addToSet": "$_id"
},
"count": {
"$sum": 1
}
}
}, {
"$match": {
"count": {
"$gt": 1
}
}
}, {
"$sort": {
"count": -1
}
}]).get('result')
# [
# {
# 'count': 5,
# '_id': '*****@*****.**',
# 'ids': [
# 'listo','fidst','hatma','tchth','euser','name!'
# ]
# }
# ]
print('Found {} duplicate usernames.'.format(len(duplicates)))
for duplicate in duplicates:
print('Found {} copies of {}'.format(len(duplicate.get('ids')),
duplicate.get('_id')))
if duplicate.get('_id'):
# _id is an email address, merge users keeping the one that was logged into last
users = list(
MODMUser.find(MQ('_id', 'in',
duplicate.get('ids'))).sort('-last_login'))
best_match = users.pop()
for user in users:
print('Merging user {} into user {}'.format(
user._id, best_match._id))
best_match.merge_user(user)
else:
# _id is null, set all usernames to their guid
users = MODMUser.find(MQ('_id', 'in', duplicate.get('ids')))
for user in users:
print('Setting username for {}'.format(user._id))
user.username = user._id
user.save()
print('Done with {} in {} seconds...'.format(
sys._getframe().f_code.co_name,
(timezone.now() - start).total_seconds()))
def get_default_odm_query(self):
base_query = (
MQ('is_registered', 'eq', True) &
MQ('date_disabled', 'eq', None)
)
if self.request.version >= '2.3':
return base_query & MQ('merged_by', 'eq', None)
return base_query
def get_default_odm_query(self):
user = self.get_user()
current_user = self.request.user
query = (
MQ('is_deleted', 'ne', True) &
MQ('type', 'eq', 'osf.registration') &
MQ('contributors', 'eq', user)
)
permission_query = MQ('is_public', 'eq', True)
if not current_user.is_anonymous:
permission_query = (permission_query | MQ('contributors', 'eq', current_user))
query = query & permission_query
return query
def convert_special_params_to_odm_query(self, field_name, query_params, key, data):
if isinstance(data, list):
if utils.is_falsy(query_params[key]):
# Use `or` when looking for not-preprints, to include both no file and is_orphaned
sub_query = functools.reduce(operator.or_, [
MQ(item['source_field_name'], item['op'], item['value'])
for item in data
])
else:
sub_query = functools.reduce(operator.and_, [
MQ(item['source_field_name'], item['op'], item['value'])
for item in data
])
return sub_query
else:
raise InvalidFilterError('Expected type list for field {}, got {}'.format(field_name, type(data)))
def get_queryset(self):
parent = self.request.query_params.get(
'filter[parents]', None) or self.request.query_params.get(
'filter[parent]', None)
provider = get_object_or_error(PreprintProvider,
self.kwargs['provider_id'],
display_name='PreprintProvider')
if parent:
if parent == 'null':
return provider.top_level_subjects
if provider.subjects.exists():
return provider.subjects.filter(parent___id=parent)
else:
# TODO: Delet this when all PreprintProviders have a mapping
# Calculate this here to only have to do it once.
allowed_parents = [
id_ for sublist in provider.subjects_acceptable
for id_ in sublist[0]
]
allows_children = [
subs[0][-1] for subs in provider.subjects_acceptable
if subs[1]
]
return [
sub
for sub in Subject.find(MQ('parent___id', 'eq', parent))
if provider.subjects_acceptable == []
or self.is_valid_subject(allows_children=allows_children,
allowed_parents=allowed_parents,
sub=sub)
]
return provider.all_subjects
def get_queryset(self):
parent = self.request.query_params.get('filter[parents]', None)
provider = PreprintProvider.load(self.kwargs['provider_id'])
if parent:
if parent == 'null':
return provider.top_level_subjects
# Calculate this here to only have to do it once.
allowed_parents = [
id_ for sublist in provider.subjects_acceptable
for id_ in sublist[0]
]
allows_children = [
subs[0][-1] for subs in provider.subjects_acceptable if subs[1]
]
return [
sub for sub in Subject.find(MQ('parents___id', 'eq', parent))
if provider.subjects_acceptable == []
or self.is_valid_subject(allows_children=allows_children,
allowed_parents=allowed_parents,
sub=sub)
]
return provider.all_subjects
def _operation_to_query(self, operation):
return MQ(operation['source_field_name'], operation['op'],
operation['value'])
def build_query(fields, model):
queries = (MQ(field, 'ne', None)
for field in list(set(fields) & set(model._fields.keys())))
if queries == []:
return None
return functools.reduce(operator.and_, queries)
class Preprint(AbstractNode):
# TODO DELETE ME POST MIGRATION
modm_model_path = 'website.project.model.Node'
modm_query = MQ('preprint_file', 'ne', None) | MQ('_is_preprint_orphan', 'eq', True)
# /TODO DELETE ME POST MIGRATION
# TODO: Uncomment when StoredFileNode is implemented
# file = models.ForeignKey('StoredFileNode', on_delete=models.SET_NULL, null=True, blank=True)
preprint_created = models.DateTimeField(null=True, blank=True)
subjects = models.ManyToManyField(Subject, related_name='preprints')
providers = models.ManyToManyField(PreprintProvider, related_name='preprints')
doi = models.CharField(max_length=128, null=True, blank=True, validators=[validate_doi])
_is_orphan = models.NullBooleanField(default=False)
@classmethod
def migrate_from_modm(cls, modm_obj):
django_obj = super(Preprint, cls).migrate_from_modm(modm_obj)
django_obj.doi = modm_obj.preprint_doi
django_obj._is_orphan = modm_obj._is_preprint_orphan
return django_obj
@property
def is_preprint(self):
"""For v1 compat."""
return True
def add_preprint_provider(self, preprint_provider, user, save=False):
if not self.has_permission(user, ADMIN):
raise PermissionsError('Only admins can update a preprint provider.')
if not preprint_provider:
raise ValueError('Must specify a provider to set as the preprint_provider')
self.providers.add(preprint_provider)
if save:
self.save()
def remove_preprint_provider(self, preprint_provider, user, save=False):
if not self.has_permission(user, ADMIN):
raise PermissionsError('Only admins can remove a preprint provider.')
if not preprint_provider:
raise ValueError('Must specify a provider to remove from this preprint.')
if self.providers.filter(id=preprint_provider.id).exists():
self.providers.remove(preprint_provider)
if save:
self.save()
return True
return False
def set_preprint_subjects(self, preprint_subjects, auth, save=False):
if not self.has_permission(auth.user, ADMIN):
raise PermissionsError('Only admins can change a preprint\'s subjects.')
self.subjects.clear()
self.subjects.add(
*Subject.objects.filter(guid__object_id__in=preprint_subjects).values_list('pk', flat=True)
)
if save:
self.save()
def set_preprint_file(self, preprint_file, auth, save=False):
if not self.has_permission(auth.user, ADMIN):
raise PermissionsError('Only admins can change a preprint\'s primary file.')
class Registration(AbstractNode):
# TODO DELETE ME POST MIGRATION
modm_model_path = 'website.project.model.Node'
modm_query = MQ('is_registration', 'eq', True)
# /TODO DELETE ME POST MIGRATION
objects = RegistrationManager()
registered_date = models.DateTimeField(db_index=True,
null=True,
blank=True)
registered_user = models.ForeignKey(OSFUser,
related_name='related_to',
on_delete=models.SET_NULL,
null=True,
blank=True)
registered_schema = models.ManyToManyField(MetaSchema)
registered_meta = DateTimeAwareJSONField(default=dict, blank=True)
# TODO Add back in once dependencies are resolved
registration_approval = models.ForeignKey(RegistrationApproval,
null=True,
blank=True)
retraction = models.ForeignKey(Retraction, null=True, blank=True)
embargo = models.ForeignKey(Embargo, null=True, blank=True)
registered_from = models.ForeignKey('self',
related_name='registrations',
on_delete=models.SET_NULL,
null=True,
blank=True)
# Sanctions
registration_approval = models.ForeignKey('RegistrationApproval',
related_name='registrations',
null=True,
blank=True,
on_delete=models.SET_NULL)
retraction = models.ForeignKey('Retraction',
related_name='registrations',
null=True,
blank=True,
on_delete=models.SET_NULL)
embargo = models.ForeignKey('Embargo',
related_name='registrations',
null=True,
blank=True,
on_delete=models.SET_NULL)
embargo_termination_approval = models.ForeignKey(
'EmbargoTerminationApproval',
related_name='registrations',
null=True,
blank=True,
on_delete=models.SET_NULL)
@property
def is_registration(self):
"""For v1 compat."""
return True
@property
def is_collection(self):
"""For v1 compat."""
return False
@property
def archive_job(self):
return self.archive_jobs.first() if self.archive_jobs.count() else None
@property
def sanction(self):
sanction = (self.embargo_termination_approval or self.retraction
or self.embargo or self.registration_approval)
if sanction:
return sanction
elif self.parent_node:
return self.parent_node.sanction
else:
return None
@property
def is_registration_approved(self):
if self.registration_approval is None:
if self.parent_node:
return self.parent_node.is_registration_approved
return False
return self.registration_approval.is_approved
@property
def is_pending_embargo(self):
if self.embargo is None:
if self.parent_node:
return self.parent_node.is_pending_embargo
return False
return self.embargo.is_pending_approval
@property
def is_pending_embargo_for_existing_registration(self):
""" Returns True if Node has an Embargo pending approval for an
existing registrations. This is used specifically to ensure
registrations pre-dating the Embargo feature do not get deleted if
their respective Embargo request is rejected.
"""
if self.embargo is None:
if self.parent_node:
return self.parent_node.is_pending_embargo_for_existing_registration
return False
return self.embargo.pending_registration
@property
def is_retracted(self):
if self.retraction is None:
if self.parent_node:
return self.parent_node.is_retracted
return False
return self.retraction.is_approved
@property
def is_pending_registration(self):
if not self.is_registration:
return False
if self.registration_approval is None:
if self.parent_node:
return self.parent_node.is_pending_registration
return False
return self.registration_approval.is_pending_approval
@property
def is_pending_retraction(self):
if self.retraction is None:
if self.parent_node:
return self.parent_node.is_pending_retraction
return False
return self.retraction.is_pending_approval
@property
def is_embargoed(self):
"""A Node is embargoed if:
- it has an associated Embargo record
- that record has been approved
- the node is not public (embargo not yet lifted)
"""
if self.embargo is None:
if self.parent_node:
return self.parent_node.is_embargoed
return self.embargo and self.embargo.is_approved and not self.is_public
@property
def embargo_end_date(self):
if self.embargo is None:
if self.parent_node:
return self.parent_node.embargo_end_date
return False
return self.embargo.end_date
@property
def archiving(self):
job = self.archive_job
return job and not job.done and not job.archive_tree_finished()
def _is_embargo_date_valid(self, end_date):
now = timezone.now()
if (end_date - now) >= settings.EMBARGO_END_DATE_MIN:
if (end_date - now) <= settings.EMBARGO_END_DATE_MAX:
return True
return False
def _initiate_embargo(self,
user,
end_date,
for_existing_registration=False,
notify_initiator_on_complete=False):
"""Initiates the retraction process for a registration
:param user: User who initiated the retraction
:param end_date: Date when the registration should be made public
"""
self.embargo = Embargo.objects.create(
initiated_by=user,
end_date=datetime.datetime.combine(end_date,
datetime.datetime.min.time()),
for_existing_registration=for_existing_registration,
notify_initiator_on_complete=notify_initiator_on_complete)
self.save() # Set foreign field reference Node.embargo
admins = self.get_admin_contributors_recursive(unique_users=True)
for (admin, node) in admins:
self.embargo.add_authorizer(admin, node)
self.embargo.save() # Save embargo's approval_state
return self.embargo
def embargo_registration(self,
user,
end_date,
for_existing_registration=False,
notify_initiator_on_complete=False):
"""Enter registration into an embargo period at end of which, it will
be made public
:param user: User initiating the embargo
:param end_date: Date when the registration should be made public
:raises: NodeStateError if Node is not a registration
:raises: PermissionsError if user is not an admin for the Node
:raises: ValidationValueError if end_date is not within time constraints
"""
if not self.has_permission(user, 'admin'):
raise PermissionsError('Only admins may embargo a registration')
if not self._is_embargo_date_valid(end_date):
if (end_date - timezone.now()) >= settings.EMBARGO_END_DATE_MIN:
raise ValidationValueError(
'Registrations can only be embargoed for up to four years.'
)
raise ValidationValueError(
'Embargo end date must be at least three days in the future.')
embargo = self._initiate_embargo(
user,
end_date,
for_existing_registration=for_existing_registration,
notify_initiator_on_complete=notify_initiator_on_complete)
self.registered_from.add_log(
action=NodeLog.EMBARGO_INITIATED,
params={
'node': self.registered_from._id,
'registration': self._id,
'embargo_id': embargo._id,
},
auth=Auth(user),
save=True,
)
if self.is_public:
self.set_privacy('private', Auth(user))
def request_embargo_termination(self, auth):
"""Initiates an EmbargoTerminationApproval to lift this Embargoed Registration's
embargo early."""
if not self.is_embargoed:
raise NodeStateError('This node is not under active embargo')
if not self.root == self:
raise NodeStateError(
'Only the root of an embargoed registration can request termination'
)
approval = EmbargoTerminationApproval(
initiated_by=auth.user,
embargoed_registration=self,
)
admins = [
admin for admin in self.root.get_admin_contributors_recursive(
unique_users=True)
]
for (admin, node) in admins:
approval.add_authorizer(admin, node=node)
approval.save()
approval.ask(admins)
self.embargo_termination_approval = approval
self.save()
return approval
def terminate_embargo(self, auth):
"""Handles the actual early termination of an Embargoed registration.
Adds a log to the registered_from Node.
"""
if not self.is_embargoed:
raise NodeStateError('This node is not under active embargo')
self.registered_from.add_log(action=NodeLog.EMBARGO_TERMINATED,
params={
'project': self._id,
'node': self.registered_from._id,
'registration': self._id,
},
auth=None,
save=True)
self.embargo.mark_as_completed()
for node in self.node_and_primary_descendants():
node.set_privacy(self.PUBLIC, auth=None, log=False, save=True)
return True
def _initiate_retraction(self, user, justification=None):
"""Initiates the retraction process for a registration
:param user: User who initiated the retraction
:param justification: Justification, if given, for retraction
"""
self.retraction = Retraction.objects.create(
initiated_by=user,
justification=justification or None, # make empty strings None
state=Retraction.UNAPPROVED)
self.save()
admins = self.get_admin_contributors_recursive(unique_users=True)
for (admin, node) in admins:
self.retraction.add_authorizer(admin, node)
self.retraction.save() # Save retraction approval state
return self.retraction
def retract_registration(self, user, justification=None, save=True):
"""Retract public registration. Instantiate new Retraction object
and associate it with the respective registration.
"""
if not self.is_public and not (self.embargo_end_date
or self.is_pending_embargo):
raise NodeStateError(
'Only public or embargoed registrations may be withdrawn.')
if self.root is not self:
raise NodeStateError(
'Withdrawal of non-parent registrations is not permitted.')
retraction = self._initiate_retraction(user, justification)
self.registered_from.add_log(
action=NodeLog.RETRACTION_INITIATED,
params={
'node': self.registered_from._id,
'registration': self._id,
'retraction_id': retraction._id,
},
auth=Auth(user),
)
self.retraction = retraction
if save:
self.save()
return retraction
def delete_registration_tree(self, save=False):
self.is_deleted = True
if not getattr(self.embargo, 'for_existing_registration', False):
self.registered_from = None
if save:
self.save()
self.update_search()
for child in self.nodes_primary:
child.delete_registration_tree(save=save)
def get_default_odm_query(self):
user = self.get_user()
query = MQ('contributors', 'eq', user) & default_node_list_query()
if user != self.request.user:
query &= default_node_permission_query(self.request.user)
return query
def get_default_queryset(self):
user = self.get_user()
query = MQ('contributors', 'eq', user) & default_node_list_query()
if user != self.request.user:
query &= default_node_permission_query(self.request.user)
return AbstractNode.find(query)
class Institution(Loggable, base.GuidMixin, base.BaseModel):
# TODO DELETE ME POST MIGRATION
modm_model_path = 'website.project.model.Node'
modm_query = MQ('is_institution', 'eq', True)
# /TODO DELETE ME POST MIGRATION
# TODO Remove null=True for things that shouldn't be nullable
auth_url = models.URLField(null=True)
banner_name = models.CharField(max_length=255, null=True)
contributors = models.ManyToManyField(settings.AUTH_USER_MODEL,
through=InstitutionalContributor,
related_name='institutions')
domains = fields.ArrayField(models.CharField(max_length=255),
db_index=True,
null=True)
email_domains = fields.ArrayField(models.CharField(max_length=255),
db_index=True,
null=True)
logo_name = models.CharField(
max_length=255, null=True) # TODO: Could this be a FilePathField?
logout_url = models.URLField(null=True)
name = models.CharField(max_length=255)
def __init__(self, *args, **kwargs):
kwargs.pop('node', None)
super(Institution, self).__init__(*args, **kwargs)
@classmethod
def migrate_from_modm(cls, modm_obj):
guid, created = Guid.objects.get_or_create(guid=modm_obj._id)
inst = Institution()
inst._guid = guid
inst.auth_url = modm_obj.institution_auth_url
inst.banner_name = modm_obj.institution_banner_name
inst.domains = modm_obj.institution_domains
inst.email_domains = modm_obj.institution_email_domains
inst.logo_name = modm_obj.institution_logo_name
inst.logout_url = modm_obj.institution_logout_url
inst.name = modm_obj.title
inst.description = modm_obj.description
inst.is_deleted = modm_obj.is_deleted
return inst
@property
def api_v2_url(self):
return reverse('institutions:institution-detail',
kwargs={'institution_id': self._id})
@property
def absolute_api_v2_url(self):
from api.base.utils import absolute_reverse
return absolute_reverse('institutions:institution-detail',
kwargs={'institution_id': self._id})
@property
def nodes_url(self):
return self.absolute_api_v2_url + 'nodes/'
@property
def nodes_relationship_url(self):
return self.absolute_api_v2_url + 'relationships/nodes/'
@property
def logo_path(self):
if self.logo_name:
return '/static/img/institutions/{}'.format(self.logo_name)
else:
return None
@property
def logo_path_rounded_corners(self):
logo_base = '/static/img/institutions/shields-rounded-corners/{}-rounded-corners.png'
if self.logo_name:
return logo_base.format(self.logo_name.replace('.png', ''))
else:
return None
@property
def banner_path(self):
if self.banner_name:
return '/static/img/institutions/{}'.format(self.banner_name)
else:
return None