def conference_submissions(**kwargs):
"""Return data for all OSF4M submissions.
The total number of submissions for each meeting is calculated and cached
in the Conference.num_submissions field.
"""
conferences = Conference.find(Q('is_meeting', 'ne', False))
# TODO: Revisit this loop, there has to be a way to optimize it
for conf in conferences:
# For efficiency, we filter by tag first, then node
# instead of doing a single Node query
projects = set()
tags = Tag.find(Q('system', 'eq', False) & Q('name', 'iexact', conf.endpoint.lower())).values_list('pk', flat=True)
nodes = Node.find(
Q('tags', 'in', tags) &
Q('is_public', 'eq', True) &
Q('is_deleted', 'ne', True)
).include('guids')
projects.update(list(nodes))
num_submissions = len(projects)
# Cache the number of submissions
conf.num_submissions = num_submissions
bulk_update(conferences, update_fields=['num_submissions'])
return {'success': True}
def conference_submissions(**kwargs):
"""Return data for all OSF4M submissions.
The total number of submissions for each meeting is calculated and cached
in the Conference.num_submissions field.
"""
conferences = Conference.find(Q('is_meeting', 'ne', False))
# TODO: Revisit this loop, there has to be a way to optimize it
for conf in conferences:
# For efficiency, we filter by tag first, then node
# instead of doing a single Node query
projects = set()
tags = Tag.find(
Q('system', 'eq', False)
& Q('name', 'iexact', conf.endpoint.lower())).values_list(
'pk', flat=True)
nodes = AbstractNode.find(
Q('tags', 'in', tags) & Q('is_public', 'eq', True)
& Q('is_deleted', 'ne', True)).include('guids')
projects.update(list(nodes))
num_submissions = len(projects)
# Cache the number of submissions
conf.num_submissions = num_submissions
bulk_update(conferences, update_fields=['num_submissions'])
return {'success': True}
def test_integration(self, mock_upload, mock_send_mail):
fullname = 'John Deacon'
username = '******'
title = 'good songs'
conference = ConferenceFactory()
body = 'dragon on my back'
content = 'dragon attack'
recipient = '{0}{1}[email protected]'.format(
'test-' if settings.DEV_MODE else '',
conference.endpoint,
)
self.app.post(
api_url_for('meeting_hook'),
{
'X-Mailgun-Sscore':
0,
'timestamp':
'123',
'token':
'secret',
'signature':
hmac.new(
key=settings.MAILGUN_API_KEY,
msg='{}{}'.format('123', 'secret'),
digestmod=hashlib.sha256,
).hexdigest(),
'attachment-count':
'1',
'X-Mailgun-Sscore':
0,
'from':
'{0} <{1}>'.format(fullname, username),
'recipient':
recipient,
'subject':
title,
'stripped-text':
body,
},
upload_files=[
('attachment-1', 'attachment-1', content),
],
)
assert_true(mock_upload.called)
users = OSFUser.find(Q('username', 'eq', username))
assert_equal(users.count(), 1)
nodes = AbstractNode.find(Q('title', 'eq', title))
assert_equal(nodes.count(), 1)
node = nodes[0]
assert_equal(node.get_wiki_page('home').content, body)
assert_true(mock_send_mail.called)
call_args, call_kwargs = mock_send_mail.call_args
assert_absolute(call_kwargs['conf_view_url'])
assert_absolute(call_kwargs['set_password_url'])
assert_absolute(call_kwargs['profile_url'])
assert_absolute(call_kwargs['file_url'])
assert_absolute(call_kwargs['node_url'])
def get_queryset(self):
blacklisted = self.is_blacklisted()
nodes = self.get_queryset_from_request().distinct()
# If attempting to filter on a blacklisted field, exclude withdrawals.
if blacklisted:
non_withdrawn_list = [node._id for node in nodes if not node.is_retracted]
non_withdrawn_nodes = Node.find(Q('_id', 'in', non_withdrawn_list))
return non_withdrawn_nodes
return nodes
def get_default_queryset(self):
base_query = (Q('is_deleted', 'ne', True)
& Q('type', 'eq', 'osf.registration'))
user = self.request.user
permission_query = Q('is_public', 'eq', True)
if not user.is_anonymous:
permission_query = (permission_query
| Q('contributors', 'eq', user))
query = base_query & permission_query
return AbstractNode.find(query)
def get_queryset(self):
blacklisted = self.is_blacklisted()
nodes = self.get_queryset_from_request().distinct()
# If attempting to filter on a blacklisted field, exclude withdrawals.
if blacklisted:
non_withdrawn_list = [
node._id for node in nodes if not node.is_retracted
]
non_withdrawn_nodes = AbstractNode.find(
Q('_id', 'in', non_withdrawn_list))
return non_withdrawn_nodes
return nodes
def get_default_queryset(self):
base_query = (
Q('is_deleted', 'ne', True) &
Q('type', 'eq', 'osf.registration')
)
user = self.request.user
permission_query = Q('is_public', 'eq', True)
if not user.is_anonymous:
permission_query = (permission_query | Q('contributors', 'eq', user))
query = base_query & permission_query
return Node.find(query)
def get_queryset(self):
node = self.get_node()
req_query = self.get_query_from_request()
node_pks = node.node_relations.filter(
is_node_link=False).select_related('child').values_list(
'child__pk', flat=True)
query = (Q('pk', 'in', node_pks) & req_query)
nodes = AbstractNode.find(query).order_by('-date_modified')
auth = get_user_auth(self.request)
pks = [each.pk for each in nodes if each.can_view(auth)]
return AbstractNode.objects.filter(
pk__in=pks).order_by('-date_modified')
def get_default_queryset(self):
user = self.get_user()
current_user = self.request.user
query = (
MQ('is_deleted', 'ne', True) &
MQ('type', 'eq', 'osf.registration') &
MQ('contributors', 'eq', user)
)
permission_query = MQ('is_public', 'eq', True)
if not current_user.is_anonymous:
permission_query = (permission_query | MQ('contributors', 'eq', current_user))
query = query & permission_query
return AbstractNode.find(query)
def get_default_queryset(self):
user = self.get_user()
current_user = self.request.user
query = (
MQ('is_deleted', 'ne', True) &
MQ('type', 'eq', 'osf.registration') &
MQ('contributors', 'eq', user)
)
permission_query = MQ('is_public', 'eq', True)
if not current_user.is_anonymous:
permission_query = (permission_query | MQ('contributors', 'eq', current_user))
query = query & permission_query
return Node.find(query)
def get_queryset(self):
node = self.get_node()
req_query = self.get_query_from_request()
node_pks = node.node_relations.filter(is_node_link=False).select_related('child').values_list('child__pk', flat=True)
query = (
Q('pk', 'in', node_pks) &
req_query
)
nodes = Node.find(query).order_by('-date_modified')
auth = get_user_auth(self.request)
pks = [each.pk for each in nodes if each.can_view(auth)]
return Node.objects.filter(pk__in=pks).order_by('-date_modified')
def migrate_nodes(index, query=None):
logger.info('Migrating nodes to index: {}'.format(index))
node_query = Q('is_public', 'eq', True) & Q('is_deleted', 'eq', False)
if query:
node_query = query & node_query
total = Node.find(node_query).count()
increment = 200
total_pages = (total // increment) + 1
pages = paginated(Node, query=node_query, increment=increment, each=False, include=['contributor__user__guids'])
for page_number, page in enumerate(pages):
logger.info('Updating page {} / {}'.format(page_number + 1, total_pages))
Node.bulk_update_search(page, index=index)
logger.info('Nodes migrated: {}'.format(total))
def project_tag(tag, auth, **kwargs):
tag_obj = Tag.load(tag)
if tag_obj:
nodes = AbstractNode.find(Q('tags', 'eq', tag_obj._id))
else:
nodes = []
visible_nodes = [obj for obj in nodes if obj.can_view(auth)]
return {
'nodes': [{
'title': node.title,
'url': node.url,
} for node in visible_nodes],
'tag':
tag,
}
def test_integration(self, mock_upload, mock_send_mail):
fullname = 'John Deacon'
username = '******'
title = 'good songs'
conference = ConferenceFactory()
body = 'dragon on my back'
content = 'dragon attack'
recipient = '{0}{1}[email protected]'.format(
'test-' if settings.DEV_MODE else '',
conference.endpoint,
)
self.app.post(
api_url_for('meeting_hook'),
{
'X-Mailgun-Sscore': 0,
'timestamp': '123',
'token': 'secret',
'signature': hmac.new(
key=settings.MAILGUN_API_KEY,
msg='{}{}'.format('123', 'secret'),
digestmod=hashlib.sha256,
).hexdigest(),
'attachment-count': '1',
'X-Mailgun-Sscore': 0,
'from': '{0} <{1}>'.format(fullname, username),
'recipient': recipient,
'subject': title,
'stripped-text': body,
},
upload_files=[
('attachment-1', 'attachment-1', content),
],
)
assert_true(mock_upload.called)
users = User.find(Q('username', 'eq', username))
assert_equal(users.count(), 1)
nodes = Node.find(Q('title', 'eq', title))
assert_equal(nodes.count(), 1)
node = nodes[0]
assert_equal(node.get_wiki_page('home').content, body)
assert_true(mock_send_mail.called)
call_args, call_kwargs = mock_send_mail.call_args
assert_absolute(call_kwargs['conf_view_url'])
assert_absolute(call_kwargs['set_password_url'])
assert_absolute(call_kwargs['profile_url'])
assert_absolute(call_kwargs['file_url'])
assert_absolute(call_kwargs['node_url'])
def search_node(auth, **kwargs):
"""
"""
# Get arguments
node = AbstractNode.load(request.json.get('nodeId'))
include_public = request.json.get('includePublic')
size = float(request.json.get('size', '5').strip())
page = request.json.get('page', 0)
query = request.json.get('query', '').strip()
start = (page * size)
if not query:
return {'nodes': []}
# Build ODM query
title_query = Q('title', 'icontains', query)
not_deleted_query = Q('is_deleted', 'eq', False)
visibility_query = Q('contributors', 'eq', auth.user)
if include_public:
visibility_query = visibility_query | Q('is_public', 'eq', True)
odm_query = title_query & not_deleted_query & visibility_query
# Exclude current node from query if provided
nin = [node.id] + list(node._nodes.values_list('pk',
flat=True)) if node else []
nodes = AbstractNode.find(odm_query).exclude(id__in=nin).exclude(
type='osf.collection')
count = nodes.count()
pages = math.ceil(count / size)
validate_page_num(page, pages)
return {
'nodes': [
_serialize_node_search(each)
for each in islice(nodes, start, start + size) if each.contributors
],
'total':
count,
'pages':
pages,
'page':
page
}
def test_POST_register_embargo_does_not_make_project_or_children_public(self, mock_enqueue):
self.project.is_public = True
self.project.save()
NodeFactory( # component
creator=self.user,
parent=self.project,
title='Component',
is_public=True
)
subproject = ProjectFactory(
creator=self.user,
parent=self.project,
title='Subproject',
is_public=True
)
NodeFactory( # subproject's component
creator=self.user,
parent=subproject,
title='Subcomponent',
is_public=True
)
res = self.app.post(
self.project.api_url_for('register_draft_registration', draft_id=self.draft._id),
self.valid_embargo_payload,
content_type='application/json',
auth=self.user.auth
)
self.project.reload()
assert_equal(res.status_code, 202)
assert_equal(res.json['urls']['registrations'], self.project.web_url_for('node_registrations'))
# Last node directly registered from self.project
registration = Node.find(
Q('registered_from', 'eq', self.project)
).order_by('-registered_date')[0]
assert_true(registration.is_registration)
assert_false(registration.is_public)
assert_true(registration.is_pending_embargo_for_existing_registration)
assert_is_not_none(registration.embargo)
for node in registration.get_descendants_recursive():
assert_true(node.is_registration)
assert_false(node.is_public)
def migrate_nodes(index, query=None):
logger.info('Migrating nodes to index: {}'.format(index))
node_query = Q('is_public', 'eq', True) & Q('is_deleted', 'eq', False)
if query:
node_query = query & node_query
total = AbstractNode.find(node_query).count()
increment = 200
total_pages = (total // increment) + 1
pages = paginated(AbstractNode,
query=node_query,
increment=increment,
each=False,
include=['contributor__user__guids'])
for page_number, page in enumerate(pages):
logger.info('Updating page {} / {}'.format(page_number + 1,
total_pages))
AbstractNode.bulk_update_search(page, index=index)
logger.info('Nodes migrated: {}'.format(total))
def search_node(auth, **kwargs):
"""
"""
# Get arguments
node = Node.load(request.json.get('nodeId'))
include_public = request.json.get('includePublic')
size = float(request.json.get('size', '5').strip())
page = request.json.get('page', 0)
query = request.json.get('query', '').strip()
start = (page * size)
if not query:
return {'nodes': []}
# Build ODM query
title_query = Q('title', 'icontains', query)
not_deleted_query = Q('is_deleted', 'eq', False)
visibility_query = Q('contributors', 'eq', auth.user)
if include_public:
visibility_query = visibility_query | Q('is_public', 'eq', True)
odm_query = title_query & not_deleted_query & visibility_query
# Exclude current node from query if provided
nin = [node.id] + list(node._nodes.values_list('pk', flat=True)) if node else []
nodes = Node.find(odm_query).exclude(id__in=nin).exclude(type='osf.collection')
count = nodes.count()
pages = math.ceil(count / size)
validate_page_num(page, pages)
return {
'nodes': [
_serialize_node_search(each)
for each in islice(nodes, start, start + size)
if each.contributors
],
'total': count,
'pages': pages,
'page': page
}
def test_POST_register_embargo_does_not_make_project_or_children_public(
self, mock_enqueue):
self.project.is_public = True
self.project.save()
NodeFactory( # component
creator=self.user,
parent=self.project,
title='Component',
is_public=True)
subproject = ProjectFactory(creator=self.user,
parent=self.project,
title='Subproject',
is_public=True)
NodeFactory( # subproject's component
creator=self.user,
parent=subproject,
title='Subcomponent',
is_public=True)
res = self.app.post(self.project.api_url_for(
'register_draft_registration', draft_id=self.draft._id),
self.valid_embargo_payload,
content_type='application/json',
auth=self.user.auth)
self.project.reload()
assert_equal(res.status_code, 202)
assert_equal(res.json['urls']['registrations'],
self.project.web_url_for('node_registrations'))
# Last node directly registered from self.project
registration = AbstractNode.find(
Q('registered_from', 'eq',
self.project)).order_by('-registered_date')[0]
assert_true(registration.is_registration)
assert_false(registration.is_public)
assert_true(registration.is_pending_embargo_for_existing_registration)
assert_is_not_none(registration.embargo)
for node in registration.get_descendants_recursive():
assert_true(node.is_registration)
assert_false(node.is_public)
def search_projects_by_title(**kwargs):
""" Search for nodes by title. Can pass in arguments from the URL to modify the search
:arg term: The substring of the title.
:arg category: Category of the node.
:arg isDeleted: yes, no, or either. Either will not add a qualifier for that argument in the search.
:arg isFolder: yes, no, or either. Either will not add a qualifier for that argument in the search.
:arg isRegistration: yes, no, or either. Either will not add a qualifier for that argument in the search.
:arg includePublic: yes or no. Whether the projects listed should include public projects.
:arg includeContributed: yes or no. Whether the search should include projects the current user has
contributed to.
:arg ignoreNode: a list of nodes that should not be included in the search.
:return: a list of dictionaries of projects
"""
# TODO(fabianvf): At some point, it would be nice to do this with elastic search
user = kwargs['auth'].user
term = request.args.get('term', '')
max_results = int(request.args.get('maxResults', '10'))
category = request.args.get('category', 'project').lower()
is_deleted = request.args.get('isDeleted', 'no').lower()
is_collection = request.args.get('isFolder', 'no').lower()
is_registration = request.args.get('isRegistration', 'no').lower()
include_public = request.args.get('includePublic', 'yes').lower()
include_contributed = request.args.get('includeContributed', 'yes').lower()
ignore_nodes = request.args.getlist('ignoreNode', [])
matching_title = (
Q('title', 'icontains', term) & # search term (case insensitive)
Q('category', 'eq', category) # is a project
)
matching_title = conditionally_add_query_item(matching_title, 'is_deleted', is_deleted)
# TODO: Why.
if is_registration == 'yes':
matching_title &= Q('type', 'eq', 'osf.registration')
elif is_registration == 'no':
matching_title &= Q('type', 'ne', 'osf.registration')
if is_collection == 'yes':
matching_title &= Q('type', 'eq', 'osf.collection')
elif is_collection == 'no':
matching_title &= Q('type', 'ne', 'osf.collection')
if len(ignore_nodes) > 0:
for node_id in ignore_nodes:
matching_title = matching_title & Q('_id', 'ne', node_id)
my_projects = []
my_project_count = 0
public_projects = []
if include_contributed == 'yes':
my_projects = AbstractNode.find(
matching_title &
Q('contributors', 'eq', user) # user is a contributor
).limit(max_results)
my_project_count = my_project_count
if my_project_count < max_results and include_public == 'yes':
public_projects = AbstractNode.find(
matching_title &
Q('is_public', 'eq', True) # is public
).limit(max_results - my_project_count)
results = list(my_projects) + list(public_projects)
ret = process_project_search_results(results, **kwargs)
return ret
def get_default_queryset(self):
user = self.get_user()
query = MQ('contributors', 'eq', user) & default_node_list_query()
if user != self.request.user:
query &= default_node_permission_query(self.request.user)
return Node.find(query)
def search_projects_by_title(**kwargs):
""" Search for nodes by title. Can pass in arguments from the URL to modify the search
:arg term: The substring of the title.
:arg category: Category of the node.
:arg isDeleted: yes, no, or either. Either will not add a qualifier for that argument in the search.
:arg isFolder: yes, no, or either. Either will not add a qualifier for that argument in the search.
:arg isRegistration: yes, no, or either. Either will not add a qualifier for that argument in the search.
:arg includePublic: yes or no. Whether the projects listed should include public projects.
:arg includeContributed: yes or no. Whether the search should include projects the current user has
contributed to.
:arg ignoreNode: a list of nodes that should not be included in the search.
:return: a list of dictionaries of projects
"""
# TODO(fabianvf): At some point, it would be nice to do this with elastic search
user = kwargs['auth'].user
term = request.args.get('term', '')
max_results = int(request.args.get('maxResults', '10'))
category = request.args.get('category', 'project').lower()
is_deleted = request.args.get('isDeleted', 'no').lower()
is_collection = request.args.get('isFolder', 'no').lower()
is_registration = request.args.get('isRegistration', 'no').lower()
include_public = request.args.get('includePublic', 'yes').lower()
include_contributed = request.args.get('includeContributed', 'yes').lower()
ignore_nodes = request.args.getlist('ignoreNode', [])
matching_title = (
Q('title', 'icontains', term) & # search term (case insensitive)
Q('category', 'eq', category) # is a project
)
matching_title = conditionally_add_query_item(matching_title, 'is_deleted', is_deleted)
# TODO: Why.
if is_registration == 'yes':
matching_title &= Q('type', 'eq', 'osf.registration')
elif is_registration == 'no':
matching_title &= Q('type', 'ne', 'osf.registration')
if is_collection == 'yes':
matching_title &= Q('type', 'eq', 'osf.collection')
elif is_collection == 'no':
matching_title &= Q('type', 'ne', 'osf.collection')
if len(ignore_nodes) > 0:
for node_id in ignore_nodes:
matching_title = matching_title & Q('_id', 'ne', node_id)
my_projects = []
my_project_count = 0
public_projects = []
if include_contributed == 'yes':
my_projects = AbstractNode.find(
matching_title &
Q('contributors', 'eq', user) # user is a contributor
)[:max_results]
my_project_count = my_project_count
if my_project_count < max_results and include_public == 'yes':
public_projects = AbstractNode.find(
matching_title &
Q('is_public', 'eq', True) # is public
)[:max_results - my_project_count]
results = list(my_projects) + list(public_projects)
ret = process_project_search_results(results, **kwargs)
return ret
def nodes_since(user, date):
return AbstractNode.find(
Q('creator', 'eq', user._id) &
Q('date_created', 'gt', date)
)
def get_default_queryset(self):
user = self.get_user()
query = MQ('contributors', 'eq', user) & default_node_list_query()
if user != self.request.user:
query &= default_node_permission_query(self.request.user)
return AbstractNode.find(query)
