def conference_submissions(**kwargs): """Return data for all OSF4M submissions. The total number of submissions for each meeting is calculated and cached in the Conference.num_submissions field. """ conferences = Conference.find(Q('is_meeting', 'ne', False)) # TODO: Revisit this loop, there has to be a way to optimize it for conf in conferences: # For efficiency, we filter by tag first, then node # instead of doing a single Node query projects = set() tags = Tag.find(Q('system', 'eq', False) & Q('name', 'iexact', conf.endpoint.lower())).values_list('pk', flat=True) nodes = Node.find( Q('tags', 'in', tags) & Q('is_public', 'eq', True) & Q('is_deleted', 'ne', True) ).include('guids') projects.update(list(nodes)) num_submissions = len(projects) # Cache the number of submissions conf.num_submissions = num_submissions bulk_update(conferences, update_fields=['num_submissions']) return {'success': True}
def conference_submissions(**kwargs): """Return data for all OSF4M submissions. The total number of submissions for each meeting is calculated and cached in the Conference.num_submissions field. """ conferences = Conference.find(Q('is_meeting', 'ne', False)) # TODO: Revisit this loop, there has to be a way to optimize it for conf in conferences: # For efficiency, we filter by tag first, then node # instead of doing a single Node query projects = set() tags = Tag.find( Q('system', 'eq', False) & Q('name', 'iexact', conf.endpoint.lower())).values_list( 'pk', flat=True) nodes = AbstractNode.find( Q('tags', 'in', tags) & Q('is_public', 'eq', True) & Q('is_deleted', 'ne', True)).include('guids') projects.update(list(nodes)) num_submissions = len(projects) # Cache the number of submissions conf.num_submissions = num_submissions bulk_update(conferences, update_fields=['num_submissions']) return {'success': True}
def test_integration(self, mock_upload, mock_send_mail): fullname = 'John Deacon' username = '******' title = 'good songs' conference = ConferenceFactory() body = 'dragon on my back' content = 'dragon attack' recipient = '{0}{1}[email protected]'.format( 'test-' if settings.DEV_MODE else '', conference.endpoint, ) self.app.post( api_url_for('meeting_hook'), { 'X-Mailgun-Sscore': 0, 'timestamp': '123', 'token': 'secret', 'signature': hmac.new( key=settings.MAILGUN_API_KEY, msg='{}{}'.format('123', 'secret'), digestmod=hashlib.sha256, ).hexdigest(), 'attachment-count': '1', 'X-Mailgun-Sscore': 0, 'from': '{0} <{1}>'.format(fullname, username), 'recipient': recipient, 'subject': title, 'stripped-text': body, }, upload_files=[ ('attachment-1', 'attachment-1', content), ], ) assert_true(mock_upload.called) users = OSFUser.find(Q('username', 'eq', username)) assert_equal(users.count(), 1) nodes = AbstractNode.find(Q('title', 'eq', title)) assert_equal(nodes.count(), 1) node = nodes[0] assert_equal(node.get_wiki_page('home').content, body) assert_true(mock_send_mail.called) call_args, call_kwargs = mock_send_mail.call_args assert_absolute(call_kwargs['conf_view_url']) assert_absolute(call_kwargs['set_password_url']) assert_absolute(call_kwargs['profile_url']) assert_absolute(call_kwargs['file_url']) assert_absolute(call_kwargs['node_url'])
def get_queryset(self): blacklisted = self.is_blacklisted() nodes = self.get_queryset_from_request().distinct() # If attempting to filter on a blacklisted field, exclude withdrawals. if blacklisted: non_withdrawn_list = [node._id for node in nodes if not node.is_retracted] non_withdrawn_nodes = Node.find(Q('_id', 'in', non_withdrawn_list)) return non_withdrawn_nodes return nodes
def get_default_queryset(self): base_query = (Q('is_deleted', 'ne', True) & Q('type', 'eq', 'osf.registration')) user = self.request.user permission_query = Q('is_public', 'eq', True) if not user.is_anonymous: permission_query = (permission_query | Q('contributors', 'eq', user)) query = base_query & permission_query return AbstractNode.find(query)
def get_queryset(self): blacklisted = self.is_blacklisted() nodes = self.get_queryset_from_request().distinct() # If attempting to filter on a blacklisted field, exclude withdrawals. if blacklisted: non_withdrawn_list = [ node._id for node in nodes if not node.is_retracted ] non_withdrawn_nodes = AbstractNode.find( Q('_id', 'in', non_withdrawn_list)) return non_withdrawn_nodes return nodes
def get_default_queryset(self): base_query = ( Q('is_deleted', 'ne', True) & Q('type', 'eq', 'osf.registration') ) user = self.request.user permission_query = Q('is_public', 'eq', True) if not user.is_anonymous: permission_query = (permission_query | Q('contributors', 'eq', user)) query = base_query & permission_query return Node.find(query)
def get_queryset(self): node = self.get_node() req_query = self.get_query_from_request() node_pks = node.node_relations.filter( is_node_link=False).select_related('child').values_list( 'child__pk', flat=True) query = (Q('pk', 'in', node_pks) & req_query) nodes = AbstractNode.find(query).order_by('-date_modified') auth = get_user_auth(self.request) pks = [each.pk for each in nodes if each.can_view(auth)] return AbstractNode.objects.filter( pk__in=pks).order_by('-date_modified')
def get_default_queryset(self): user = self.get_user() current_user = self.request.user query = ( MQ('is_deleted', 'ne', True) & MQ('type', 'eq', 'osf.registration') & MQ('contributors', 'eq', user) ) permission_query = MQ('is_public', 'eq', True) if not current_user.is_anonymous: permission_query = (permission_query | MQ('contributors', 'eq', current_user)) query = query & permission_query return AbstractNode.find(query)
def get_default_queryset(self): user = self.get_user() current_user = self.request.user query = ( MQ('is_deleted', 'ne', True) & MQ('type', 'eq', 'osf.registration') & MQ('contributors', 'eq', user) ) permission_query = MQ('is_public', 'eq', True) if not current_user.is_anonymous: permission_query = (permission_query | MQ('contributors', 'eq', current_user)) query = query & permission_query return Node.find(query)
def get_queryset(self): node = self.get_node() req_query = self.get_query_from_request() node_pks = node.node_relations.filter(is_node_link=False).select_related('child').values_list('child__pk', flat=True) query = ( Q('pk', 'in', node_pks) & req_query ) nodes = Node.find(query).order_by('-date_modified') auth = get_user_auth(self.request) pks = [each.pk for each in nodes if each.can_view(auth)] return Node.objects.filter(pk__in=pks).order_by('-date_modified')
def migrate_nodes(index, query=None): logger.info('Migrating nodes to index: {}'.format(index)) node_query = Q('is_public', 'eq', True) & Q('is_deleted', 'eq', False) if query: node_query = query & node_query total = Node.find(node_query).count() increment = 200 total_pages = (total // increment) + 1 pages = paginated(Node, query=node_query, increment=increment, each=False, include=['contributor__user__guids']) for page_number, page in enumerate(pages): logger.info('Updating page {} / {}'.format(page_number + 1, total_pages)) Node.bulk_update_search(page, index=index) logger.info('Nodes migrated: {}'.format(total))
def project_tag(tag, auth, **kwargs): tag_obj = Tag.load(tag) if tag_obj: nodes = AbstractNode.find(Q('tags', 'eq', tag_obj._id)) else: nodes = [] visible_nodes = [obj for obj in nodes if obj.can_view(auth)] return { 'nodes': [{ 'title': node.title, 'url': node.url, } for node in visible_nodes], 'tag': tag, }
def test_integration(self, mock_upload, mock_send_mail): fullname = 'John Deacon' username = '******' title = 'good songs' conference = ConferenceFactory() body = 'dragon on my back' content = 'dragon attack' recipient = '{0}{1}[email protected]'.format( 'test-' if settings.DEV_MODE else '', conference.endpoint, ) self.app.post( api_url_for('meeting_hook'), { 'X-Mailgun-Sscore': 0, 'timestamp': '123', 'token': 'secret', 'signature': hmac.new( key=settings.MAILGUN_API_KEY, msg='{}{}'.format('123', 'secret'), digestmod=hashlib.sha256, ).hexdigest(), 'attachment-count': '1', 'X-Mailgun-Sscore': 0, 'from': '{0} <{1}>'.format(fullname, username), 'recipient': recipient, 'subject': title, 'stripped-text': body, }, upload_files=[ ('attachment-1', 'attachment-1', content), ], ) assert_true(mock_upload.called) users = User.find(Q('username', 'eq', username)) assert_equal(users.count(), 1) nodes = Node.find(Q('title', 'eq', title)) assert_equal(nodes.count(), 1) node = nodes[0] assert_equal(node.get_wiki_page('home').content, body) assert_true(mock_send_mail.called) call_args, call_kwargs = mock_send_mail.call_args assert_absolute(call_kwargs['conf_view_url']) assert_absolute(call_kwargs['set_password_url']) assert_absolute(call_kwargs['profile_url']) assert_absolute(call_kwargs['file_url']) assert_absolute(call_kwargs['node_url'])
def search_node(auth, **kwargs): """ """ # Get arguments node = AbstractNode.load(request.json.get('nodeId')) include_public = request.json.get('includePublic') size = float(request.json.get('size', '5').strip()) page = request.json.get('page', 0) query = request.json.get('query', '').strip() start = (page * size) if not query: return {'nodes': []} # Build ODM query title_query = Q('title', 'icontains', query) not_deleted_query = Q('is_deleted', 'eq', False) visibility_query = Q('contributors', 'eq', auth.user) if include_public: visibility_query = visibility_query | Q('is_public', 'eq', True) odm_query = title_query & not_deleted_query & visibility_query # Exclude current node from query if provided nin = [node.id] + list(node._nodes.values_list('pk', flat=True)) if node else [] nodes = AbstractNode.find(odm_query).exclude(id__in=nin).exclude( type='osf.collection') count = nodes.count() pages = math.ceil(count / size) validate_page_num(page, pages) return { 'nodes': [ _serialize_node_search(each) for each in islice(nodes, start, start + size) if each.contributors ], 'total': count, 'pages': pages, 'page': page }
def test_POST_register_embargo_does_not_make_project_or_children_public(self, mock_enqueue): self.project.is_public = True self.project.save() NodeFactory( # component creator=self.user, parent=self.project, title='Component', is_public=True ) subproject = ProjectFactory( creator=self.user, parent=self.project, title='Subproject', is_public=True ) NodeFactory( # subproject's component creator=self.user, parent=subproject, title='Subcomponent', is_public=True ) res = self.app.post( self.project.api_url_for('register_draft_registration', draft_id=self.draft._id), self.valid_embargo_payload, content_type='application/json', auth=self.user.auth ) self.project.reload() assert_equal(res.status_code, 202) assert_equal(res.json['urls']['registrations'], self.project.web_url_for('node_registrations')) # Last node directly registered from self.project registration = Node.find( Q('registered_from', 'eq', self.project) ).order_by('-registered_date')[0] assert_true(registration.is_registration) assert_false(registration.is_public) assert_true(registration.is_pending_embargo_for_existing_registration) assert_is_not_none(registration.embargo) for node in registration.get_descendants_recursive(): assert_true(node.is_registration) assert_false(node.is_public)
def migrate_nodes(index, query=None): logger.info('Migrating nodes to index: {}'.format(index)) node_query = Q('is_public', 'eq', True) & Q('is_deleted', 'eq', False) if query: node_query = query & node_query total = AbstractNode.find(node_query).count() increment = 200 total_pages = (total // increment) + 1 pages = paginated(AbstractNode, query=node_query, increment=increment, each=False, include=['contributor__user__guids']) for page_number, page in enumerate(pages): logger.info('Updating page {} / {}'.format(page_number + 1, total_pages)) AbstractNode.bulk_update_search(page, index=index) logger.info('Nodes migrated: {}'.format(total))
def search_node(auth, **kwargs): """ """ # Get arguments node = Node.load(request.json.get('nodeId')) include_public = request.json.get('includePublic') size = float(request.json.get('size', '5').strip()) page = request.json.get('page', 0) query = request.json.get('query', '').strip() start = (page * size) if not query: return {'nodes': []} # Build ODM query title_query = Q('title', 'icontains', query) not_deleted_query = Q('is_deleted', 'eq', False) visibility_query = Q('contributors', 'eq', auth.user) if include_public: visibility_query = visibility_query | Q('is_public', 'eq', True) odm_query = title_query & not_deleted_query & visibility_query # Exclude current node from query if provided nin = [node.id] + list(node._nodes.values_list('pk', flat=True)) if node else [] nodes = Node.find(odm_query).exclude(id__in=nin).exclude(type='osf.collection') count = nodes.count() pages = math.ceil(count / size) validate_page_num(page, pages) return { 'nodes': [ _serialize_node_search(each) for each in islice(nodes, start, start + size) if each.contributors ], 'total': count, 'pages': pages, 'page': page }
def test_POST_register_embargo_does_not_make_project_or_children_public( self, mock_enqueue): self.project.is_public = True self.project.save() NodeFactory( # component creator=self.user, parent=self.project, title='Component', is_public=True) subproject = ProjectFactory(creator=self.user, parent=self.project, title='Subproject', is_public=True) NodeFactory( # subproject's component creator=self.user, parent=subproject, title='Subcomponent', is_public=True) res = self.app.post(self.project.api_url_for( 'register_draft_registration', draft_id=self.draft._id), self.valid_embargo_payload, content_type='application/json', auth=self.user.auth) self.project.reload() assert_equal(res.status_code, 202) assert_equal(res.json['urls']['registrations'], self.project.web_url_for('node_registrations')) # Last node directly registered from self.project registration = AbstractNode.find( Q('registered_from', 'eq', self.project)).order_by('-registered_date')[0] assert_true(registration.is_registration) assert_false(registration.is_public) assert_true(registration.is_pending_embargo_for_existing_registration) assert_is_not_none(registration.embargo) for node in registration.get_descendants_recursive(): assert_true(node.is_registration) assert_false(node.is_public)
def search_projects_by_title(**kwargs): """ Search for nodes by title. Can pass in arguments from the URL to modify the search :arg term: The substring of the title. :arg category: Category of the node. :arg isDeleted: yes, no, or either. Either will not add a qualifier for that argument in the search. :arg isFolder: yes, no, or either. Either will not add a qualifier for that argument in the search. :arg isRegistration: yes, no, or either. Either will not add a qualifier for that argument in the search. :arg includePublic: yes or no. Whether the projects listed should include public projects. :arg includeContributed: yes or no. Whether the search should include projects the current user has contributed to. :arg ignoreNode: a list of nodes that should not be included in the search. :return: a list of dictionaries of projects """ # TODO(fabianvf): At some point, it would be nice to do this with elastic search user = kwargs['auth'].user term = request.args.get('term', '') max_results = int(request.args.get('maxResults', '10')) category = request.args.get('category', 'project').lower() is_deleted = request.args.get('isDeleted', 'no').lower() is_collection = request.args.get('isFolder', 'no').lower() is_registration = request.args.get('isRegistration', 'no').lower() include_public = request.args.get('includePublic', 'yes').lower() include_contributed = request.args.get('includeContributed', 'yes').lower() ignore_nodes = request.args.getlist('ignoreNode', []) matching_title = ( Q('title', 'icontains', term) & # search term (case insensitive) Q('category', 'eq', category) # is a project ) matching_title = conditionally_add_query_item(matching_title, 'is_deleted', is_deleted) # TODO: Why. if is_registration == 'yes': matching_title &= Q('type', 'eq', 'osf.registration') elif is_registration == 'no': matching_title &= Q('type', 'ne', 'osf.registration') if is_collection == 'yes': matching_title &= Q('type', 'eq', 'osf.collection') elif is_collection == 'no': matching_title &= Q('type', 'ne', 'osf.collection') if len(ignore_nodes) > 0: for node_id in ignore_nodes: matching_title = matching_title & Q('_id', 'ne', node_id) my_projects = [] my_project_count = 0 public_projects = [] if include_contributed == 'yes': my_projects = AbstractNode.find( matching_title & Q('contributors', 'eq', user) # user is a contributor ).limit(max_results) my_project_count = my_project_count if my_project_count < max_results and include_public == 'yes': public_projects = AbstractNode.find( matching_title & Q('is_public', 'eq', True) # is public ).limit(max_results - my_project_count) results = list(my_projects) + list(public_projects) ret = process_project_search_results(results, **kwargs) return ret
def get_default_queryset(self): user = self.get_user() query = MQ('contributors', 'eq', user) & default_node_list_query() if user != self.request.user: query &= default_node_permission_query(self.request.user) return Node.find(query)
def search_projects_by_title(**kwargs): """ Search for nodes by title. Can pass in arguments from the URL to modify the search :arg term: The substring of the title. :arg category: Category of the node. :arg isDeleted: yes, no, or either. Either will not add a qualifier for that argument in the search. :arg isFolder: yes, no, or either. Either will not add a qualifier for that argument in the search. :arg isRegistration: yes, no, or either. Either will not add a qualifier for that argument in the search. :arg includePublic: yes or no. Whether the projects listed should include public projects. :arg includeContributed: yes or no. Whether the search should include projects the current user has contributed to. :arg ignoreNode: a list of nodes that should not be included in the search. :return: a list of dictionaries of projects """ # TODO(fabianvf): At some point, it would be nice to do this with elastic search user = kwargs['auth'].user term = request.args.get('term', '') max_results = int(request.args.get('maxResults', '10')) category = request.args.get('category', 'project').lower() is_deleted = request.args.get('isDeleted', 'no').lower() is_collection = request.args.get('isFolder', 'no').lower() is_registration = request.args.get('isRegistration', 'no').lower() include_public = request.args.get('includePublic', 'yes').lower() include_contributed = request.args.get('includeContributed', 'yes').lower() ignore_nodes = request.args.getlist('ignoreNode', []) matching_title = ( Q('title', 'icontains', term) & # search term (case insensitive) Q('category', 'eq', category) # is a project ) matching_title = conditionally_add_query_item(matching_title, 'is_deleted', is_deleted) # TODO: Why. if is_registration == 'yes': matching_title &= Q('type', 'eq', 'osf.registration') elif is_registration == 'no': matching_title &= Q('type', 'ne', 'osf.registration') if is_collection == 'yes': matching_title &= Q('type', 'eq', 'osf.collection') elif is_collection == 'no': matching_title &= Q('type', 'ne', 'osf.collection') if len(ignore_nodes) > 0: for node_id in ignore_nodes: matching_title = matching_title & Q('_id', 'ne', node_id) my_projects = [] my_project_count = 0 public_projects = [] if include_contributed == 'yes': my_projects = AbstractNode.find( matching_title & Q('contributors', 'eq', user) # user is a contributor )[:max_results] my_project_count = my_project_count if my_project_count < max_results and include_public == 'yes': public_projects = AbstractNode.find( matching_title & Q('is_public', 'eq', True) # is public )[:max_results - my_project_count] results = list(my_projects) + list(public_projects) ret = process_project_search_results(results, **kwargs) return ret
def nodes_since(user, date): return AbstractNode.find( Q('creator', 'eq', user._id) & Q('date_created', 'gt', date) )
def get_default_queryset(self): user = self.get_user() query = MQ('contributors', 'eq', user) & default_node_list_query() if user != self.request.user: query &= default_node_permission_query(self.request.user) return AbstractNode.find(query)