def do_validate(self, value, obj): # Check if required if value is None: if getattr(self, '_required', None): raise exceptions.ValidationError( 'Value <{0}> is required.'.format(self._field_name)) return True # Check if unique if value is not None and self._unique: unique_query = Q(self._field_name, 'eq', value) # If object has primary key, don't crash if unique value is # already associated with its key if obj._is_loaded: unique_query = unique_query & Q(obj._primary_name, 'ne', obj._primary_key) if obj.find(unique_query).limit(1).count(): raise exceptions.ValidationValueError('Value must be unique') # Field-level validation cls = self.__class__ if hasattr(cls, 'validate') and self.validate is not False: cls.validate(value) # Schema-level validation if self._validate and hasattr(self, 'validate'): if hasattr(self.validate, '__iter__'): for validator in self.validate: validator(value) elif hasattr(self.validate, '__call__'): self.validate(value) # Success return True
def test_or(self): """Find the union of two or more queries.""" result = self.Foo.find(Q('a', 'eq', 0) | Q('a', 'eq', 1)) self.assertEqual( len(result), 6, )
def test_and(self): """Find the intersection of two or more queries.""" result = self.Foo.find(Q('a', 'eq', 0) & Q('b', 'eq', 1)) self.assertEqual( len(result), 1, ) self.assertEqual(result[0].a, 0) self.assertEqual(result[0].b, 1)
def test_and_or(self): """Join multiple OR queries with an AND. """ result = self.Foo.find((Q('a', 'eq', 0) | Q('a', 'eq', 1)) & (Q('b', 'eq', 1) | Q('b', 'eq', 2))) self.assertEqual( len(result), 4, )
def migrate_nodes(index): logger.info("Migrating nodes to index: {}".format(index)) n_iter = 0 nodes = Node.find( Q('is_public', 'eq', True) & Q('is_deleted', 'eq', False)) for node in nodes: search.update_node(node, index=index) n_iter += 1 logger.info('Nodes migrated: {}'.format(n_iter))
def migrate_nodes(index): logger.info('Migrating nodes to index: {}'.format(index)) query = Q('is_public', 'eq', True) & Q('is_deleted', 'eq', False) total = Node.find(query).count() increment = 200 total_pages = (total // increment) + 1 pages = paginated(Node, query=query, increment=increment, each=False) for page_number, page in enumerate(pages): logger.info('Updating page {} / {}'.format(page_number + 1, total_pages)) Node.bulk_update_search(page, index=index) Node._clear_caches() logger.info('Nodes migrated: {}'.format(total))
def test_not(self): """Find the inverse of a query.""" result = self.Foo.find(~Q('a', 'eq', 0)) self.assertEqual( len(result), 6, )
def find(self, query=None): """ Find backrefs matching a given query. """ combined_query = Q(self._base_class._primary_name, 'in', self._to_primary_keys()) if query is not None: combined_query = combined_query & query return self._base_class.find(combined_query)
def migrate(delete, index=None, app=None): index = index or settings.ELASTIC_INDEX app = app or init_app('website.settings', set_backends=True, routes=True) script_utils.add_file_logger(logger, __file__) # NOTE: We do NOT use the app.text_request_context() as a # context manager because we don't want the teardown_request # functions to be triggered ctx = app.test_request_context() ctx.push() new_index = set_up_index(index) start_time = timezone.now() if settings.ENABLE_INSTITUTIONS: migrate_institutions(new_index) migrate_nodes(new_index) migrate_users(new_index) set_up_alias(index, new_index) # migrate nodes modified since start migrate_nodes(new_index, query=Q('date_modified', 'gte', start_time)) if delete: delete_old(new_index) ctx.pop()
def create_fake_project(creator, n_users, privacy, n_components, name, n_tags, presentation_name, is_registration, is_preprint, preprint_provider): auth = Auth(user=creator) project_title = name if name else fake.science_sentence() if is_preprint: provider = None if preprint_provider: try: provider = models.PreprintProvider.find_one( Q('_id', 'eq', provider)) except NoResultsFound: pass if not provider: provider = PreprintProviderFactory(name=fake.science_word()) privacy = 'public' mock_change_identifier = mock.patch( 'website.identifiers.client.EzidClient.change_status_identifier') mock_change_identifier.start() project = PreprintFactory(title=project_title, description=fake.science_paragraph(), creator=creator, provider=provider) node = project.node elif is_registration: project = RegistrationFactory(title=project_title, description=fake.science_paragraph(), creator=creator) node = project else: project = ProjectFactory(title=project_title, description=fake.science_paragraph(), creator=creator) node = project node.set_privacy(privacy) for _ in range(n_users): contrib = create_fake_user() node.add_contributor(contrib, auth=auth) if isinstance(n_components, int): for _ in range(n_components): NodeFactory(parent=node, title=fake.science_sentence(), description=fake.science_paragraph(), creator=creator) elif isinstance(n_components, list): render_generations_from_node_structure_list(node, creator, n_components) for _ in range(n_tags): node.add_tag(fake.science_word(), auth=auth) if presentation_name is not None: node.add_tag(presentation_name, auth=auth) node.add_tag('poster', auth=auth) node.save() project.save() logger.info('Created project: {0}'.format(node.title)) return project
def main(): args = parse_args() creator = models.User.find(Q('username', 'eq', args.user))[0] for i in range(args.n_projects): name = args.name + str(i) if args.name else '' create_fake_project(creator, args.n_users, args.privacy, args.n_components, name, args.n_tags, args.presentation_name, args.is_registration) print('Created {n} fake projects.'.format(n=args.n_projects)) sys.exit(0)
def test_update_one(self): """ Given a primary key, update the referenced object according to the update clause """ self.Foo.update_one(which=Q('_id', 'eq', 2), data={'modified': True}) self.assertEqual( [x.modified for x in self.foos], [False, False, True, False, False], )
def test_update(self): """ Given a query, and an update clause, update all (and only) object returned by query. """ self.Foo.update(query=Q('_id', 'eq', 2), data={'modified': True}) self.assertEqual( [x.modified for x in self.foos], [False, False, True, False, False], )
def migrate_nodes(index, query=None): logger.info('Migrating nodes to index: {}'.format(index)) node_query = Q('is_public', 'eq', True) & Q('is_deleted', 'eq', False) if query: node_query = query & node_query total = Node.find(node_query).count() increment = 200 total_pages = (total // increment) + 1 pages = paginated(Node, query=node_query, increment=increment, each=False, include=['contributor__user__guids']) for page_number, page in enumerate(pages): logger.info('Updating page {} / {}'.format(page_number + 1, total_pages)) Node.bulk_update_search(page, index=index) logger.info('Nodes migrated: {}'.format(total))
def create_fake_project(creator, n_users, privacy, n_components, name, n_tags, presentation_name, is_registration, is_preprint, preprint_providers): auth = Auth(user=creator) project_title = name if name else fake.science_sentence() if is_preprint: providers_to_add = [] if preprint_providers: providers = preprint_providers.split(',') for provider in providers: try: preprint_provider = models.PreprintProvider.find_one( Q('_id', 'eq', provider)) except NoResultsFound: preprint_provider = PreprintProviderFactory(name=provider) providers_to_add.append(preprint_provider) privacy = 'public' project = PreprintFactory(title=project_title, description=fake.science_paragraph(), creator=creator, providers=providers_to_add) elif is_registration: project = RegistrationFactory(title=project_title, description=fake.science_paragraph(), creator=creator) else: project = ProjectFactory(title=project_title, description=fake.science_paragraph(), creator=creator) project.set_privacy(privacy) for _ in range(n_users): contrib = create_fake_user() project.add_contributor(contrib, auth=auth) if isinstance(n_components, int): for _ in range(n_components): NodeFactory(project=project, title=fake.science_sentence(), description=fake.science_paragraph(), creator=creator) elif isinstance(n_components, list): render_generations_from_node_structure_list(project, creator, n_components) for _ in range(n_tags): project.add_tag(fake.science_word(), auth=auth) if presentation_name is not None: project.add_tag(presentation_name, auth=auth) project.add_tag('poster', auth=auth) project.save() logger.info('Created project: {0}'.format(project.title)) return project
def test_not(self): """ Finds the inverse of a query.""" # This is failing in Mongo, but works in Pickle. Mongo is getting: # {'$not': {'a': 0}} # but it should be getting # {'a': {'$not': 0}} # # See: http://docs.mongodb.org/manual/reference/operator/not/ result = self.Foo.find(~Q('a', 'eq', 0)) self.assertEqual( len(result), 6, )
def activity(): popular_public_projects = [] popular_public_registrations = [] hits = {} if settings.PIWIK_HOST: client = PiwikClient( url=settings.PIWIK_HOST, auth_token=settings.PIWIK_ADMIN_TOKEN, site_id=settings.PIWIK_SITE_ID, period='week', date='today', ) popular_project_ids = [ x for x in client.custom_variables if x.label == 'Project ID' ][0].values for nid in popular_project_ids: node = Node.load(nid.value) if node is None: continue if node.is_public and not node.is_registration and not node.is_deleted: if len(popular_public_projects) < 10: popular_public_projects.append(node) elif node.is_public and node.is_registration and not node.is_deleted: if len(popular_public_registrations) < 10: popular_public_registrations.append(node) if len(popular_public_projects) >= 10 and len(popular_public_registrations) >= 10: break hits = { x.value: { 'hits': x.actions, 'visits': x.visits } for x in popular_project_ids } # Projects recent_query = ( Q('category', 'eq', 'project') & Q('is_public', 'eq', True) & Q('is_deleted', 'eq', False) ) # Temporary bug fix: Skip projects with empty contributor lists # Todo: Fix underlying bug and remove this selector recent_query = recent_query & Q('contributors', 'ne', []) recent_public_projects = Node.find( recent_query & Q('is_registration', 'eq', False) ).sort( '-date_created' ).limit(10) # Registrations recent_public_registrations = Node.find( recent_query & Q('is_registration', 'eq', True) ).sort( '-registered_date' ).limit(10) return { 'recent_public_projects': recent_public_projects, 'recent_public_registrations': recent_public_registrations, 'popular_public_projects': popular_public_projects, 'popular_public_registrations': popular_public_registrations, 'hits': hits, }
def migrate_institutions(index): for inst in Institution.find(Q('is_deleted', 'ne', True)): update_institution(inst, index)
def activity(): popular_public_projects = [] popular_public_registrations = [] hits = {} max_popular_projects = 20 if settings.KEEN['public']['read_key']: client = KeenClient( project_id=settings.KEEN['public']['project_id'], read_key=settings.KEEN['public']['read_key'], ) node_pageviews = client.count( event_collection='pageviews', timeframe='this_7_days', group_by='node.id', filters=[ { 'property_name': 'node.id', 'operator': 'exists', 'property_value': True } ] ) node_visits = client.count_unique( event_collection='pageviews', target_property='anon.id', timeframe='this_7_days', group_by='node.id', filters=[ { 'property_name': 'node.id', 'operator': 'exists', 'property_value': True } ] ) node_data = [{'node': x['node.id'], 'views': x['result']} for x in node_pageviews[0:max_popular_projects]] for node_visit in node_visits[0:max_popular_projects]: for node_result in node_data: if node_visit['node.id'] == node_result['node']: node_result.update({'visits': node_visit['result']}) node_data.sort(key=lambda datum: datum['views'], reverse=True) for nid in node_data: node = Node.load(nid['node']) if node is None: continue if node.is_public and not node.is_registration and not node.is_deleted: if len(popular_public_projects) < 10: popular_public_projects.append(node) elif node.is_public and node.is_registration and not node.is_deleted and not node.is_retracted: if len(popular_public_registrations) < 10: popular_public_registrations.append(node) if len(popular_public_projects) >= 10 and len(popular_public_registrations) >= 10: break hits = { datum['node']: { 'hits': datum['views'], 'visits': datum['visits'] } for datum in node_data } # Projects new_and_noteworthy_pointers = Node.find_one(Q('_id', 'eq', settings.NEW_AND_NOTEWORTHY_LINKS_NODE)).nodes_pointer new_and_noteworthy_projects = [pointer.node for pointer in new_and_noteworthy_pointers] return { 'new_and_noteworthy_projects': new_and_noteworthy_projects, 'recent_public_registrations': recent_public_registrations(), 'popular_public_projects': popular_public_projects, 'popular_public_registrations': popular_public_registrations, 'hits': hits, }
# blog2.tags.append(tag1) blog2.save() blog3.tag = tag1 blog3.save() blog4 = Blog(tags=[tag1]) blog4.save() import pdb pdb.set_trace() res = Tag.find( Q('count', 'startswith', 'count_') & Q('misc', 'endswith', 'bar')) # Tag.find(Q('foo', 'bar', 'baz')) # todo: accept list of strings res = Tag.find_all().sort('misc2', '-misc') print 'here', [(r.misc, r.misc2) for r in res] res = Tag.find(Q('count', 'eq', 'count_1')) print 'here', res.count(), list(res) res = Tag.find(Q('misc', 'startswith', 'foo')) print 'here', res.count(), list(res) res = Tag.find(Q('misc', 'endswith', 'bar')) print 'here', res.count(), list(res)
def migrate_nodes(): for node in Node.find( Q('is_public', 'eq', True) & Q('is_deleted', 'eq', False)): node.update_search()
def migrate_users(): for user in User.find( Q('is_registered', 'eq', True) & Q('date_confirmed', 'ne', None)): user.update_search()
def test_remove(self): """ Given a query, remove all (and only) object returned by query. """ self.Foo.remove(Q('_id', 'eq', 2)) self.assertEqual(self.Foo.find().count(), 4)
blog2.save() blog3.tag = tag1 blog3.save() blog4 = Blog(title='tbdtbdtbd', tags=[tag1, tag2, tag3, tag4, tag5]) blog4.save() logging.debug("bar") import pdb pdb.set_trace() Blog.remove(Q('title', 'startswith', 'tbd')) # res = Tag.find(Q('count', 'startswith', 'count_') & Q('misc', 'endswith', 'bar')) # print 'before rm', res.count() # # Tag.remove(Q('count', 'startswith', 'count_') & Q('misc', 'endswith', 'bar')) # # res = Tag.find(Q('count', 'startswith', 'count_') & Q('misc', 'endswith', 'bar')) # print 'after rm', res.count() Tag.update(Q('count', 'startswith', 'count_'), {'count': 'shutup'}) print tag1.count import pdb pdb.set_trace()
def test_remove_one(self): """ Given a primary key, remove the referenced object. """ self.Foo.remove_one(Q('_id', 'eq', 2)) self.assertEqual(self.Foo.find().count(), 4)
def test_remove_one_returns_none(self): """ Given a primary key, remove the referenced object. """ with self.assertRaises(exceptions.ModularOdmException): self.Foo.remove_one(Q('_id', 'eq', 100))
def activity(): popular_public_projects = [] popular_public_registrations = [] hits = {} # get the date for exactly one week ago target_date = datetime.date.today() - datetime.timedelta(weeks=1) if settings.PIWIK_HOST: client = PiwikClient( url=settings.PIWIK_HOST, auth_token=settings.PIWIK_ADMIN_TOKEN, site_id=settings.PIWIK_SITE_ID, period='week', date=target_date.strftime('%Y-%m-%d'), ) popular_project_ids = [ x for x in client.custom_variables if x.label == 'Project ID' ][0].values for nid in popular_project_ids: node = Node.load(nid.value) if node is None: continue if node.is_public and not node.is_registration and not node.is_deleted: if len(popular_public_projects) < 10: popular_public_projects.append(node) elif node.is_public and node.is_registration and not node.is_deleted: if len(popular_public_registrations) < 10: popular_public_registrations.append(node) if len(popular_public_projects) >= 10 and len(popular_public_registrations) >= 10: break hits = { x.value: { 'hits': x.actions, 'visits': x.visits } for x in popular_project_ids } # Projects recent_query = ( Q('category', 'eq', 'project') & Q('is_public', 'eq', True) & Q('is_deleted', 'eq', False) ) recent_public_projects = Node.find( recent_query & Q('is_registration', 'eq', False) ).sort( '-date_created' ).limit(10) return { 'recent_public_projects': recent_public_projects, 'recent_public_registrations': recent_public_registrations(), 'popular_public_projects': popular_public_projects, 'popular_public_registrations': popular_public_registrations, 'hits': hits, }
def find(self, query=None): combined_query = Q(self._base_class._primary_name, 'in', self._to_primary_keys()) if query is not None: combined_query = combined_query & query return self._base_class.find(combined_query)