def do_validate(self, value, obj):
# Check if required
if value is None:
if getattr(self, '_required', None):
raise exceptions.ValidationError(
'Value <{0}> is required.'.format(self._field_name))
return True
# Check if unique
if value is not None and self._unique:
unique_query = Q(self._field_name, 'eq', value)
# If object has primary key, don't crash if unique value is
# already associated with its key
if obj._is_loaded:
unique_query = unique_query & Q(obj._primary_name, 'ne',
obj._primary_key)
if obj.find(unique_query).limit(1).count():
raise exceptions.ValidationValueError('Value must be unique')
# Field-level validation
cls = self.__class__
if hasattr(cls, 'validate') and self.validate is not False:
cls.validate(value)
# Schema-level validation
if self._validate and hasattr(self, 'validate'):
if hasattr(self.validate, '__iter__'):
for validator in self.validate:
validator(value)
elif hasattr(self.validate, '__call__'):
self.validate(value)
# Success
return True
def test_or(self):
"""Find the union of two or more queries."""
result = self.Foo.find(Q('a', 'eq', 0) | Q('a', 'eq', 1))
self.assertEqual(
len(result),
6,
)
def test_and(self):
"""Find the intersection of two or more queries."""
result = self.Foo.find(Q('a', 'eq', 0) & Q('b', 'eq', 1))
self.assertEqual(
len(result),
1,
)
self.assertEqual(result[0].a, 0)
self.assertEqual(result[0].b, 1)
def test_and_or(self):
"""Join multiple OR queries with an AND.
"""
result = self.Foo.find((Q('a', 'eq', 0) | Q('a', 'eq', 1))
& (Q('b', 'eq', 1) | Q('b', 'eq', 2)))
self.assertEqual(
len(result),
4,
)
def migrate_nodes(index):
logger.info("Migrating nodes to index: {}".format(index))
n_iter = 0
nodes = Node.find(
Q('is_public', 'eq', True) & Q('is_deleted', 'eq', False))
for node in nodes:
search.update_node(node, index=index)
n_iter += 1
logger.info('Nodes migrated: {}'.format(n_iter))
def migrate_nodes(index):
logger.info('Migrating nodes to index: {}'.format(index))
query = Q('is_public', 'eq', True) & Q('is_deleted', 'eq', False)
total = Node.find(query).count()
increment = 200
total_pages = (total // increment) + 1
pages = paginated(Node, query=query, increment=increment, each=False)
for page_number, page in enumerate(pages):
logger.info('Updating page {} / {}'.format(page_number + 1,
total_pages))
Node.bulk_update_search(page, index=index)
Node._clear_caches()
logger.info('Nodes migrated: {}'.format(total))
def test_not(self):
"""Find the inverse of a query."""
result = self.Foo.find(~Q('a', 'eq', 0))
self.assertEqual(
len(result),
6,
)
def find(self, query=None):
""" Find backrefs matching a given query. """
combined_query = Q(self._base_class._primary_name, 'in',
self._to_primary_keys())
if query is not None:
combined_query = combined_query & query
return self._base_class.find(combined_query)
def migrate(delete, index=None, app=None):
index = index or settings.ELASTIC_INDEX
app = app or init_app('website.settings', set_backends=True, routes=True)
script_utils.add_file_logger(logger, __file__)
# NOTE: We do NOT use the app.text_request_context() as a
# context manager because we don't want the teardown_request
# functions to be triggered
ctx = app.test_request_context()
ctx.push()
new_index = set_up_index(index)
start_time = timezone.now()
if settings.ENABLE_INSTITUTIONS:
migrate_institutions(new_index)
migrate_nodes(new_index)
migrate_users(new_index)
set_up_alias(index, new_index)
# migrate nodes modified since start
migrate_nodes(new_index, query=Q('date_modified', 'gte', start_time))
if delete:
delete_old(new_index)
ctx.pop()
def create_fake_project(creator, n_users, privacy, n_components, name, n_tags,
presentation_name, is_registration, is_preprint,
preprint_provider):
auth = Auth(user=creator)
project_title = name if name else fake.science_sentence()
if is_preprint:
provider = None
if preprint_provider:
try:
provider = models.PreprintProvider.find_one(
Q('_id', 'eq', provider))
except NoResultsFound:
pass
if not provider:
provider = PreprintProviderFactory(name=fake.science_word())
privacy = 'public'
mock_change_identifier = mock.patch(
'website.identifiers.client.EzidClient.change_status_identifier')
mock_change_identifier.start()
project = PreprintFactory(title=project_title,
description=fake.science_paragraph(),
creator=creator,
provider=provider)
node = project.node
elif is_registration:
project = RegistrationFactory(title=project_title,
description=fake.science_paragraph(),
creator=creator)
node = project
else:
project = ProjectFactory(title=project_title,
description=fake.science_paragraph(),
creator=creator)
node = project
node.set_privacy(privacy)
for _ in range(n_users):
contrib = create_fake_user()
node.add_contributor(contrib, auth=auth)
if isinstance(n_components, int):
for _ in range(n_components):
NodeFactory(parent=node,
title=fake.science_sentence(),
description=fake.science_paragraph(),
creator=creator)
elif isinstance(n_components, list):
render_generations_from_node_structure_list(node, creator,
n_components)
for _ in range(n_tags):
node.add_tag(fake.science_word(), auth=auth)
if presentation_name is not None:
node.add_tag(presentation_name, auth=auth)
node.add_tag('poster', auth=auth)
node.save()
project.save()
logger.info('Created project: {0}'.format(node.title))
return project
def main():
args = parse_args()
creator = models.User.find(Q('username', 'eq', args.user))[0]
for i in range(args.n_projects):
name = args.name + str(i) if args.name else ''
create_fake_project(creator, args.n_users, args.privacy, args.n_components, name, args.n_tags,
args.presentation_name, args.is_registration)
print('Created {n} fake projects.'.format(n=args.n_projects))
sys.exit(0)
def test_update_one(self):
""" Given a primary key, update the referenced object according to the
update clause
"""
self.Foo.update_one(which=Q('_id', 'eq', 2), data={'modified': True})
self.assertEqual(
[x.modified for x in self.foos],
[False, False, True, False, False],
)
def test_update(self):
""" Given a query, and an update clause, update all (and only) object
returned by query.
"""
self.Foo.update(query=Q('_id', 'eq', 2), data={'modified': True})
self.assertEqual(
[x.modified for x in self.foos],
[False, False, True, False, False],
)
def migrate_nodes(index, query=None):
logger.info('Migrating nodes to index: {}'.format(index))
node_query = Q('is_public', 'eq', True) & Q('is_deleted', 'eq', False)
if query:
node_query = query & node_query
total = Node.find(node_query).count()
increment = 200
total_pages = (total // increment) + 1
pages = paginated(Node,
query=node_query,
increment=increment,
each=False,
include=['contributor__user__guids'])
for page_number, page in enumerate(pages):
logger.info('Updating page {} / {}'.format(page_number + 1,
total_pages))
Node.bulk_update_search(page, index=index)
logger.info('Nodes migrated: {}'.format(total))
def create_fake_project(creator, n_users, privacy, n_components, name, n_tags,
presentation_name, is_registration, is_preprint,
preprint_providers):
auth = Auth(user=creator)
project_title = name if name else fake.science_sentence()
if is_preprint:
providers_to_add = []
if preprint_providers:
providers = preprint_providers.split(',')
for provider in providers:
try:
preprint_provider = models.PreprintProvider.find_one(
Q('_id', 'eq', provider))
except NoResultsFound:
preprint_provider = PreprintProviderFactory(name=provider)
providers_to_add.append(preprint_provider)
privacy = 'public'
project = PreprintFactory(title=project_title,
description=fake.science_paragraph(),
creator=creator,
providers=providers_to_add)
elif is_registration:
project = RegistrationFactory(title=project_title,
description=fake.science_paragraph(),
creator=creator)
else:
project = ProjectFactory(title=project_title,
description=fake.science_paragraph(),
creator=creator)
project.set_privacy(privacy)
for _ in range(n_users):
contrib = create_fake_user()
project.add_contributor(contrib, auth=auth)
if isinstance(n_components, int):
for _ in range(n_components):
NodeFactory(project=project,
title=fake.science_sentence(),
description=fake.science_paragraph(),
creator=creator)
elif isinstance(n_components, list):
render_generations_from_node_structure_list(project, creator,
n_components)
for _ in range(n_tags):
project.add_tag(fake.science_word(), auth=auth)
if presentation_name is not None:
project.add_tag(presentation_name, auth=auth)
project.add_tag('poster', auth=auth)
project.save()
logger.info('Created project: {0}'.format(project.title))
return project
def test_not(self):
""" Finds the inverse of a query."""
# This is failing in Mongo, but works in Pickle. Mongo is getting:
# {'$not': {'a': 0}}
# but it should be getting
# {'a': {'$not': 0}}
#
# See: http://docs.mongodb.org/manual/reference/operator/not/
result = self.Foo.find(~Q('a', 'eq', 0))
self.assertEqual(
len(result),
6,
)
def activity():
popular_public_projects = []
popular_public_registrations = []
hits = {}
if settings.PIWIK_HOST:
client = PiwikClient(
url=settings.PIWIK_HOST,
auth_token=settings.PIWIK_ADMIN_TOKEN,
site_id=settings.PIWIK_SITE_ID,
period='week',
date='today',
)
popular_project_ids = [
x for x in client.custom_variables if x.label == 'Project ID'
][0].values
for nid in popular_project_ids:
node = Node.load(nid.value)
if node is None:
continue
if node.is_public and not node.is_registration and not node.is_deleted:
if len(popular_public_projects) < 10:
popular_public_projects.append(node)
elif node.is_public and node.is_registration and not node.is_deleted:
if len(popular_public_registrations) < 10:
popular_public_registrations.append(node)
if len(popular_public_projects) >= 10 and len(popular_public_registrations) >= 10:
break
hits = {
x.value: {
'hits': x.actions,
'visits': x.visits
} for x in popular_project_ids
}
# Projects
recent_query = (
Q('category', 'eq', 'project') &
Q('is_public', 'eq', True) &
Q('is_deleted', 'eq', False)
)
# Temporary bug fix: Skip projects with empty contributor lists
# Todo: Fix underlying bug and remove this selector
recent_query = recent_query & Q('contributors', 'ne', [])
recent_public_projects = Node.find(
recent_query &
Q('is_registration', 'eq', False)
).sort(
'-date_created'
).limit(10)
# Registrations
recent_public_registrations = Node.find(
recent_query &
Q('is_registration', 'eq', True)
).sort(
'-registered_date'
).limit(10)
return {
'recent_public_projects': recent_public_projects,
'recent_public_registrations': recent_public_registrations,
'popular_public_projects': popular_public_projects,
'popular_public_registrations': popular_public_registrations,
'hits': hits,
}
def migrate_institutions(index):
for inst in Institution.find(Q('is_deleted', 'ne', True)):
update_institution(inst, index)
def activity():
popular_public_projects = []
popular_public_registrations = []
hits = {}
max_popular_projects = 20
if settings.KEEN['public']['read_key']:
client = KeenClient(
project_id=settings.KEEN['public']['project_id'],
read_key=settings.KEEN['public']['read_key'],
)
node_pageviews = client.count(
event_collection='pageviews',
timeframe='this_7_days',
group_by='node.id',
filters=[
{
'property_name': 'node.id',
'operator': 'exists',
'property_value': True
}
]
)
node_visits = client.count_unique(
event_collection='pageviews',
target_property='anon.id',
timeframe='this_7_days',
group_by='node.id',
filters=[
{
'property_name': 'node.id',
'operator': 'exists',
'property_value': True
}
]
)
node_data = [{'node': x['node.id'], 'views': x['result']} for x in node_pageviews[0:max_popular_projects]]
for node_visit in node_visits[0:max_popular_projects]:
for node_result in node_data:
if node_visit['node.id'] == node_result['node']:
node_result.update({'visits': node_visit['result']})
node_data.sort(key=lambda datum: datum['views'], reverse=True)
for nid in node_data:
node = Node.load(nid['node'])
if node is None:
continue
if node.is_public and not node.is_registration and not node.is_deleted:
if len(popular_public_projects) < 10:
popular_public_projects.append(node)
elif node.is_public and node.is_registration and not node.is_deleted and not node.is_retracted:
if len(popular_public_registrations) < 10:
popular_public_registrations.append(node)
if len(popular_public_projects) >= 10 and len(popular_public_registrations) >= 10:
break
hits = {
datum['node']: {
'hits': datum['views'],
'visits': datum['visits']
} for datum in node_data
}
# Projects
new_and_noteworthy_pointers = Node.find_one(Q('_id', 'eq', settings.NEW_AND_NOTEWORTHY_LINKS_NODE)).nodes_pointer
new_and_noteworthy_projects = [pointer.node for pointer in new_and_noteworthy_pointers]
return {
'new_and_noteworthy_projects': new_and_noteworthy_projects,
'recent_public_registrations': recent_public_registrations(),
'popular_public_projects': popular_public_projects,
'popular_public_registrations': popular_public_registrations,
'hits': hits,
}
# blog2.tags.append(tag1)
blog2.save()
blog3.tag = tag1
blog3.save()
blog4 = Blog(tags=[tag1])
blog4.save()
import pdb
pdb.set_trace()
res = Tag.find(
Q('count', 'startswith', 'count_') & Q('misc', 'endswith', 'bar'))
# Tag.find(Q('foo', 'bar', 'baz'))
# todo: accept list of strings
res = Tag.find_all().sort('misc2', '-misc')
print 'here', [(r.misc, r.misc2) for r in res]
res = Tag.find(Q('count', 'eq', 'count_1'))
print 'here', res.count(), list(res)
res = Tag.find(Q('misc', 'startswith', 'foo'))
print 'here', res.count(), list(res)
res = Tag.find(Q('misc', 'endswith', 'bar'))
print 'here', res.count(), list(res)
def migrate_nodes():
for node in Node.find(
Q('is_public', 'eq', True)
& Q('is_deleted', 'eq', False)):
node.update_search()
def migrate_users():
for user in User.find(
Q('is_registered', 'eq', True)
& Q('date_confirmed', 'ne', None)):
user.update_search()
def test_remove(self):
""" Given a query, remove all (and only) object returned by query. """
self.Foo.remove(Q('_id', 'eq', 2))
self.assertEqual(self.Foo.find().count(), 4)
blog2.save()
blog3.tag = tag1
blog3.save()
blog4 = Blog(title='tbdtbdtbd', tags=[tag1, tag2, tag3, tag4, tag5])
blog4.save()
logging.debug("bar")
import pdb
pdb.set_trace()
Blog.remove(Q('title', 'startswith', 'tbd'))
# res = Tag.find(Q('count', 'startswith', 'count_') & Q('misc', 'endswith', 'bar'))
# print 'before rm', res.count()
#
# Tag.remove(Q('count', 'startswith', 'count_') & Q('misc', 'endswith', 'bar'))
#
# res = Tag.find(Q('count', 'startswith', 'count_') & Q('misc', 'endswith', 'bar'))
# print 'after rm', res.count()
Tag.update(Q('count', 'startswith', 'count_'), {'count': 'shutup'})
print tag1.count
import pdb
pdb.set_trace()
def test_remove_one(self):
""" Given a primary key, remove the referenced object. """
self.Foo.remove_one(Q('_id', 'eq', 2))
self.assertEqual(self.Foo.find().count(), 4)
def test_remove_one_returns_none(self):
""" Given a primary key, remove the referenced object. """
with self.assertRaises(exceptions.ModularOdmException):
self.Foo.remove_one(Q('_id', 'eq', 100))
def activity():
popular_public_projects = []
popular_public_registrations = []
hits = {}
# get the date for exactly one week ago
target_date = datetime.date.today() - datetime.timedelta(weeks=1)
if settings.PIWIK_HOST:
client = PiwikClient(
url=settings.PIWIK_HOST,
auth_token=settings.PIWIK_ADMIN_TOKEN,
site_id=settings.PIWIK_SITE_ID,
period='week',
date=target_date.strftime('%Y-%m-%d'),
)
popular_project_ids = [
x for x in client.custom_variables if x.label == 'Project ID'
][0].values
for nid in popular_project_ids:
node = Node.load(nid.value)
if node is None:
continue
if node.is_public and not node.is_registration and not node.is_deleted:
if len(popular_public_projects) < 10:
popular_public_projects.append(node)
elif node.is_public and node.is_registration and not node.is_deleted:
if len(popular_public_registrations) < 10:
popular_public_registrations.append(node)
if len(popular_public_projects) >= 10 and len(popular_public_registrations) >= 10:
break
hits = {
x.value: {
'hits': x.actions,
'visits': x.visits
} for x in popular_project_ids
}
# Projects
recent_query = (
Q('category', 'eq', 'project') &
Q('is_public', 'eq', True) &
Q('is_deleted', 'eq', False)
)
recent_public_projects = Node.find(
recent_query &
Q('is_registration', 'eq', False)
).sort(
'-date_created'
).limit(10)
return {
'recent_public_projects': recent_public_projects,
'recent_public_registrations': recent_public_registrations(),
'popular_public_projects': popular_public_projects,
'popular_public_registrations': popular_public_registrations,
'hits': hits,
}
def find(self, query=None):
combined_query = Q(self._base_class._primary_name, 'in',
self._to_primary_keys())
if query is not None:
combined_query = combined_query & query
return self._base_class.find(combined_query)
