def create(self, validated_data):
inst = self.context["view"].get_object()["self"]
user = self.context["request"].user
node_dicts = validated_data["data"]
changes_flag = False
for node_dict in node_dicts:
node = Node.load(node_dict["_id"])
if not node:
raise exceptions.NotFound(detail='Node with id "{}" was not found'.format(node_dict["_id"]))
if not node.has_permission(user, osf_permissions.ADMIN):
raise exceptions.PermissionDenied(
detail="Admin permission on node {} required".format(node_dict["_id"])
)
if inst not in node.affiliated_institutions:
node.add_affiliated_institution(inst, user, save=True)
changes_flag = True
if not changes_flag:
raise RelationshipPostMakesNoChanges
return {
"data": list(
Node.find_by_institutions(inst, Q("is_registration", "eq", False) & Q("is_deleted", "ne", True))
),
"self": inst,
}
def find_dashboard(user):
dashboard_folder = Node.find_for_user(user, subquery=Q("is_dashboard", "eq", True))
if dashboard_folder.count() == 0:
new_dashboard(user)
dashboard_folder = Node.find_for_user(user, Q("is_dashboard", "eq", True))
return dashboard_folder[0]
def _kwargs_to_nodes(kwargs):
"""Retrieve project and component objects from keyword arguments.
:param dict kwargs: Dictionary of keyword arguments
:return: Tuple of project and component
"""
project = kwargs.get('project') or Node.load(kwargs.get('pid', kwargs.get('nid')))
if not project:
raise HTTPError(http.NOT_FOUND)
if project.category != 'project':
raise HTTPError(http.BAD_REQUEST)
if project.is_deleted:
raise HTTPError(http.GONE)
if kwargs.get('nid') or kwargs.get('node'):
node = kwargs.get('node') or Node.load(kwargs.get('nid'))
if not node:
raise HTTPError(http.NOT_FOUND)
if node.is_deleted:
raise HTTPError(http.GONE)
else:
node = None
return project, node
def search_projects_by_title(**kwargs):
""" Search for nodes by title. Can pass in arguments from the URL to modify the search
:arg term: The substring of the title.
:arg category: Category of the node.
:arg isDeleted: yes, no, or either. Either will not add a qualifier for that argument in the search.
:arg isFolder: yes, no, or either. Either will not add a qualifier for that argument in the search.
:arg isRegistration: yes, no, or either. Either will not add a qualifier for that argument in the search.
:arg includePublic: yes or no. Whether the projects listed should include public projects.
:arg includeContributed: yes or no. Whether the search should include projects the current user has
contributed to.
:arg ignoreNode: a list of nodes that should not be included in the search.
:return: a list of dictionaries of projects
"""
# TODO(fabianvf): At some point, it would be nice to do this with elastic search
user = kwargs['auth'].user
term = request.args.get('term', '')
max_results = int(request.args.get('maxResults', '10'))
category = request.args.get('category', 'project').lower()
is_deleted = request.args.get('isDeleted', 'no').lower()
is_folder = request.args.get('isFolder', 'no').lower()
is_registration = request.args.get('isRegistration', 'no').lower()
include_public = request.args.get('includePublic', 'yes').lower()
include_contributed = request.args.get('includeContributed', 'yes').lower()
ignore_nodes = request.args.getlist('ignoreNode', [])
matching_title = (
Q('title', 'icontains', term) & # search term (case insensitive)
Q('category', 'eq', category) # is a project
)
matching_title = conditionally_add_query_item(matching_title, 'is_deleted', is_deleted)
matching_title = conditionally_add_query_item(matching_title, 'is_folder', is_folder)
matching_title = conditionally_add_query_item(matching_title, 'is_registration', is_registration)
if len(ignore_nodes) > 0:
for node_id in ignore_nodes:
matching_title = matching_title & Q('_id', 'ne', node_id)
my_projects = []
my_project_count = 0
public_projects = []
if include_contributed == "yes":
my_projects = Node.find(
matching_title &
Q('contributors', 'eq', user._id) # user is a contributor
).limit(max_results)
my_project_count = my_project_count
if my_project_count < max_results and include_public == "yes":
public_projects = Node.find(
matching_title &
Q('is_public', 'eq', True) # is public
).limit(max_results - my_project_count)
results = list(my_projects) + list(public_projects)
ret = process_project_search_results(results, **kwargs)
return ret
def create(self, validated_data):
node = Node(**validated_data)
try:
node.save()
except ValidationValueError as e:
raise InvalidModelValueError(detail=e.message)
return node
def set_tag_many_to_many_on_nodes(page_size=10000):
print 'Starting {}...'.format(sys._getframe().f_code.co_name)
node_count = 0
m2m_count = 0
start = datetime.now()
total = MODMNode.find(build_query(m2m_tag_fields, MODMNode)).count()
print '{} Nodes'.format(total)
while node_count < total:
with transaction.atomic():
for modm_node in MODMNode.find(build_query(
m2m_tag_fields, MODMNode)).sort('-date_modified')[
node_count:page_size + node_count]:
django_node = Node.objects.get(
pk=modm_to_django[modm_node._id])
for m2m_tag_field in m2m_tag_fields:
try:
attr = getattr(django_node, m2m_tag_field)
except AttributeError as ex:
# node field doesn't exist on node
pass
else:
# node field exists, do the stuff
django_pks = []
for modm_m2m_value in getattr(modm_node, m2m_tag_field,
[]):
suffix = 'system' if m2m_tag_field == 'system_tags' else 'not_system'
if isinstance(modm_m2m_value, MODMTag):
django_pks.append(modm_to_django[
'{}:{}'.format(modm_m2m_value._id,
suffix)])
elif isinstance(modm_m2m_value, basestring):
django_pks.append(modm_to_django[
'{}:{}'.format(modm_m2m_value, suffix)])
elif modm_m2m_value is None:
print 'Tag of None found on Node {}'.format(
modm_node._id)
else:
# wth
print '\a' # bells!
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
import bpdb
bpdb.set_trace()
if len(django_pks) > 0:
attr.add(*django_pks)
m2m_count += len(django_pks)
node_count += 1
if node_count % page_size == 0 or node_count == total:
print 'Through {} nodes and {} m2m'.format(node_count,
m2m_count)
print 'Done with {} in {} seconds...'.format(
sys._getframe().f_code.co_name,
(datetime.now() - start).total_seconds())
def create(self, validated_data):
node = Node(**validated_data)
node.is_folder = True
node.category = ''
try:
node.save()
except ValidationValueError as e:
raise InvalidModelValueError(detail=e.message)
return node
def get_dashboard_nodes(auth):
"""Get summary information about the current user's dashboard nodes.
:param-query no_components: Exclude components from response.
NOTE: By default, components will only be shown if the current user
is contributor on a comonent but not its parent project. This query
parameter forces ALL components to be excluded from the request.
:param-query permissions: Filter upon projects for which the current user
has the specified permissions. Examples: 'write', 'admin'
"""
user = auth.user
nodes = Node.find_for_user(
user,
subquery=(
Q("category", "eq", "project")
& Q("is_deleted", "eq", False)
& Q("is_registration", "eq", False)
& Q("is_folder", "eq", False)
),
)
if request.args.get("no_components") not in [True, "true", "True", "1", 1]:
comps = Node.find_for_user( # NOTE - this used to be a find on nodes above. Does this mess it up?
user,
(
# components only
Q("category", "ne", "project")
&
# exclude deleted nodes
Q("is_deleted", "eq", False)
&
# exclude registrations
Q("is_registration", "eq", False)
),
)
else:
comps = []
nodes = list(nodes) + list(comps)
if request.args.get("permissions"):
perm = request.args["permissions"].strip().lower()
if perm not in permissions.PERMISSIONS:
raise HTTPError(
http.BAD_REQUEST,
dict(
message_short="Invalid query parameter",
message_long="{0} is not in {1}".format(perm, permissions.PERMISSIONS),
),
)
response_nodes = [node for node in nodes if node.has_permission(user, permission=perm)]
else:
response_nodes = nodes
return _render_nodes(response_nodes, auth)
def create(self, validated_data):
node = Node(**validated_data)
node.is_collection = True
node.category = ''
try:
node.save()
except ValidationValueError as e:
raise InvalidModelValueError(detail=e.message)
except NodeStateError:
raise ser.ValidationError('Each user cannot have more than one Bookmark collection.')
return node
def migrate_nodes(index):
logger.info('Migrating nodes to index: {}'.format(index))
query = Q('is_public', 'eq', True) & Q('is_deleted', 'eq', False)
total = Node.find(query).count()
increment = 200
total_pages = (total // increment) + 1
pages = paginated(Node, query=query, increment=increment, each=False)
for page_number, page in enumerate(pages):
logger.info('Updating page {} / {}'.format(page_number + 1, total_pages))
Node.bulk_update_search(page, index=index)
Node._clear_caches()
logger.info('Nodes migrated: {}'.format(total))
def set_node_many_to_many_on_nodes(page_size=5000):
print 'Starting {}...'.format(sys._getframe().f_code.co_name)
node_count = 0
m2m_count = 0
start = datetime.now()
total = MODMNode.find(
build_query(m2m_node_fields, MODMNode),
allow_institution=True).count()
print '{} Nodes'.format(total)
while node_count < total:
with transaction.atomic():
for modm_node in MODMNode.find(
build_query(m2m_node_fields, MODMNode),
allow_institution=True).sort('-date_modified')[
node_count:page_size + node_count]:
django_node = Node.objects.get(
pk=modm_to_django[modm_node._id])
for m2m_node_field in m2m_node_fields:
attr = getattr(django_node, m2m_node_field)
django_pks = []
for modm_m2m_value in getattr(modm_node, m2m_node_field,
[]):
if isinstance(modm_m2m_value, MODMNode):
django_pks.append(modm_to_django[
modm_m2m_value._id])
elif isinstance(modm_m2m_value, basestring):
django_pks.append(modm_to_django[modm_m2m_value])
elif isinstance(modm_m2m_value, Pointer):
django_pks.append(modm_to_django[
modm_m2m_value.node._id])
else:
# wth
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
import bpdb
bpdb.set_trace()
if len(django_pks) > 0:
attr.add(*django_pks)
m2m_count += len(django_pks)
node_count += 1
if node_count % page_size == 0 or node_count == total:
print 'Through {} nodes and {} m2m'.format(node_count,
m2m_count)
print 'Done with {} in {} seconds...'.format(
sys._getframe().f_code.co_name,
(datetime.now() - start).total_seconds())
def get_queryset(self):
# For bulk requests, queryset is formed from request body.
if is_bulk_request(self.request):
query = Q("_id", "in", [node["id"] for node in self.request.data])
auth = get_user_auth(self.request)
nodes = Node.find(query)
for node in nodes:
if not node.can_edit(auth):
raise PermissionDenied
return nodes
else:
query = self.get_query_from_request()
return Node.find(query)
def save_bare_nodes(page_size=20000):
print 'Starting {}...'.format(sys._getframe().f_code.co_name)
count = 0
start = datetime.now()
total = MODMNode.find(allow_institution=True).count()
while count < total:
with transaction.atomic():
nids = []
for modm_node in MODMNode.find(
allow_institution=True).sort('-date_modified')[
count:count + page_size]:
guid = Guid.objects.get(guid=modm_node._id)
node_fields = dict(_guid_id=guid.pk, **modm_node.to_storage())
# remove fields not yet implemented
cleaned_node_fields = {key: node_fields[key]
for key in node_fields
if key not in node_key_blacklist}
# make datetimes not naive
for k, v in cleaned_node_fields.iteritems():
if isinstance(v, datetime):
cleaned_node_fields[k] = pytz.utc.localize(v)
# remove null fields, postgres hate null fields
cleaned_node_fields = {k: v
for k, v in
cleaned_node_fields.iteritems()
if v is not None}
nids.append(Node(**cleaned_node_fields))
count += 1
if count % page_size == 0 or count == total:
then = datetime.now()
print 'Saving nodes {} through {}...'.format(
count - page_size, count)
woot = Node.objects.bulk_create(nids)
for wit in woot:
modm_to_django[wit._guid.guid] = wit.pk
now = datetime.now()
print 'Done with {} nodes in {} seconds...'.format(
len(woot), (now - then).total_seconds())
nids = []
trash = gc.collect()
print 'Took out {} trashes'.format(trash)
print 'Modm Nodes: {}'.format(total)
print 'django Nodes: {}'.format(Node.objects.all().count())
print 'Done with {} in {} seconds...'.format(
sys._getframe().f_code.co_name,
(datetime.now() - start).total_seconds())
def setUp(self, *args, **kwargs):
OsfTestCase.setUp(self, *args, **kwargs)
if not self.kind:
return
self.sanction = self.Factory()
self.reg = Node.find_one(Q(self.Model.SHORT_NAME, 'eq', self.sanction))
self.user = self.reg.creator
def get_projects_registered():
projects_registered = Node.find(
Q('parent_node', 'eq', None) &
Q('is_registration', 'eq', True) &
CONTENT_NODE_QUERY
)
return projects_registered
def get_projects_forked():
projects_forked = Node.find(
Q('parent_node', 'eq', None) &
Q('is_fork', 'eq', True) &
CONTENT_NODE_QUERY
)
return projects_forked
def get_projects_public():
projects_public = Node.find(
Q('parent_node', 'eq', None) &
Q('is_public', 'eq', True) &
CONTENT_NODE_QUERY
)
return projects_public
def get_configured_projects(user):
"""Filter all user subscriptions for ones that are on parent projects
and return the project ids.
:param user: modular odm User object
:return: list of project ids for projects with no parent
"""
configured_project_ids = set()
user_subscriptions = get_all_user_subscriptions(user)
for subscription in user_subscriptions:
if subscription is None:
continue
# If the user has opted out of emails skip
node = subscription.owner
if not isinstance(node, Node) or (user in subscription.none and not node.parent_id):
continue
while node.parent_id and not node.is_deleted:
node = Node.load(node.parent_id)
if not node.is_deleted:
configured_project_ids.add(node._id)
return list(configured_project_ids)
def project_new_post(auth, **kwargs):
user = auth.user
title = strip_html(request.json.get('title'))
template = request.json.get('template')
description = strip_html(request.json.get('description'))
title = title.strip()
if not title or len(title) > 200:
raise HTTPError(http.BAD_REQUEST)
if template:
original_node = Node.load(template)
changes = {
'title': title
}
if description:
changes['description'] = description
project = original_node.use_as_template(
auth=auth,
changes={
template: changes
})
else:
project = new_node('project', title, user, description)
return {
'projectUrl': project.url
}, http.CREATED
def conference_submissions(**kwargs):
"""Return data for all OSF4M submissions.
The total number of submissions for each meeting is calculated and cached
in the Conference.num_submissions field.
"""
submissions = []
# TODO: Revisit this loop, there has to be a way to optimize it
for conf in Conference.find():
# For efficiency, we filter by tag first, then node
# instead of doing a single Node query
projects = set()
tags = Tag.find(Q('lower', 'eq', conf.endpoint.lower())).get_keys()
nodes = Node.find(
Q('tags', 'in', tags) &
Q('is_public', 'eq', True) &
Q('is_deleted', 'ne', True)
)
projects.update(list(nodes))
for idx, node in enumerate(projects):
submissions.append(_render_conference_node(node, idx, conf))
num_submissions = len(projects)
# Cache the number of submissions
conf.num_submissions = num_submissions
conf.save()
if num_submissions < settings.CONFERENCE_MIN_COUNT:
continue
submissions.sort(key=lambda submission: submission['dateCreated'], reverse=True)
return {'submissions': submissions}
def test_POST_register_make_public_does_not_make_children_public(self, mock_enqueue):
component = NodeFactory(
creator=self.user,
parent=self.project,
title='Component'
)
subproject = ProjectFactory(
creator=self.user,
parent=self.project,
title='Subproject'
)
subproject_component = NodeFactory(
creator=self.user,
parent=subproject,
title='Subcomponent'
)
res = self.app.post(
self.project.api_url_for('register_draft_registration', draft_id=self.draft._id),
self.valid_make_public_payload,
content_type='application/json',
auth=self.user.auth
)
self.project.reload()
# Last node directly registered from self.project
registration = Node.load(self.project.node__registrations[-1])
assert_false(registration.is_public)
for node in registration.get_descendants_recursive():
assert_true(node.is_registration)
assert_false(node.is_public)
def test_bulk_creates_children_and_sanitizes_html_logged_in_owner(self):
title = '<em>Cool</em> <strong>Project</strong>'
description = 'An <script>alert("even cooler")</script> child'
res = self.app.post_json_api(self.url, {
'data': [{
'type': 'nodes',
'attributes': {
'title': title,
'description': description,
'category': 'project',
'public': True
}
}]
}, auth=self.user.auth, bulk=True)
child_id = res.json['data'][0]['id']
assert_equal(res.status_code, 201)
url = '/{}nodes/{}/'.format(API_BASE, child_id)
res = self.app.get(url, auth=self.user.auth)
assert_equal(res.json['data']['attributes']['title'], strip_html(title))
assert_equal(res.json['data']['attributes']['description'], strip_html(description))
assert_equal(res.json['data']['attributes']['category'], 'project')
self.project.reload()
child_id = res.json['data']['id']
assert_equal(child_id, self.project.nodes[0]._id)
assert_equal(Node.load(child_id).logs[0].action, NodeLog.PROJECT_CREATED)
def get_projects():
# This count includes projects, forks, and registrations
projects = Node.find(
Q('parent_node', 'eq', None) &
CONTENT_NODE_QUERY
)
return projects
def test_POST_register_embargo_does_not_make_project_or_children_public(self, mock_enqueue):
public_project = ProjectFactory(creator=self.user, is_public=True)
component = NodeFactory(creator=self.user, parent=public_project, title="Component", is_public=True)
subproject = ProjectFactory(creator=self.user, parent=public_project, title="Subproject", is_public=True)
subproject_component = NodeFactory(creator=self.user, parent=subproject, title="Subcomponent", is_public=True)
res = self.app.post(
public_project.api_url_for("node_register_template_page_post", template=u"Open-Ended_Registration"),
self.valid_embargo_payload,
content_type="application/json",
auth=self.user.auth,
)
public_project.reload()
assert_equal(res.status_code, 201)
# Last node directly registered from self.project
registration = Node.load(public_project.node__registrations[-1])
assert_true(registration.is_registration)
assert_false(registration.is_public)
assert_true(registration.is_pending_embargo_for_existing_registration)
assert_is_not_none(registration.embargo)
for node in registration.get_descendants_recursive():
assert_true(node.is_registration)
assert_false(node.is_public)
def add_pointer(auth):
"""Add a single pointer to a node using only JSON parameters
"""
to_node_id = request.json.get('toNodeID')
pointer_to_move = request.json.get('pointerID')
if not (to_node_id and pointer_to_move):
raise HTTPError(http.BAD_REQUEST)
pointer = Node.load(pointer_to_move)
to_node = Node.load(to_node_id)
try:
_add_pointers(to_node, [pointer], auth)
except ValueError:
raise HTTPError(http.BAD_REQUEST)
def test_fix_templated(self):
assert_equal(
2,
fix_nodes(get_broken_templated())
)
Node._clear_caches()
broken_nodes = list(get_broken_templated())
assert_equal(0, len(broken_nodes))
assert_is_none(self.bad_template_project.piwik_site_id)
assert_is_none(self.bad_template_component.piwik_site_id)
assert_is_not_none(self.template_project.piwik_site_id)
assert_is_not_none(self.template_component.piwik_site_id)
def project_contributors_post(auth, node, **kwargs):
""" Add contributors to a node. """
user_dicts = request.json.get('users')
node_ids = request.json.get('node_ids')
if user_dicts is None or node_ids is None:
raise HTTPError(http.BAD_REQUEST)
# Prepare input data for `Node::add_contributors`
contribs = deserialize_contributors(node, user_dicts, auth=auth)
node.add_contributors(contributors=contribs, auth=auth)
node.save()
# Disconnect listener to avoid multiple invite emails
unreg_contributor_added.disconnect(finalize_invitation)
for child_id in node_ids:
child = Node.load(child_id)
# Only email unreg users once
child_contribs = deserialize_contributors(
child, user_dicts, auth=auth
)
child.add_contributors(contributors=child_contribs, auth=auth)
child.save()
# Reconnect listener
unreg_contributor_added.connect(finalize_invitation)
return {'status': 'success'}, 201
def main():
nodes = Node.objects.all()
total = len(nodes)
count = 0
page_size = 1000
while count < total:
for node in nodes[count:count+page_size]:
modm_node = MODMNode.load(node._guid.guid)
verify_contributors(node, modm_node)
verify_tags(node, modm_node)
count += 1
if count % (total * .001) == 0:
floatal = float(total)
flount = float(count)
print 'Verified nodes {}%'.format((
(floatal - flount) / floatal - 1.0) * -100.0)
# clear out
modm_node = None
node = None
floatal = None
flount = None
if count % page_size == 0:
garbage = gc.collect()
print '{}:{} Collected {} whole garbages...'.format(count, total,
garbage)
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
def project_generate_private_link_post(auth, node, **kwargs):
""" creata a new private link object and add it to the node and its selected children"""
node_ids = request.json.get('node_ids', [])
name = request.json.get('name', '')
anonymous = request.json.get('anonymous', False)
if node._id not in node_ids:
node_ids.insert(0, node._id)
nodes = [Node.load(node_id) for node_id in node_ids]
has_public_node = any(node.is_public for node in nodes)
new_link = new_private_link(
name=name, user=auth.user, nodes=nodes, anonymous=anonymous
)
if anonymous and has_public_node:
status.push_status_message(
'Anonymized view-only links <b>DO NOT</b> '
'anonymize contributors of public project or component.'
)
return new_link
def load(cls, key):
from website.models import Node
try:
node = Node.find_one(Q('institution_id', 'eq', key), allow_institution=True)
return cls(node)
except NoResultsFound:
return None
def load_parent(parent_id):
parent = Node.load(parent_id)
if parent is None:
return None
parent_info = {}
if parent is not None and parent.is_public:
parent_info['title'] = parent.title
parent_info['url'] = parent.url
parent_info['is_registration'] = parent.is_registration
parent_info['id'] = parent._id
else:
parent_info['title'] = '-- private project --'
parent_info['url'] = ''
parent_info['is_registration'] = None
parent_info['id'] = None
return parent_info
def replace_unclaimed_user_with_registered(user):
"""Listens for the user_registered signal. If unreg_user is stored in the
session, then the current user is trying to claim themselves as a contributor.
Replaces the old, unregistered contributor with the newly registered
account.
"""
unreg_user_info = session.data.get('unreg_user')
if unreg_user_info:
unreg_user = User.load(unreg_user_info['uid'])
pid = unreg_user_info['pid']
node = Node.load(pid)
node.replace_contributor(old=unreg_user, new=user)
node.save()
status.push_status_message(
'Successfully claimed contributor.', kind='success', trust=False)
def get_pointers_to_add_remove(self, pointers, new_pointers):
diff = relationship_diff(
current_items={pointer.node._id: pointer
for pointer in pointers},
new_items={val['node']['_id']: val
for val in new_pointers})
nodes_to_add = []
for node_id in diff['add']:
node = Node.load(node_id)
if not node:
raise exceptions.NotFound(
detail='Node with id "{}" was not found'.format(node_id))
nodes_to_add.append(node)
return nodes_to_add, diff['remove'].values()
def test_POST_register_embargo_is_not_public(self, mock_enqueue):
res = self.app.post(self.project.api_url_for(
'node_register_template_page_post',
template=u'Open-Ended_Registration'),
self.valid_embargo_payload,
content_type='application/json',
auth=self.user.auth)
assert_equal(res.status_code, 201)
registration = Node.find().sort('-registered_date')[0]
assert_true(registration.is_registration)
assert_false(registration.is_public)
assert_true(registration.pending_registration)
assert_is_not_none(registration.embargo)
def get_queryset(self):
node = self.get_node()
req_query = self.get_query_from_request()
query = (
Q('_id', 'in', [e._id for e in node.nodes if e.primary]) &
req_query
)
nodes = Node.find(query)
user = self.request.user
if user.is_anonymous():
auth = Auth(None)
else:
auth = Auth(user)
children = [each for each in nodes if each.can_view(auth)]
return children
def test_POST_register_embargo_is_not_public(self, mock_enqueue):
res = self.app.post(
self.project.api_url_for('register_draft_registration', draft_id=self.draft._id),
self.valid_embargo_payload,
content_type='application/json',
auth=self.user.auth
)
assert_equal(res.status_code, 202)
registration = Node.find().sort('-registered_date')[0]
assert_true(registration.is_registration)
assert_false(registration.is_public)
assert_true(registration.is_pending_embargo_for_existing_registration)
assert_is_not_none(registration.embargo)
def test_integration(self, mock_upload, mock_send_mail):
fullname = 'John Deacon'
username = '******'
title = 'good songs'
conference = ConferenceFactory()
body = 'dragon on my back'
content = 'dragon attack'
recipient = '{0}{1}[email protected]'.format(
'test-' if settings.DEV_MODE else '',
conference.endpoint,
)
self.app.post(
api_url_for('meeting_hook'),
{
'X-Mailgun-Sscore': 0,
'timestamp': '123',
'token': 'secret',
'signature': hmac.new(
key=settings.MAILGUN_API_KEY,
msg='{}{}'.format('123', 'secret'),
digestmod=hashlib.sha256,
).hexdigest(),
'attachment-count': '1',
'X-Mailgun-Sscore': 0,
'from': '{0} <{1}>'.format(fullname, username),
'recipient': recipient,
'subject': title,
'stripped-text': body,
},
upload_files=[
('attachment-1', 'attachment-1', content),
],
)
assert_true(mock_upload.called)
users = User.find(Q('username', 'eq', username))
assert_equal(users.count(), 1)
nodes = Node.find(Q('title', 'eq', title))
assert_equal(nodes.count(), 1)
node = nodes[0]
assert_equal(node.get_wiki_page('home').content, body)
assert_true(mock_send_mail.called)
call_args, call_kwargs = mock_send_mail.call_args
assert_absolute(call_kwargs['conf_view_url'])
assert_absolute(call_kwargs['set_password_url'])
assert_absolute(call_kwargs['profile_url'])
assert_absolute(call_kwargs['file_url'])
assert_absolute(call_kwargs['node_url'])
def node_child_tree(user, node_ids):
""" Format data to test for node privacy settings for use in treebeard.
"""
items = []
for node_id in node_ids:
node = Node.load(node_id)
assert node, '{} is not a valid Node.'.format(node_id)
can_read = node.has_permission(user, 'read')
can_read_children = node.has_permission_on_children(user, 'read')
if not can_read and not can_read_children:
continue
children = []
# List project/node if user has at least 'read' permissions (contributor or admin viewer) or if
# user is contributor on a component of the project/node
can_write = node.has_permission(user, 'admin')
children.extend(
node_child_tree(
user,
[n._id for n in node.nodes if n.primary and not n.is_deleted]))
item = {
'node': {
'id': node_id,
'url': node.url if can_read else '',
'title': node.title if can_read else 'Private Project',
'is_public': node.is_public,
'can_write': can_write
},
'user_id':
user._id,
'children':
children,
'kind':
'folder' if not node.node__parent
or not node.parent_node.has_permission(user, 'read') else 'node',
'nodeType':
node.project_or_component,
'category':
node.category,
'permissions': {
'view': can_read,
}
}
items.append(item)
return items
def create(self, validated_data):
node = Node(**validated_data)
node.is_collection = True
node.category = ''
try:
node.save()
except ValidationValueError as e:
raise InvalidModelValueError(detail=e.message)
except NodeStateError:
raise ser.ValidationError('Each user cannot have more than one Bookmark collection.')
return node
def fix_log_params(targets):
"""
Restores params['registration'] field and points params['node'] and original_node fields to the node instead of registration
"""
logger.info('Migrating registration_cancelled, registration_approved, retraction_cancelled, embargo_approved, embargo_cancelled, and embargo_terminated logs.')
count = 0
for log in targets:
node_id = log.params['node']
node = Node.load(node_id)
if node.is_registration:
log.params['node'] = get_registered_from(node)
log.params['registration'] = node._id
log.original_node = log.params['node']
logger.info('Updating params of log {}. params[node]={}, params[registration]={}, and original_node = {}'.format(log._id, log.params['node'], log.params['registration'], log.original_node))
log.save()
count += 1
logger.info('{} logs migrated'.format(count))
def conference_data(meeting):
try:
conf = Conference.find_one(Q('endpoint', 'iexact', meeting))
except ModularOdmException:
raise HTTPError(httplib.NOT_FOUND)
nodes = Node.find(
Q('tags', 'iexact', meeting) &
Q('is_public', 'eq', True) &
Q('is_deleted', 'eq', False)
)
ret = [
_render_conference_node(each, idx, conf)
for idx, each in enumerate(nodes)
]
return ret
def do_migration(logs):
# ... perform the migration using a list of logs ...
for log in logs:
registration_id = log.params.get('registration')
if registration_id:
registration = Node.load(registration_id)
if registration.date_modified < log.date:
registration.date_modified = log.date
registration.save()
logger.info('{} date updated to {}'.format(
registration, log.date))
else:
logger.info('Date modified is more recent than retraction ' +
log._id)
else:
logger.warning('No parent registration found for retraction log ' +
log._id)
def add_pointers(auth, node, **kwargs):
"""Add pointers to a node.
"""
node_ids = request.json.get('nodeIds')
if not node_ids:
raise HTTPError(http.BAD_REQUEST)
nodes = [Node.load(node_id) for node_id in node_ids]
try:
_add_pointers(node, nodes, auth)
except ValueError:
raise HTTPError(http.BAD_REQUEST)
return {}
def wrapped(payload, *args, **kwargs):
try:
user = User.load(payload['user'])
dest_node = Node.load(payload['destination']['node'])
source = models.OsfStorageFileNode.get(payload['source'], kwargs['node'])
dest_parent = models.OsfStorageFolder.get(payload['destination']['parent'], dest_node)
kwargs.update({
'user': user,
'source': source,
'destination': dest_parent,
'name': payload['destination']['name'],
})
except KeyError:
raise HTTPError(httplib.BAD_REQUEST)
return func(*args, **kwargs)
def project_contributors_post(auth, node, **kwargs):
""" Add contributors to a node. """
user_dicts = request.json.get('users')
node_ids = request.json.get('node_ids')
if node._id in node_ids:
node_ids.remove(node._id)
if user_dicts is None or node_ids is None:
raise HTTPError(http.BAD_REQUEST)
# Prepare input data for `Node::add_contributors`
try:
contribs = deserialize_contributors(node, user_dicts, auth=auth, validate=True)
except ValidationError as e:
return {'status': 400, 'message': e.message}, 400
node.add_contributors(contributors=contribs, auth=auth)
node.save()
# Disconnect listener to avoid multiple invite emails
unreg_contributor_added.disconnect(finalize_invitation)
for child_id in node_ids:
child = Node.load(child_id)
# Only email unreg users once
try:
child_contribs = deserialize_contributors(
child, user_dicts, auth=auth, validate=True
)
except ValidationError as e:
return {'status': 400, 'message': e.message}, 400
child.add_contributors(contributors=child_contribs, auth=auth)
child.save()
# Reconnect listeners
unreg_contributor_added.connect(finalize_invitation)
return {
'status': 'success',
'contributors': profile_utils.serialize_contributors(
node.visible_contributors,
node=node,
)
}, 201
def add_folder(**kwargs):
auth = kwargs['auth']
user = auth.user
title = strip_html(request.json.get('title'))
node_id = request.json.get('node_id')
node = Node.load(node_id)
if node.is_deleted or node.is_registration or not node.is_folder:
raise HTTPError(http.BAD_REQUEST)
folder = new_folder(
title, user
)
folders = [folder]
try:
_add_pointers(node, folders, auth)
except ValueError:
raise HTTPError(http.BAD_REQUEST)
return {}, 201, None
def migrate_registrations_metadata_key(schema):
"""
Finds Veer registrations whose registered_meta has an undefined key and corrects.
"""
registrations = Node.find(Q('is_registration', 'eq', True) & Q('registered_schema', 'eq', schema))
total_reg = registrations.count()
logger.info('Examining {} registrations for improper key'.format(total_reg))
reg_count = 0
for reg in registrations:
reg_count += 1
if reg.registered_meta.get(schema._id, {}).get('recommended-methods', {}).get('value', {}).get('undefined', {}):
reg.registered_meta[schema._id]['recommended-methods']['value']['procedure'] = \
reg.registered_meta[schema._id]['recommended-methods']['value'].pop('undefined')
reg.save()
logger.info('{}/{} Migrated key for {}'.format(reg_count, total_reg, reg._id))
else:
logger.info('{}/{} Key already correct for {}. No change.'.format(reg_count, total_reg, reg._id))
def has_object_permission(self, request, view, obj):
assert isinstance(
obj,
(Node,
Pointer)), 'obj must be a Node or Pointer, got {}'.format(obj)
auth = get_user_auth(request)
parent_node = Node.load(request.parser_context['kwargs']['node_id'])
pointer_node = Pointer.load(
request.parser_context['kwargs']['node_link_id']).node
if request.method in permissions.SAFE_METHODS:
has_parent_auth = parent_node.can_view(auth)
has_pointer_auth = pointer_node.can_view(auth)
public = obj.is_public
has_auth = public or (has_parent_auth and has_pointer_auth)
return has_auth
else:
has_auth = parent_node.can_edit(auth)
return has_auth
def has_object_permission(self, request, view, obj):
node_link = Pointer.load(
request.parser_context['kwargs']['node_link_id'])
node = Node.load(
request.parser_context['kwargs'][view.node_lookup_url_kwarg])
auth = get_user_auth(request)
if request.method == 'DELETE' and node.is_registration:
raise exceptions.MethodNotAllowed(method=request.method)
if node.is_collection or node.is_registration:
raise exceptions.NotFound
if node_link.node.is_registration:
if request.method not in permissions.SAFE_METHODS:
raise exceptions.MethodNotAllowed
if node not in node_link.parent:
raise exceptions.NotFound
if request.method == 'DELETE' and not node.can_edit(auth):
return False
return True
def new_public_project(email):
""" Will check to make sure the project that triggered this presend is still public
before sending the email. It also checks to make sure this is the first (and only)
new public project email to be sent
:param email: QueuedMail object, with 'nid' in its data field
:return: boolean based on whether the email should be sent
"""
# In line import to prevent circular importing
from website.models import Node
node = Node.load(email.data['nid'])
if not node:
return False
public = email.find_sent_of_same_type_and_user()
return node.is_public and not len(public)
def migrate(dry=True):
registrations = Node.find(
Q('is_registration', 'eq', True) & Q('registered_meta', 'ne', None))
regs_migrated, reg_errored = migrate_extras(registrations, dry=dry)
drafts = DraftRegistration.find(Q('registration_metadata', 'ne', {}))
drafts_migrated, drafts_errored = migrate_extras(drafts, dry=dry)
logger.info('Migrated registered_meta for {} registrations'.format(
len(regs_migrated)))
if reg_errored:
logger.error('{} errored: {}'.format(len(reg_errored), reg_errored))
logger.info('Migrated registered_meta for {} draft registrations'.format(
len(drafts_migrated)))
if drafts_errored:
logger.error('{} errored: {}'.format(len(drafts_errored),
drafts_errored))
def test_POST_register_embargo_does_not_make_project_or_children_public(self, mock_enqueue):
self.project.is_public = True
self.project.save()
component = NodeFactory(
creator=self.user,
parent=self.project,
title='Component',
is_public=True
)
subproject = ProjectFactory(
creator=self.user,
parent=self.project,
title='Subproject',
is_public=True
)
subproject_component = NodeFactory(
creator=self.user,
parent=subproject,
title='Subcomponent',
is_public=True
)
res = self.app.post(
self.project.api_url_for('register_draft_registration', draft_id=self.draft._id),
self.valid_embargo_payload,
content_type='application/json',
auth=self.user.auth
)
self.project.reload()
assert_equal(res.status_code, 202)
assert_equal(res.json['urls']['registrations'], self.project.web_url_for('node_registrations'))
# Last node directly registered from self.project
registration = Node.find(
Q('registered_from', 'eq', self.project)
).sort('-registered_date')[0]
assert_true(registration.is_registration)
assert_false(registration.is_public)
assert_true(registration.is_pending_embargo_for_existing_registration)
assert_is_not_none(registration.embargo)
for node in registration.get_descendants_recursive():
assert_true(node.is_registration)
assert_false(node.is_public)
def get_auth(**kwargs):
try:
action = request.args['action']
node_id = request.args['nid']
provider_name = request.args['provider']
except KeyError:
raise HTTPError(httplib.BAD_REQUEST)
cookie = request.args.get('cookie')
view_only = request.args.get('view_only')
if 'auth_user_id' in session.data:
user = User.load(session.data['auth_user_id'])
elif cookie:
user = User.from_cookie(cookie)
else:
user = None
node = Node.load(node_id)
if not node:
raise HTTPError(httplib.NOT_FOUND)
check_access(node, user, action, key=view_only)
provider_settings = node.get_addon(provider_name)
if not provider_settings:
raise HTTPError(httplib.BAD_REQUEST)
try:
credentials = provider_settings.serialize_waterbutler_credentials()
settings = provider_settings.serialize_waterbutler_settings()
except exceptions.AddonError:
log_exception()
raise HTTPError(httplib.BAD_REQUEST)
return {
'auth': make_auth(user),
'credentials': credentials,
'settings': settings,
'callback_url': node.api_url_for(
('create_waterbutler_log' if not node.is_registration else 'registration_callbacks'),
_absolute=True,
),
}
def main():
init_app(set_backends=True, routes=False)
staff_registrations = defaultdict(list)
users = [User.load(each) for each in STAFF_GUIDS]
for registration in Node.find(
Q('is_registration', 'eq', True) & Q('is_public', 'eq', True)):
for user in users:
if registration in user.node__contributed:
staff_registrations[user._id].append(registration)
for uid in staff_registrations:
user = User.load(uid)
user_regs = staff_registrations[uid]
logger.info('{} ({}) on {} Public Registrations:'.format(
user.fullname, user._id, len(user_regs)))
for registration in user_regs:
logger.info('\t{} ({}): {}'.format(registration.title,
registration._id,
registration.absolute_url))
def get_all_registrations_smart_folder(auth, **kwargs):
# TODO: Unit tests
user = auth.user
contributed = Node.find_for_user(
user,
subquery=(
Q('is_deleted', 'eq', False) &
Q('is_registration', 'eq', True) &
Q('is_folder', 'eq', False)
)
)
nodes = contributed.sort('-title')
# Note(hrybacki): is_retracted and is_pending_embargo are property methods
# and cannot be directly queried
nodes = filter(lambda node: not node.is_retracted and not node.is_pending_embargo, nodes)
keys = [node._id for node in nodes]
return [rubeus.to_project_root(node, auth, **kwargs) for node in nodes if node.ids_above.isdisjoint(keys)]
def main():
dry_run = False
if '--dry' in sys.argv:
dry_run = True
if not dry_run:
script_utils.add_file_logger(logger, __file__)
init_app(set_backends=True, routes=False)
node_id = sys.argv[1]
node = Node.load(node_id)
if not node:
logger.error('Node "{}" not found'.format(node_id))
sys.exit(1)
with TokuTransaction():
for each in node.node_and_primary_descendants():
restore_node(each)
if dry_run:
raise Exception('Dry Run -- Aborting Transaction')
logger.info('Finished restoring node {}'.format(node_id))
def get_dashboard(auth, nid=None, **kwargs):
user = auth.user
if nid is None:
node = find_dashboard(user)
dashboard_projects = [rubeus.to_project_root(node, auth, **kwargs)]
return_value = {'data': dashboard_projects}
elif nid == ALL_MY_PROJECTS_ID:
return_value = {'data': get_all_projects_smart_folder(**kwargs)}
elif nid == ALL_MY_REGISTRATIONS_ID:
return_value = {'data': get_all_registrations_smart_folder(**kwargs)}
else:
node = Node.load(nid)
dashboard_projects = rubeus.to_project_hgrid(node, auth, **kwargs)
return_value = {'data': dashboard_projects}
return_value['timezone'] = user.timezone
return_value['locale'] = user.locale
return_value['id'] = user._id
return return_value
def create(self, validated_data):
request = self.context['request']
user = request.user
auth = Auth(user)
node = self.context['view'].get_node()
target_node_id = validated_data['_id']
pointer_node = Node.load(target_node_id)
if not pointer_node or pointer_node.is_collection:
raise InvalidModelValueError(
source={'pointer': '/data/relationships/node_links/data/id'},
detail='Target Node \'{}\' not found.'.format(target_node_id))
try:
pointer = node.add_pointer(pointer_node, auth, save=True)
return pointer
except ValueError:
raise InvalidModelValueError(
source={'pointer': '/data/relationships/node_links/data/id'},
detail='Target Node \'{}\' already pointed to by \'{}\'.'.
format(target_node_id, node._id))
def get_auth(**kwargs):
try:
action = request.args['action']
cookie = request.args['cookie']
node_id = request.args['nid']
provider_name = request.args['provider']
except KeyError:
raise HTTPError(httplib.BAD_REQUEST)
view_only = request.args.get('view_only')
user = get_user_from_cookie(cookie)
node = Node.load(node_id)
if not node:
raise HTTPError(httplib.NOT_FOUND)
check_access(node, user, action, key=view_only)
provider_settings = node.get_addon(provider_name)
if not provider_settings:
raise HTTPError(httplib.BAD_REQUEST)
try:
credentials = provider_settings.serialize_waterbutler_credentials()
settings = provider_settings.serialize_waterbutler_settings()
except exceptions.AddonError:
log_exception()
raise HTTPError(httplib.BAD_REQUEST)
return {
'auth':
make_auth(user),
'credentials':
credentials,
'settings':
settings,
'callback_url':
node.api_url_for(
'create_waterbutler_log',
_absolute=True,
),
}
def project_new_post(auth, **kwargs):
user = auth.user
data = request.get_json()
title = strip_html(data.get('title'))
title = title.strip()
category = data.get('category', 'project')
template = data.get('template')
description = strip_html(data.get('description'))
new_project = {}
if template:
original_node = Node.load(template)
changes = {
'title': title,
'category': category,
'template_node': original_node,
}
if description:
changes['description'] = description
project = original_node.use_as_template(
auth=auth,
changes={
template: changes,
}
)
else:
try:
project = new_node(category, title, user, description)
except ValidationValueError as e:
raise HTTPError(
http.BAD_REQUEST,
data=dict(message_long=e.message)
)
new_project = _view_project(project, auth)
return {
'projectUrl': project.url,
'newNode': new_project['node'] if new_project else None
}, http.CREATED