def test_filter_build(self):
qs = TestFile._filter(Q('test', 'eq', 'test'))
_, is_file, provider = qs.nodes
assert_equal(is_file.__dict__, Q('is_file', 'eq', True).__dict__)
assert_equal(provider.__dict__, Q('provider', 'eq', 'test').__dict__)
def get_file(cls, path, node_settings):
return cls.find_one(
Q('_id', 'eq', path.strip('/')) & Q('kind', 'eq', 'file')
& Q('node_settings', 'eq', node_settings))
def find_child_by_name(self, name, kind='file'):
return self.__class__.find_one(
Q('name', 'eq', name) & Q('kind', 'eq', kind)
& Q('parent', 'eq', self))
def get_queryset(self):
return User.find(Q('_id', 'eq', self.context['request'].user._id))
def get_prereg_schema():
from website.models import MetaSchema # noqa
return MetaSchema.find_one(
Q('name', 'eq', 'Prereg Challenge') & Q('schema_version', 'eq', 2))
def get_default_odm_query(self):
return Q('_id', 'ne', None) & Q('is_deleted', 'ne', True)
def get_institutions(self):
institutions = Institution.find(Q('_id', 'ne', None))
return institutions
def glacier_targets():
return models.FileVersion.find(
Q('status', 'ne', 'cached') & Q('location.object', 'exists', True)
& Q('metadata.archive', 'eq', None))
def parity_targets():
# TODO: Add metadata.parity information from wb so we do not need to check remote services
return models.FileVersion.find(
Q('status', 'ne', 'cached') & Q('location.object', 'exists', True)
# & Q('metadata.parity', 'eq', None)
)
def default_node_list_query():
return (Q('is_deleted', 'ne', True) & Q('type', 'eq', 'osf.node'))
def figshare_view_file(*args, **kwargs):
auth = kwargs['auth']
node = kwargs['node'] or kwargs['project']
node_settings = kwargs['node_addon']
article_id = kwargs.get('aid') or None
file_id = kwargs.get('fid') or None
anonymous = has_anonymous_link(node, auth)
if not article_id or not file_id:
raise HTTPError(http.NOT_FOUND)
connect = Figshare.from_settings(node_settings.user_settings)
if node_settings.figshare_type == 'project':
item = connect.project(node_settings, node_settings.figshare_id)
else:
item = connect.article(node_settings, node_settings.figshare_id)
if article_id not in str(item):
raise HTTPError(http.NOT_FOUND)
article = connect.article(node_settings, article_id)
found = False
for f in article['items'][0]['files']:
if f['id'] == int(file_id):
found = f
break
if not found:
raise HTTPError(http.NOT_FOUND)
try:
# If GUID has already been created, we won't redirect, and can check
# whether the file exists below
guid = FigShareGuidFile.find_one(
Q('node', 'eq', node) & Q('article_id', 'eq', article_id)
& Q('file_id', 'eq', file_id))
except:
guid = FigShareGuidFile(node=node,
article_id=article_id,
file_id=file_id)
guid.save()
redirect_url = check_file_guid(guid)
if redirect_url:
return redirect(redirect_url)
private = not (article['items'][0]['status'] == 'Public')
figshare_url = 'http://figshare.com/'
if private:
figshare_url += 'preview/_preview/{0}'.format(
article['items'][0]['article_id'])
else:
figshare_url += 'articles/{0}/{1}'.format(
article['items'][0]['title'].replace(' ', '_'),
article['items'][0]['article_id'])
version_url = "http://figshare.com/articles/{filename}/{file_id}".format(
filename=article['items'][0]['title'],
file_id=article['items'][0]['article_id'])
download_url = node.api_url + 'figshare/download/article/{aid}/file/{fid}'.format(
aid=article_id, fid=file_id)
render_url = node.api_url + \
'figshare/render/article/{aid}/file/{fid}'.format(aid=article_id, fid=file_id)
delete_url = node.api_url + 'figshare/article/{aid}/file/{fid}/'.format(
aid=article_id, fid=file_id)
filename = found['name']
cache_file_name = get_cache_file(article_id, file_id)
rendered = get_cache_content(node_settings, cache_file_name)
if private:
rendered = messages.FIGSHARE_VIEW_FILE_PRIVATE.format(
url='http://figshare.com/')
elif rendered is None:
filename, size, filedata = connect.get_file(node_settings, found)
if figshare_settings.MAX_RENDER_SIZE is not None and size > figshare_settings.MAX_RENDER_SIZE:
rendered = messages.FIGSHARE_VIEW_FILE_OVERSIZED.format(
url=found.get('download_url'))
else:
rendered = get_cache_content(
node_settings,
cache_file_name,
start_render=True,
remote_path=filename,
file_content=filedata,
download_url=download_url,
)
# categories = connect.categories()['items'] # TODO Cache this
# categories = ''.join(
# ["<option value='{val}'>{label}</option>".format(val=i['id'], label=i['name']) for i in categories])
rv = {
'node': {
'id': node._id,
'title': node.title
},
'file_name':
filename,
'rendered':
rendered,
'file_status':
article['items'][0]['status'],
'file_version':
article['items'][0]['version'],
'doi':
'http://dx.doi.org/10.6084/m9.figshare.{0}'.format(
article['items'][0]['article_id']),
'parent_type':
'fileset'
if article['items'][0]['defined_type'] == 'fileset' else 'singlefile',
'parent_id':
article['items'][0]['article_id'],
# 'figshare_categories': categories,
'figshare_title':
article['items'][0]['title'],
'figshare_desc':
article['items'][0]['description'],
'render_url':
render_url,
'urls': {
'render': render_url,
'download': found.get('download_url'),
'version': version_url,
'figshare': privacy_info_handle(figshare_url, anonymous),
'delete': delete_url,
'files': node.web_url_for('collect_file_trees')
}
}
rv.update(_view_project(node, auth, primary=True))
return rv
def get_targets():
return model.OsfStorageFileVersion.find(
Q('status', 'ne', 'cached') & Q('location.object', 'exists', True))
def find_nested_projects():
return Node.find(
Q('__backrefs.parent.node.nodes.0', 'exists', True)
& Q('category', 'eq', 'project') & Q('is_deleted', 'eq', False))
def get_has_children(self, obj):
return Comment.find(Q('target', 'eq', Guid.load(obj._id))).count() > 0
def get_default_queryset(self):
return Node.find(
(Q('is_deleted', 'ne', True) & Q('is_public', 'eq', True)))
def send_confirm_email(user, email, renew=False, external_id_provider=None, external_id=None, destination=None):
"""
Sends `user` a confirmation to the given `email`.
:param user: the user
:param email: the email
:param renew: refresh the token
:param external_id_provider: user's external id provider
:param external_id: user's external id
:param destination: the destination page to redirect after confirmation
:return:
:raises: KeyError if user does not have a confirmation token for the given email.
"""
confirmation_url = user.get_confirmation_url(
email,
external=True,
force=True,
renew=renew,
external_id_provider=external_id_provider,
destination=destination
)
try:
merge_target = User.find_one(Q('emails', 'eq', email))
except NoResultsFound:
merge_target = None
campaign = campaigns.campaign_for_user(user)
branded_preprints_provider = None
# Choose the appropriate email template to use and add existing_user flag if a merge or adding an email.
if external_id_provider and external_id:
# First time login through external identity provider, link or create an OSF account confirmation
if user.external_identity[external_id_provider][external_id] == 'CREATE':
mail_template = mails.EXTERNAL_LOGIN_CONFIRM_EMAIL_CREATE
elif user.external_identity[external_id_provider][external_id] == 'LINK':
mail_template = mails.EXTERNAL_LOGIN_CONFIRM_EMAIL_LINK
elif merge_target:
# Merge account confirmation
mail_template = mails.CONFIRM_MERGE
confirmation_url = '{}?logout=1'.format(confirmation_url)
elif user.is_active:
# Add email confirmation
mail_template = mails.CONFIRM_EMAIL
confirmation_url = '{}?logout=1'.format(confirmation_url)
elif campaign:
# Account creation confirmation: from campaign
mail_template = campaigns.email_template_for_campaign(campaign)
if campaigns.is_proxy_login(campaign) and campaigns.get_service_provider(campaign) != 'OSF':
branded_preprints_provider = campaigns.get_service_provider(campaign)
else:
# Account creation confirmation: from OSF
mail_template = mails.INITIAL_CONFIRM_EMAIL
mails.send_mail(
email,
mail_template,
'plain',
user=user,
confirmation_url=confirmation_url,
email=email,
merge_target=merge_target,
external_id_provider=external_id_provider,
branded_preprints_provider=branded_preprints_provider
)
def get_default_odm_query(self):
inst = self.get_institution()
query = Q('affiliated_institutions', 'eq', inst)
return query
def get_targets():
return Node.find(Q('is_deleted', 'ne', True))
def get_users_with_none_in_email_verifications():
return models.User.find(Q('email_verifications', 'eq', None))
def get_targets():
return NodeLog.find(
Q('action', 'eq', NodeLog.EMBARGO_APPROVED)
& Q('params.user', 'eq', None))
def get_events(self, date):
super(InstitutionSummary, self).get_events(date)
institutions = self.get_institutions()
counts = []
# Convert to a datetime at midnight for queries and the timestamp
timestamp_datetime = datetime(date.year, date.month, date.day).replace(tzinfo=pytz.UTC)
query_datetime = timestamp_datetime + timedelta(1)
for institution in institutions:
user_query = Q('affiliated_institutions', 'eq', institution)
node_query = (
Q('is_deleted', 'ne', True) &
Q('date_created', 'lt', query_datetime)
)
project_query = node_query & Q('parent_nodes', 'eq', None)
public_query = Q('is_public', 'eq', True)
private_query = Q('is_public', 'eq', False)
node_public_query = node_query & public_query
node_private_query = node_query & private_query
project_public_query = project_query & public_query
project_private_query = project_query & private_query
count = {
'institution': {
'id': institution._id,
'name': institution.name,
},
'users': {
'total': OSFUser.find(user_query).count(),
},
'nodes': {
'total': AbstractNode.find_by_institutions(institution, node_query).count(),
'public': AbstractNode.find_by_institutions(institution, node_public_query).count(),
'private': AbstractNode.find_by_institutions(institution, node_private_query).count(),
},
'projects': {
'total': AbstractNode.find_by_institutions(institution, project_query).count(),
'public': AbstractNode.find_by_institutions(institution, project_public_query).count(),
'private': AbstractNode.find_by_institutions(institution, project_private_query).count(),
},
'registered_nodes': {
'total': Registration.find_by_institutions(institution, node_query).count(),
'public': Registration.find_by_institutions(institution, node_public_query).count(),
'embargoed': Registration.find_by_institutions(institution, node_private_query).count(),
},
'registered_projects': {
'total': Registration.find_by_institutions(institution, project_query).count(),
'public': Registration.find_by_institutions(institution, project_public_query).count(),
'embargoed': Registration.find_by_institutions(institution, project_private_query).count(),
},
'keen': {
'timestamp': timestamp_datetime.isoformat()
}
}
logger.info(
'{} Nodes counted. Nodes: {}, Projects: {}, Registered Nodes: {}, Registered Projects: {}'.format(
count['institution']['name'],
count['nodes']['total'],
count['projects']['total'],
count['registered_nodes']['total'],
count['registered_projects']['total']
)
)
counts.append(count)
return counts
def auth_callback(self, user):
"""Exchange temporary credentials for permanent credentials
This is called in the view that handles the user once they are returned
to the OSF after authenticating on the external service.
"""
session = get_session()
# make sure the user has temporary credentials for this provider
try:
cached_credentials = session.data['oauth_states'][self.short_name]
except KeyError:
raise PermissionsError("OAuth flow not recognized.")
if self._oauth_version == OAUTH1:
request_token = request.args.get('oauth_token')
# make sure this is the same user that started the flow
if cached_credentials.get('token') != request_token:
raise PermissionsError("Request token does not match")
response = OAuth1Session(
client_key=self.client_id,
client_secret=self.client_secret,
resource_owner_key=cached_credentials.get('token'),
resource_owner_secret=cached_credentials.get('secret'),
verifier=request.args.get('oauth_verifier'),
).fetch_access_token(self.callback_url)
elif self._oauth_version == OAUTH2:
state = request.args.get('state')
# make sure this is the same user that started the flow
if cached_credentials.get('state') != state:
raise PermissionsError("Request token does not match")
try:
response = OAuth2Session(
self.client_id,
redirect_uri=web_url_for(
'oauth_callback',
service_name=self.short_name,
_absolute=True
),
).fetch_token(
self.callback_url,
client_secret=self.client_secret,
code=request.args.get('code'),
)
except (MissingTokenError, RequestsHTTPError):
raise HTTPError(http.SERVICE_UNAVAILABLE)
# pre-set as many values as possible for the ``ExternalAccount``
info = self._default_handle_callback(response)
# call the hook for subclasses to parse values from the response
info.update(self.handle_callback(response))
try:
# create a new ``ExternalAccount`` ...
self.account = ExternalAccount(
provider=self.short_name,
provider_id=info['provider_id'],
provider_name=self.name,
)
self.account.save()
except KeyExistsException:
# ... or get the old one
self.account = ExternalAccount.find_one(
Q('provider', 'eq', self.short_name) &
Q('provider_id', 'eq', info['provider_id'])
)
assert self.account is not None
# ensure that provider_name is correct
self.account.provider_name = self.name
# required
self.account.oauth_key = info['key']
# only for OAuth1
self.account.oauth_secret = info.get('secret')
# only for OAuth2
self.account.expires_at = info.get('expires_at')
self.account.refresh_token = info.get('refresh_token')
# additional information
self.account.display_name = info.get('display_name')
self.account.profile_url = info.get('profile_url')
self.account.save()
# add it to the user's list of ``ExternalAccounts``
if self.account not in user.external_accounts:
user.external_accounts.append(self.account)
user.save()
def generate_schema_from_data(data):
def from_property(id, prop):
if isinstance(prop.get('value'), dict):
return {
'id': id,
'type': 'object',
'properties': [
from_property(pid, sp)
for pid, sp in prop['value'].items()
]
}
else:
return {
'id': id,
'type': 'osf-upload' if prop.get('extra') else 'string'
}
def from_question(qid, question):
if q.get('extra'):
return {
'qid': qid,
'type': 'osf-upload'
}
elif isinstance(q.get('value'), dict):
return {
'qid': qid,
'type': 'object',
'properties': [
from_property(id, value)
for id, value in question.get('value').items()
]
}
else:
return {
'qid': qid,
'type': 'string'
}
_schema = {
'name': 'Test',
'version': 2,
'config': {
'hasFiles': True
},
'pages': [{
'id': 'page1',
'questions': [
from_question(qid, q)
for qid, q in data.items()
]
}]
}
schema = MetaSchema(
name=_schema['name'],
schema_version=_schema['version'],
schema=_schema
)
try:
schema.save()
except KeyExistsException:
# Unfortunately, we don't have db isolation between test cases for some
# reason. Update the doc currently in the db rather than saving a new
# one.
schema = MetaSchema.find_one(
Q('name', 'eq', _schema['name']) &
Q('schema_version', 'eq', _schema['version'])
)
schema.schema = _schema
schema.save()
return schema
def get_broken_registrations():
return (
node for node
in Node.find(Q('is_registration', 'eq', True))
if has_duplicate_piwik_id(node)
)
from modularodm import Q
from website import settings
from website.project import Node
from keen import KeenClient
# Alias the project serializer
from website.project.views.node import _view_project
serialize_node = _view_project # Not recommended practice
CONTENT_NODE_QUERY = (
# Can encompass accessible projects, registrations, or forks
# Note: is_bookmark collection(s) are implicitly assumed to also be collections; that flag intentionally omitted
Q('is_collection', 'ne', True) & Q('is_deleted', 'eq', False))
PROJECT_QUERY = (
# Excludes registrations
CONTENT_NODE_QUERY & Q('is_registration', 'ne', True))
TOP_LEVEL_PROJECT_QUERY = (
# Top level project is defined based on whether node (of any category) has a parent. Can include forks.
Q('parent_node', 'eq', None) & PROJECT_QUERY)
def recent_public_registrations(n=10):
registrations = Node.find(CONTENT_NODE_QUERY & Q('parent_node', 'eq', None)
& Q('is_public', 'eq', True)
& Q('is_registration', 'eq', True)).sort(
'-registered_date')
def get_broken_forks():
return (
node for node
in Node.find(Q('is_fork', 'eq', True))
if has_duplicate_piwik_id(node)
)
def children(self):
return self.__class__.find(Q('parent', 'eq', self._id))
def get_broken_templated():
return (
node for node
in Node.find(Q('template_node', 'ne', None))
if has_duplicate_piwik_id(node)
)
def get_targets(date):
return model.OsfStorageFileVersion.find(
Q('date_created', 'lt', date - DELTA_DATE)
& Q('status', 'ne', 'cached') & Q('metadata.archive', 'exists', True))
def update_node(node, index=None, bulk=False):
index = index or INDEX
from website.addons.wiki.model import NodeWikiPage
category = get_doctype_from_node(node)
elastic_document_id = node._id
parent_id = node.parent_id
from website.files.models.osfstorage import OsfStorageFile
for file_ in paginated(OsfStorageFile, Q('node', 'eq', node)):
update_file(file_, index=index)
if node.is_deleted or not node.is_public or node.archiving:
delete_doc(elastic_document_id, node)
else:
try:
normalized_title = six.u(node.title)
except TypeError:
normalized_title = node.title
normalized_title = unicodedata.normalize('NFKD',
normalized_title).encode(
'ascii', 'ignore')
elastic_document = {
'id':
elastic_document_id,
'contributors': [{
'fullname': x.fullname,
'url': x.profile_url if x.is_active else None
} for x in node.visible_contributors if x is not None],
'title':
node.title,
'normalized_title':
normalized_title,
'category':
category,
'public':
node.is_public,
'tags': [tag._id for tag in node.tags if tag],
'description':
node.description,
'url':
node.url,
'is_registration':
node.is_registration,
'is_pending_registration':
node.is_pending_registration,
'is_retracted':
node.is_retracted,
'is_pending_retraction':
node.is_pending_retraction,
'embargo_end_date':
node.embargo_end_date.strftime("%A, %b. %d, %Y")
if node.embargo_end_date else False,
'is_pending_embargo':
node.is_pending_embargo,
'registered_date':
node.registered_date,
'wikis': {},
'parent_id':
parent_id,
'date_created':
node.date_created,
'license':
serialize_node_license_record(node.license),
'affiliated_institutions':
[inst.name for inst in node.affiliated_institutions],
'boost':
int(not node.is_registration) +
1, # This is for making registered projects less relevant
}
if not node.is_retracted:
for wiki in [
NodeWikiPage.load(x)
for x in node.wiki_pages_current.values()
]:
elastic_document['wikis'][wiki.page_name] = wiki.raw_text(node)
if bulk:
return elastic_document
else:
es.index(index=index,
doc_type=category,
id=elastic_document_id,
body=elastic_document,
refresh=True)
