def subscribe_mailchimp(list_name, user_id):
user = OSFUser.load(user_id)
m = get_mailchimp_api()
list_id = get_list_id_from_name(list_name=list_name)
if user.mailchimp_mailing_lists is None:
user.mailchimp_mailing_lists = {}
try:
m.lists.subscribe(
id=list_id,
email={'email': user.username},
merge_vars={
'fname': user.given_name,
'lname': user.family_name,
},
double_optin=False,
update_existing=True,
)
except (mailchimp.ValidationError, mailchimp.ListInvalidBounceMemberError) as error:
sentry.log_exception()
sentry.log_message(error)
user.mailchimp_mailing_lists[list_name] = False
else:
user.mailchimp_mailing_lists[list_name] = True
finally:
user.save()
def _collect_addons(self, node):
rv = []
for addon in node.get_addons():
if addon.config.has_hgrid_files:
# WARNING: get_hgrid_data can return None if the addon is added but has no credentials.
try:
temp = addon.config.get_hgrid_data(addon, self.auth, **self.extra)
except Exception as e:
logger.warn(
getattr(
e,
'data',
'Unexpected error when fetching file contents for {0}.'.format(addon.config.full_name)
)
)
sentry.log_exception()
rv.append({
KIND: FOLDER,
'unavailable': True,
'iconUrl': addon.config.icon_url,
'provider': addon.config.short_name,
'addonFullname': addon.config.full_name,
'permissions': {'view': False, 'edit': False},
'name': '{} is currently unavailable'.format(addon.config.full_name),
})
continue
rv.extend(sort_by_name(temp) or [])
return rv
def client():
global CLIENT
if CLIENT is None:
try:
CLIENT = Elasticsearch(settings.ELASTIC_URI,
request_timeout=settings.ELASTIC_TIMEOUT,
retry_on_timeout=True,
**settings.ELASTIC_KWARGS)
logging.getLogger('elasticsearch').setLevel(logging.WARN)
logging.getLogger('elasticsearch.trace').setLevel(logging.WARN)
logging.getLogger('urllib3').setLevel(logging.WARN)
logging.getLogger('requests').setLevel(logging.WARN)
CLIENT.cluster.health(wait_for_status='yellow')
except ConnectionError:
message = (
'The SEARCH_ENGINE setting is set to "elastic", but there '
'was a problem starting the elasticsearch interface. Is '
'elasticsearch running?')
if settings.SENTRY_DSN:
try:
sentry.log_exception()
sentry.log_message(message)
except AssertionError: # App has not yet been initialized
logger.exception(message)
else:
logger.error(message)
exit(1)
return CLIENT
def send_users_email(send_type):
"""Find pending Emails and amalgamates them into a single Email.
:param send_type
:return:
"""
grouped_emails = get_users_emails(send_type)
if not grouped_emails:
return
for group in grouped_emails:
user = User.load(group['user_id'])
if not user:
log_exception()
continue
info = group['info']
notification_ids = [message['_id'] for message in info]
sorted_messages = group_by_node(info)
if sorted_messages:
mails.send_mail(to_addr=user.username,
mimetype='html',
mail=mails.DIGEST,
name=user.fullname,
message=sorted_messages,
callback=remove_notifications(
email_notification_ids=notification_ids))
def sync_data_from_mailchimp(**kwargs):
"""Endpoint that the mailchimp webhook sends its data to"""
key = request.args.get('key')
if key == settings.MAILCHIMP_WEBHOOK_SECRET_KEY:
r = request
action = r.values['type']
list_name = mailchimp_utils.get_list_name_from_id(
list_id=r.values['data[list_id]'])
username = r.values['data[email]']
try:
user = User.find_one(Q('username', 'eq', username))
except NoResultsFound:
sentry.log_exception()
sentry.log_message("A user with this username does not exist.")
raise HTTPError(
404,
data=dict(
message_short='User not found',
message_long='A user with this username does not exist'))
if action == 'unsubscribe':
user.mailchimp_mailing_lists[list_name] = False
user.save()
elif action == 'subscribe':
user.mailchimp_mailing_lists[list_name] = True
user.save()
else:
# TODO: get tests to pass with sentry logging
# sentry.log_exception()
# sentry.log_message("Unauthorized request to the OSF.")
raise HTTPError(http.UNAUTHORIZED)
def send_users_email(send_type):
"""Find pending Emails and amalgamates them into a single Email.
:param send_type
:return:
"""
grouped_emails = get_users_emails(send_type)
if not grouped_emails:
return
for group in grouped_emails:
user = User.load(group["user_id"])
if not user:
log_exception()
continue
info = group["info"]
notification_ids = [message["_id"] for message in info]
sorted_messages = group_by_node(info)
if sorted_messages:
mails.send_mail(
to_addr=user.username,
mimetype="html",
mail=mails.DIGEST,
name=user.fullname,
message=sorted_messages,
callback=remove_notifications(email_notification_ids=notification_ids),
)
def on_preprint_updated(preprint_id, update_share=True):
# WARNING: Only perform Read-Only operations in an asynchronous task, until Repeatable Read/Serializable
# transactions are implemented in View and Task application layers.
from osf.models import PreprintService
preprint = PreprintService.load(preprint_id)
if preprint.node:
status = 'public' if preprint.node.is_public else 'unavailable'
try:
update_ezid_metadata_on_change(preprint, status=status)
except HTTPError as err:
sentry.log_exception()
sentry.log_message(err.args[0])
if settings.SHARE_URL and update_share:
if not preprint.provider.access_token:
raise ValueError('No access_token for {}. Unable to send {} to SHARE.'.format(preprint.provider, preprint))
resp = requests.post('{}api/v2/normalizeddata/'.format(settings.SHARE_URL), json={
'data': {
'type': 'NormalizedData',
'attributes': {
'tasks': [],
'raw': None,
'data': {'@graph': format_preprint(preprint)}
}
}
}, headers={'Authorization': 'Bearer {}'.format(preprint.provider.access_token), 'Content-Type': 'application/vnd.api+json'})
logger.debug(resp.content)
resp.raise_for_status()
def get_sub_folders(self, library_id, folder_id=None, **kwargs):
"""
Returns serialized folders underneath a specific library/group - these are the lower tiers of folders in Zotero.
If no folder_id is specified, all folders in a flat manner are returned for the group library.
If a folder_id is specified, only the subfolders within that folder are returned.
"""
try:
sub_folders = self.api._get_folders(library_id=library_id, folder_id=folder_id)
except zotero_errors.ResourceNotFound:
raise HTTPError(404)
except zotero_errors.UserNotAuthorised:
raise HTTPError(403)
except zotero_errors.HTTPError:
sentry.log_exception()
sentry.log_message('Unexpected Zotero Error when fetching folders.')
raise HTTPError(500)
serialized = []
for folder in sub_folders:
data = folder['data']
path = folder['library']['id'] if folder['library']['type'] == 'group' else 'personal'
serialized.append(self.serialize_folder('folder', data['key'], data['name'], path, data['parentCollection']))
if folder_id:
return serialized
else:
all_documents = self.serialize_folder('folder', 'ROOT', 'All Documents', library_id, '__', None)
return [all_documents] + serialized
def send_digest(grouped_digests):
""" Send digest emails and remove digests for sent messages in a callback.
:param grouped_digests: digest notification messages from the past 24 hours grouped by user
:return:
"""
for group in grouped_digests:
user = User.load(group['user_id'])
if not user:
sentry.log_exception()
sentry.log_message("A user with this username does not exist.")
return
info = group['info']
digest_notification_ids = [message['_id'] for message in info]
sorted_messages = group_messages_by_node(info)
if sorted_messages:
logger.info('Sending email digest to user {0!r}'.format(user))
mails.send_mail(
to_addr=user.username,
mimetype='html',
mail=mails.DIGEST,
name=user.fullname,
message=sorted_messages,
callback=remove_sent_digest_notifications.si(
digest_notification_ids=digest_notification_ids))
def send_users_email(send_type):
"""Find pending Emails and amalgamates them into a single Email.
:param send_type
:return:
"""
grouped_emails = get_users_emails(send_type)
for group in grouped_emails:
user = OSFUser.load(group['user_id'])
if not user:
log_exception()
continue
info = group['info']
notification_ids = [message['_id'] for message in info]
sorted_messages = group_by_node(info)
if sorted_messages:
if not user.is_disabled:
# If there's only one node in digest we can show it's preferences link in the template.
notification_nodes = sorted_messages['children'].keys()
node = AbstractNode.load(notification_nodes[0]) if len(
notification_nodes) == 1 else None
mails.send_mail(
to_addr=user.username,
mimetype='html',
can_change_node_preferences=bool(node),
node=node,
mail=mails.DIGEST,
name=user.fullname,
message=sorted_messages,
)
remove_notifications(email_notification_ids=notification_ids)
def get_folders(self, show_root=False, **kwargs):
if self.has_auth:
try:
folders = self.api._get_folders()
serialized_root_folder = {
'name': 'All Documents',
'provider_list_id': None,
'id': 'ROOT',
'parent_list_id': '__',
'kind': 'folder',
'addon': 'mendeley'
}
serialized_folders = [{
'addon': 'mendeley',
'kind': 'folder',
'id': folder.json['id'],
'name': folder.json['name'],
'path': folder.json.get('parent_id', '/'),
'parent_list_id': folder.json.get('parent_id', None),
'provider_list_id': folder.json['id']
} for folder in folders]
if show_root:
serialized_folders.insert(0, serialized_root_folder)
return serialized_folders
except MendeleyApiException as error:
sentry.log_exception()
sentry.log_message('Unexpected Mendeley Error when fetching folders.')
raise HTTPError(error.status)
else:
raise exceptions.InvalidAuthError()
def send_digest(grouped_digests):
""" Send digest emails and remove digests for sent messages in a callback.
:param grouped_digests: digest notification messages from the past 24 hours grouped by user
:return:
"""
for group in grouped_digests:
user = User.load(group['user_id'])
if not user:
sentry.log_exception()
sentry.log_message("A user with this username does not exist.")
return
info = group['info']
digest_notification_ids = [message['_id'] for message in info]
sorted_messages = group_messages_by_node(info)
if sorted_messages:
logger.info('Sending email digest to user {0!r}'.format(user))
mails.send_mail(
to_addr=user.username,
mimetype='html',
mail=mails.DIGEST,
name=user.fullname,
message=sorted_messages,
callback=remove_sent_digest_notifications.si(
digest_notification_ids=digest_notification_ids
)
)
def sync_data_from_mailchimp(**kwargs):
"""Endpoint that the mailchimp webhook sends its data to"""
key = request.args.get("key")
if key == settings.MAILCHIMP_WEBHOOK_SECRET_KEY:
r = request
action = r.values["type"]
list_name = mailchimp_utils.get_list_name_from_id(list_id=r.values["data[list_id]"])
username = r.values["data[email]"]
try:
user = User.find_one(Q("username", "eq", username))
except NoResultsFound:
sentry.log_exception()
sentry.log_message("A user with this username does not exist.")
raise HTTPError(
404, data=dict(message_short="User not found", message_long="A user with this username does not exist")
)
if action == "unsubscribe":
user.mailchimp_mailing_lists[list_name] = False
user.save()
elif action == "subscribe":
user.mailchimp_mailing_lists[list_name] = True
user.save()
else:
# TODO: get tests to pass with sentry logging
# sentry.log_exception()
# sentry.log_message("Unauthorized request to the OSF.")
raise HTTPError(http.UNAUTHORIZED)
def update_or_create_preprint_identifiers(preprint):
status = 'public' if preprint.verified_publishable and not preprint.is_retracted else 'unavailable'
try:
preprint.request_identifier_update(category='doi', status=status)
except HTTPError as err:
sentry.log_exception()
sentry.log_message(err.args[0])
def send_users_email(send_type):
"""Find pending Emails and amalgamates them into a single Email.
:param send_type
:return:
"""
grouped_emails = get_users_emails(send_type)
for group in grouped_emails:
user = OSFUser.load(group['user_id'])
if not user:
log_exception()
continue
info = group['info']
notification_ids = [message['_id'] for message in info]
sorted_messages = group_by_node(info)
if sorted_messages:
if not user.is_disabled:
mails.send_mail(
to_addr=user.username,
mimetype='html',
mail=mails.DIGEST,
name=user.fullname,
message=sorted_messages,
)
remove_notifications(email_notification_ids=notification_ids)
def update_search(self):
from website import search
try:
search.search.update_user(self)
except search.exceptions.SearchUnavailableError as e:
logger.exception(e)
log_exception()
def update_search(self):
from website import search
try:
search.search.update_user(self)
except search.exceptions.SearchUnavailableError as e:
logger.exception(e)
log_exception()
def sync_data_from_mailchimp(**kwargs):
"""Endpoint that the mailchimp webhook sends its data to"""
key = request.args.get('key')
if key == settings.MAILCHIMP_WEBHOOK_SECRET_KEY:
r = request
action = r.values['type']
list_name = mailchimp_utils.get_list_name_from_id(list_id=r.values['data[list_id]'])
username = r.values['data[email]']
try:
user = OSFUser.objects.get(username=username)
except OSFUser.DoesNotExist:
sentry.log_exception()
sentry.log_message('A user with this username does not exist.')
raise HTTPError(404, data=dict(message_short='User not found',
message_long='A user with this username does not exist'))
if action == 'unsubscribe':
user.mailchimp_mailing_lists[list_name] = False
user.save()
elif action == 'subscribe':
user.mailchimp_mailing_lists[list_name] = True
user.save()
else:
# TODO: get tests to pass with sentry logging
# sentry.log_exception()
# sentry.log_message("Unauthorized request to the OSF.")
raise HTTPError(http.UNAUTHORIZED)
def _send_global_and_node_emails(send_type):
"""
Called by `send_users_email`. Send all global and node-related notification emails.
"""
grouped_emails = get_users_emails(send_type)
for group in grouped_emails:
user = OSFUser.load(group['user_id'])
if not user:
log_exception()
continue
info = group['info']
notification_ids = [message['_id'] for message in info]
sorted_messages = group_by_node(info)
if sorted_messages:
if not user.is_disabled:
# If there's only one node in digest we can show it's preferences link in the template.
notification_nodes = list(sorted_messages['children'].keys())
node = AbstractNode.load(notification_nodes[0]) if len(
notification_nodes) == 1 else None
mails.send_mail(
to_addr=user.username,
can_change_node_preferences=bool(node),
node=node,
mail=mails.DIGEST,
name=user.fullname,
message=sorted_messages,
)
remove_notifications(email_notification_ids=notification_ids)
def _collect_addons(self, node):
rv = []
for addon in node.get_addons():
if addon.config.has_hgrid_files:
# WARNING: get_hgrid_data can return None if the addon is added but has no credentials.
try:
temp = addon.config.get_hgrid_data(addon, self.auth, **self.extra)
except Exception as e:
logger.warn(
getattr(
e,
'data',
'Unexpected error when fetching file contents for {0}.'.format(addon.config.full_name)
)
)
sentry.log_exception()
rv.append({
KIND: FOLDER,
'unavailable': True,
'iconUrl': addon.config.icon_url,
'provider': addon.config.short_name,
'addonFullname': addon.config.full_name,
'permissions': {'view': False, 'edit': False},
'name': '{} is currently unavailable'.format(addon.config.full_name),
})
continue
rv.extend(sort_by_name(temp) or [])
return rv
def test_log_not_logged_in(self, mock_capture):
session_record = Session()
set_session(session_record)
sentry.log_exception()
mock_capture.assert_called_with(extra={
'session': {},
}, )
def update_or_create_preprint_identifiers(preprint):
status = 'public' if preprint.verified_publishable else 'unavailable'
try:
preprint.request_identifier_update(category='doi', status=status)
except HTTPError as err:
sentry.log_exception()
sentry.log_message(err.args[0])
def wrapped(*args, **kwargs):
try:
return func(*args, **kwargs)
except exceptions.MalformedQueryError:
raise HTTPError(http.BAD_REQUEST,
data={
'message_short': 'Bad search query',
'message_long': language.SEARCH_QUERY_HELP,
})
except exceptions.SearchUnavailableError:
raise HTTPError(
http.SERVICE_UNAVAILABLE,
data={
'message_short':
'Search unavailable',
'message_long':
('Our search service is currently unavailable, if the issue persists, '
'please report it to <a href="mailto:[email protected]">[email protected]</a>.'
),
})
except exceptions.SearchException:
# Interim fix for issue where ES fails with 500 in some settings- ensure exception is still logged until it can be better debugged. See OSF-4538
sentry.log_exception()
sentry.log_message(
'Elasticsearch returned an unexpected error response')
# TODO: Add a test; may need to mock out the error response due to inability to reproduce error code locally
raise HTTPError(http.BAD_REQUEST,
data={
'message_short':
'Could not perform search query',
'message_long': language.SEARCH_QUERY_HELP,
})
def _send_global_and_node_emails(send_type):
"""
Called by `send_users_email`. Send all global and node-related notification emails.
"""
grouped_emails = get_users_emails(send_type)
for group in grouped_emails:
user = OSFUser.load(group['user_id'])
if not user:
log_exception()
continue
info = group['info']
notification_ids = [message['_id'] for message in info]
sorted_messages = group_by_node(info)
if sorted_messages:
if not user.is_disabled:
# If there's only one node in digest we can show it's preferences link in the template.
notification_nodes = sorted_messages['children'].keys()
node = AbstractNode.load(notification_nodes[0]) if len(
notification_nodes) == 1 else None
mails.send_mail(
to_addr=user.username,
mimetype='html',
can_change_node_preferences=bool(node),
node=node,
mail=mails.DIGEST,
name=user.fullname,
message=sorted_messages,
)
remove_notifications(email_notification_ids=notification_ids)
def update_search(self):
from website import search
try:
search.search.update_preprint(self, bulk=False, async_update=True)
except search.exceptions.SearchUnavailableError as e:
logger.exception(e)
log_exception()
def client():
global CLIENT
if CLIENT is None:
try:
CLIENT = Elasticsearch(
settings.ELASTIC_URI,
request_timeout=settings.ELASTIC_TIMEOUT,
retry_on_timeout=True,
**settings.ELASTIC_KWARGS
)
logging.getLogger('elasticsearch').setLevel(logging.WARN)
logging.getLogger('elasticsearch.trace').setLevel(logging.WARN)
logging.getLogger('urllib3').setLevel(logging.WARN)
logging.getLogger('requests').setLevel(logging.WARN)
CLIENT.cluster.health(wait_for_status='yellow')
except ConnectionError:
message = (
'The SEARCH_ENGINE setting is set to "elastic", but there '
'was a problem starting the elasticsearch interface. Is '
'elasticsearch running?'
)
if settings.SENTRY_DSN:
try:
sentry.log_exception()
sentry.log_message(message)
except AssertionError: # App has not yet been initialized
logger.exception(message)
else:
logger.error(message)
exit(1)
return CLIENT
def wrapped(*args, **kwargs):
if session:
session_error_code = session.data.get('auth_error_code')
else:
session_error_code = None
if session_error_code:
return renderer(HTTPError(session_error_code), **renderer_kwargs
or {})
try:
if renderer_kwargs:
kwargs.update(renderer_kwargs)
data = fn(*args, **kwargs)
except HTTPError as error:
data = error
except Exception as error:
logger.exception(error)
if settings.SENTRY_DSN and not app.debug:
sentry.log_exception()
if debug_mode:
raise
data = HTTPError(
http.INTERNAL_SERVER_ERROR,
message=repr(error),
)
return renderer(data, **renderer_kwargs or {})
def wrapped(*args, **kwargs):
if session:
session_error_code = session.data.get('auth_error_code')
else:
session_error_code = None
if session_error_code:
return renderer(
HTTPError(session_error_code),
**renderer_kwargs or {}
)
try:
if renderer_kwargs:
kwargs.update(renderer_kwargs)
data = fn(*args, **kwargs)
except HTTPError as error:
data = error
except Exception as error:
logger.exception(error)
if settings.SENTRY_DSN and not app.debug:
sentry.log_exception()
if debug_mode:
raise
data = HTTPError(
http.INTERNAL_SERVER_ERROR,
message=repr(error),
)
return renderer(data, **renderer_kwargs or {})
def subscribe_mailchimp(list_name, user_id):
user = User.load(user_id)
m = get_mailchimp_api()
list_id = get_list_id_from_name(list_name=list_name)
if user.mailchimp_mailing_lists is None:
user.mailchimp_mailing_lists = {}
try:
m.lists.subscribe(
id=list_id,
email={'email': user.username},
merge_vars={
'fname': user.given_name,
'lname': user.family_name,
},
double_optin=False,
update_existing=True,
)
except mailchimp.ValidationError as error:
sentry.log_exception()
sentry.log_message(error.message)
user.mailchimp_mailing_lists[list_name] = False
else:
user.mailchimp_mailing_lists[list_name] = True
finally:
user.save()
def update_search(self):
from website import search
try:
search.search.update_preprint(self, bulk=False, async_update=True)
except search.exceptions.SearchUnavailableError as e:
logger.exception(e)
log_exception()
def create(self, validated_data):
auth = get_user_auth(self.context['request'])
draft = validated_data.pop('draft')
registration_choice = validated_data.pop('registration_choice',
'immediate')
embargo_lifted = validated_data.pop('lift_embargo', None)
reviewer = is_prereg_admin_not_project_admin(self.context['request'],
draft)
children = validated_data.pop('children', [])
if children:
# First check that all children are valid
child_nodes = Node.objects.filter(guids___id__in=children)
if child_nodes.count() != len(children):
raise exceptions.ValidationError(
'Some child nodes could not be found.')
# Second check that metadata doesn't have files that are not in the child nodes being registered.
registering = children + [draft.branched_from._id]
orphan_files = self._find_orphan_files(registering, draft)
if orphan_files:
orphan_files_names = [
file_data['selectedFileName'] for file_data in orphan_files
]
raise exceptions.ValidationError(
'All files attached to this form must be registered to complete the process. '
'The following file(s) are attached, but are not part of a component being'
' registered: {}'.format(', '.join(orphan_files_names)))
try:
draft.validate_metadata(metadata=draft.registration_metadata,
reviewer=reviewer,
required_fields=True)
except ValidationValueError:
log_exception(
) # Probably indicates a bug on our end, so log to sentry
# TODO: Raise an error once our JSON schemas are updated
try:
registration = draft.register(auth, save=True, child_ids=children)
except NodeStateError as err:
raise exceptions.ValidationError(err)
if registration_choice == 'embargo':
if not embargo_lifted:
raise exceptions.ValidationError(
'lift_embargo must be specified.')
embargo_end_date = embargo_lifted.replace(tzinfo=pytz.utc)
try:
registration.embargo_registration(auth.user, embargo_end_date)
except ValidationError as err:
raise exceptions.ValidationError(err.message)
else:
try:
registration.require_approval(auth.user)
except NodeStateError as err:
raise exceptions.ValidationError(err)
registration.save()
return registration
def update_search(self):
from website.search.search import update_user
from website.search.exceptions import SearchUnavailableError
try:
update_user(self)
except SearchUnavailableError as e:
logger.exception(e)
log_exception()
def bulk_update_search(cls, preprints, index=None):
from website import search
try:
serialize = functools.partial(search.search.update_preprint, index=index, bulk=True, async_update=False)
search.search.bulk_update_nodes(serialize, preprints, index=index)
except search.exceptions.SearchUnavailableError as e:
logger.exception(e)
log_exception()
def bulk_update_search(cls, preprints, index=None):
from website import search
try:
serialize = functools.partial(search.search.update_preprint, index=index, bulk=True, async_update=False)
search.search.bulk_update_nodes(serialize, preprints, index=index)
except search.exceptions.SearchUnavailableError as e:
logger.exception(e)
log_exception()
def get_auth(auth, **kwargs):
cas_resp = None
if not auth.user:
# Central Authentication Server OAuth Bearer Token
authorization = request.headers.get('Authorization')
if authorization and authorization.startswith('Bearer '):
client = cas.get_client()
try:
access_token = cas.parse_auth_header(authorization)
cas_resp = client.profile(access_token)
except cas.CasError as err:
sentry.log_exception()
# NOTE: We assume that the request is an AJAX request
return json_renderer(err)
if cas_resp.authenticated:
auth.user = User.load(cas_resp.user)
if not auth.user:
auth.user = User.from_cookie(request.args.get('cookie'))
try:
action = request.args['action']
node_id = request.args['nid']
provider_name = request.args['provider']
except KeyError:
raise HTTPError(httplib.BAD_REQUEST)
node = Node.load(node_id)
if not node:
raise HTTPError(httplib.NOT_FOUND)
check_access(node, auth, action, cas_resp)
provider_settings = node.get_addon(provider_name)
if not provider_settings:
raise HTTPError(httplib.BAD_REQUEST)
try:
credentials = provider_settings.serialize_waterbutler_credentials()
settings = provider_settings.serialize_waterbutler_settings()
except exceptions.AddonError:
log_exception()
raise HTTPError(httplib.BAD_REQUEST)
return {
'auth':
make_auth(auth.user), # A waterbutler auth dict not an Auth object
'credentials':
credentials,
'settings':
settings,
'callback_url':
node.api_url_for(
('create_waterbutler_log'
if not node.is_registration else 'registration_callbacks'),
_absolute=True,
),
}
def test_log_not_logged_in(self, mock_capture):
session_record = Session()
set_session(session_record)
sentry.log_exception()
mock_capture.assert_called_with(
extra={
'session': {},
},
)
def update_or_create_preprint_identifiers(preprint):
status = 'public' if preprint.verified_publishable else 'unavailable'
if preprint.is_published and not preprint.get_identifier('doi'):
request_identifiers(preprint)
else:
try:
update_doi_metadata_on_change(preprint._id, status=status)
except HTTPError as err:
sentry.log_exception()
sentry.log_message(err.args[0])
def update_or_create_preprint_identifiers(preprint):
status = 'public' if preprint.verified_publishable else 'unavailable'
if preprint.is_published and not preprint.get_identifier('doi'):
get_and_set_preprint_identifiers(preprint)
else:
try:
update_ezid_metadata_on_change(preprint._id, status=status)
except HTTPError as err:
sentry.log_exception()
sentry.log_message(err.args[0])
def update_search(self, deleted_id=None):
from website import search
try:
search.search.update_group(self,
bulk=False,
async_update=True,
deleted_id=deleted_id)
except search.exceptions.SearchUnavailableError as e:
logger.exception(e)
log_exception()
def test_log_logged_in(self, mock_capture):
user = UserFactory()
session_record = Session()
session_record.data['auth_user_id'] = user._id
set_session(session_record)
sentry.log_exception()
mock_capture.assert_called_with(extra={
'session': {
'auth_user_id': user._id,
},
}, )
def update_share(resource, old_subjects=None):
data = serialize_share_data(resource, old_subjects)
resp = send_share_json(resource, data)
status_code = resp.status_code
try:
resp.raise_for_status()
except requests.HTTPError:
if status_code >= 500:
async_update_resource_share.delay(resource._id, old_subjects)
else:
log_exception()
def do_check_spam(self,
author,
author_email,
content,
request_headers,
update=True):
if self.spam_status == SpamStatus.HAM:
return False
if self.is_spammy:
return True
akismet_client = _get_akismet_client()
oopspam_client = _get_oopspam_client()
remote_addr = request_headers['Remote-Addr']
user_agent = request_headers.get('User-Agent')
referer = request_headers.get('Referer')
akismet_is_spam, pro_tip = akismet_client.check_comment(
user_ip=remote_addr,
user_agent=user_agent,
referrer=referer,
comment_content=content,
comment_author=author,
comment_author_email=author_email)
try:
oopspam_is_spam, oopspam_details = oopspam_client.check_content(
user_ip=remote_addr, content=content)
except oopspam.OOPSpamClientError:
sentry.log_exception()
oopspam_is_spam = False
if update:
self.spam_pro_tip = pro_tip
self.spam_data['headers'] = {
'Remote-Addr': remote_addr,
'User-Agent': user_agent,
'Referer': referer,
}
self.spam_data['content'] = content
self.spam_data['author'] = author
self.spam_data['author_email'] = author_email
if akismet_is_spam and oopspam_is_spam:
self.flag_spam()
self.spam_data['who_flagged'] = 'both'
self.spam_data['oopspam_data'] = oopspam_details
elif akismet_is_spam:
self.flag_spam()
self.spam_data['who_flagged'] = 'akismet'
elif oopspam_is_spam:
self.flag_spam()
self.spam_data['who_flagged'] = 'oopspam'
self.spam_data['oopspam_data'] = oopspam_details
return akismet_is_spam or oopspam_is_spam
def get_auth(auth, **kwargs):
cas_resp = None
if not auth.user:
# Central Authentication Server OAuth Bearer Token
authorization = request.headers.get('Authorization')
if authorization and authorization.startswith('Bearer '):
client = cas.get_client()
try:
access_token = cas.parse_auth_header(authorization)
cas_resp = client.profile(access_token)
except cas.CasError as err:
sentry.log_exception()
# NOTE: We assume that the request is an AJAX request
return json_renderer(err)
if cas_resp.authenticated:
auth.user = User.load(cas_resp.user)
if not auth.user:
auth.user = User.from_cookie(request.args.get('cookie'))
try:
action = request.args['action']
node_id = request.args['nid']
provider_name = request.args['provider']
except KeyError:
raise HTTPError(httplib.BAD_REQUEST)
node = Node.load(node_id)
if not node:
raise HTTPError(httplib.NOT_FOUND)
check_access(node, auth, action, cas_resp)
provider_settings = node.get_addon(provider_name)
if not provider_settings:
raise HTTPError(httplib.BAD_REQUEST)
try:
credentials = provider_settings.serialize_waterbutler_credentials()
settings = provider_settings.serialize_waterbutler_settings()
except exceptions.AddonError:
log_exception()
raise HTTPError(httplib.BAD_REQUEST)
return {
'auth': make_auth(auth.user), # A waterbutler auth dict not an Auth object
'credentials': credentials,
'settings': settings,
'callback_url': node.api_url_for(
('create_waterbutler_log' if not node.is_registration else 'registration_callbacks'),
_absolute=True,
),
}
def purge_trash(n):
qs = TrashedFile.objects.filter(purged__isnull=True, deleted__lt=timezone.now()-PURGE_DELTA, provider='osfstorage')
creds = Credentials.from_service_account_file(GCS_CREDS)
client = Client(credentials=creds)
total_bytes = 0
for tf in qs[:n]:
try:
total_bytes += tf._purge(client=client)
except Exception as e:
log_exception()
logger.error(f'Encountered Error handling {tf.id}')
return total_bytes
def test_log_logged_in(self, mock_capture):
user = UserFactory()
session_record = Session()
session_record.data['auth_user_id'] = user._id
set_session(session_record)
sentry.log_exception()
mock_capture.assert_called_with(
extra={
'session': {
'auth_user_id': user._id,
},
},
)
def unpurge_trash(ids):
qs = TrashedFile.objects.filter(purged__isnull=False, id__in=ids)
creds = Credentials.from_service_account_file(GCS_CREDS)
client = Client(credentials=creds)
if qs.count() < len(ids):
logger.warn('Some ids could not be found: {}'.format(
list(set(ids) - set(qs.values_list('id', flat=True)))))
for tf in qs.all():
logger.info(f'Unpurging {tf.id}')
try:
tf.restore(client=client)
except Exception as e:
log_exception()
logger.error(f'Encountered Error handling {tf.id}')
def get_auth(**kwargs):
try:
action = request.args['action']
node_id = request.args['nid']
provider_name = request.args['provider']
except KeyError:
raise HTTPError(httplib.BAD_REQUEST)
cookie = request.args.get('cookie')
view_only = request.args.get('view_only')
if 'auth_user_id' in session.data:
user = User.load(session.data['auth_user_id'])
elif cookie:
user = User.from_cookie(cookie)
else:
user = None
node = Node.load(node_id)
if not node:
raise HTTPError(httplib.NOT_FOUND)
check_access(node, user, action, key=view_only)
provider_settings = node.get_addon(provider_name)
if not provider_settings:
raise HTTPError(httplib.BAD_REQUEST)
try:
credentials = provider_settings.serialize_waterbutler_credentials()
settings = provider_settings.serialize_waterbutler_settings()
except exceptions.AddonError:
log_exception()
raise HTTPError(httplib.BAD_REQUEST)
return {
'auth':
make_auth(user),
'credentials':
credentials,
'settings':
settings,
'callback_url':
node.api_url_for(
('create_waterbutler_log'
if not node.is_registration else 'registration_callbacks'),
_absolute=True,
),
}
def create(self, validated_data):
auth = get_user_auth(self.context['request'])
draft = validated_data.pop('draft')
registration_choice = validated_data.pop('registration_choice', 'immediate')
embargo_lifted = validated_data.pop('lift_embargo', None)
reviewer = is_prereg_admin_not_project_admin(self.context['request'], draft)
children = validated_data.pop('children', [])
if children:
# First check that all children are valid
child_nodes = Node.objects.filter(guids___id__in=children)
if child_nodes.count() != len(children):
raise exceptions.ValidationError('Some child nodes could not be found.')
# Second check that metadata doesn't have files that are not in the child nodes being registered.
registering = children + [draft.branched_from._id]
orphan_files = self._find_orphan_files(registering, draft)
if orphan_files:
orphan_files_names = [file_data['selectedFileName'] for file_data in orphan_files]
raise exceptions.ValidationError('All files attached to this form must be registered to complete the process. '
'The following file(s) are attached, but are not part of a component being'
' registered: {}'.format(', '.join(orphan_files_names)))
try:
draft.validate_metadata(metadata=draft.registration_metadata, reviewer=reviewer, required_fields=True)
except ValidationValueError:
log_exception() # Probably indicates a bug on our end, so log to sentry
# TODO: Raise an error once our JSON schemas are updated
try:
registration = draft.register(auth, save=True, child_ids=children)
except NodeStateError as err:
raise exceptions.ValidationError(err)
if registration_choice == 'embargo':
if not embargo_lifted:
raise exceptions.ValidationError('lift_embargo must be specified.')
embargo_end_date = embargo_lifted.replace(tzinfo=pytz.utc)
try:
registration.embargo_registration(auth.user, embargo_end_date)
except ValidationError as err:
raise exceptions.ValidationError(err.message)
else:
try:
registration.require_approval(auth.user)
except NodeStateError as err:
raise exceptions.ValidationError(err)
registration.save()
return registration
def conference_data(meeting):
try:
conf = Conference.objects.get(endpoint__iexact=meeting)
except Conference.DoesNotExist:
raise HTTPError(httplib.NOT_FOUND)
nodes = AbstractNode.objects.filter(tags__id__in=Tag.objects.filter(name__iexact=meeting, system=False).values_list('id', flat=True), is_public=True, is_deleted=False)
ret = []
for idx, each in enumerate(nodes):
# To handle OSF-8864 where projects with no users caused meetings to be unable to resolve
try:
ret.append(_render_conference_node(each, idx, conf))
except IndexError:
sentry.log_exception()
return ret
def on_preprint_updated(preprint_id, update_share=True, share_type=None, old_subjects=None):
# WARNING: Only perform Read-Only operations in an asynchronous task, until Repeatable Read/Serializable
# transactions are implemented in View and Task application layers.
from osf.models import PreprintService
preprint = PreprintService.load(preprint_id)
if old_subjects is None:
old_subjects = []
if preprint.node:
status = 'public' if preprint.verified_publishable else 'unavailable'
try:
update_ezid_metadata_on_change(preprint._id, status=status)
except HTTPError as err:
sentry.log_exception()
sentry.log_message(err.args[0])
if update_share:
update_preprint_share(preprint, old_subjects, share_type)
def on_preprint_updated(preprint_id, update_share=True, share_type=None, old_subjects=None):
# WARNING: Only perform Read-Only operations in an asynchronous task, until Repeatable Read/Serializable
# transactions are implemented in View and Task application layers.
from osf.models import PreprintService
preprint = PreprintService.load(preprint_id)
if old_subjects is None:
old_subjects = []
if preprint.node:
status = 'public' if preprint.verified_publishable else 'unavailable'
try:
update_ezid_metadata_on_change(preprint._id, status=status)
except HTTPError as err:
sentry.log_exception()
sentry.log_message(err.args[0])
if update_share:
update_preprint_share(preprint, old_subjects, share_type)
def get_top_level_folders(self, **kwargs):
"""
Returns serialized group libraries - your personal library along with any group libraries.
This is the top-tier of "folders" in Zotero.
You can use kwargs to refine what data is returned - how to limit the number of group libraries,
whether to return the personal library alongside group_libraries, or append the total library count.
"""
# These kwargs are passed in from ZoteroViews > library_list
limit = kwargs.get('limit', None)
start = kwargs.get('start', None)
return_count = kwargs.get('return_count', False)
append_personal = kwargs.get('append_personal', True)
try:
# Fetch group libraries
libraries = self.api._fetch_libraries(limit=limit, start=start)
except zotero_errors.ResourceNotFound:
raise HTTPError(404)
except zotero_errors.UserNotAuthorised:
raise HTTPError(403)
except zotero_errors.HTTPError:
sentry.log_exception()
sentry.log_message(
'Unexpected Zotero Error when fetching group libraries.')
raise HTTPError(500)
# Serialize libraries
serialized = []
for library in libraries[:-1]:
data = library['data']
serialized.append(
self.serialize_folder('library', data['id'], data['name'],
str(data['id'])))
if return_count:
# Return total number of libraries as last item in list
serialized.append(libraries[-1])
if append_personal:
# Append personal library as option alongside group libraries
serialized.insert(
0,
self.serialize_folder('library', 'personal', 'My Library',
'personal'))
return serialized
def conference_data(meeting):
try:
conf = Conference.objects.get(endpoint__iexact=meeting)
except Conference.DoesNotExist:
raise HTTPError(httplib.NOT_FOUND)
nodes = AbstractNode.objects.filter(tags__id__in=Tag.objects.filter(
name__iexact=meeting, system=False).values_list('id', flat=True),
is_public=True,
is_deleted=False)
ret = []
for idx, each in enumerate(nodes):
# To handle OSF-8864 where projects with no users caused meetings to be unable to resolve
try:
ret.append(_render_conference_node(each, idx, conf))
except IndexError:
sentry.log_exception()
return ret
def get_auth(**kwargs):
try:
action = request.args['action']
node_id = request.args['nid']
provider_name = request.args['provider']
except KeyError:
raise HTTPError(httplib.BAD_REQUEST)
cookie = request.args.get('cookie')
view_only = request.args.get('view_only')
if 'auth_user_id' in session.data:
user = User.load(session.data['auth_user_id'])
elif cookie:
user = User.from_cookie(cookie)
else:
user = None
node = Node.load(node_id)
if not node:
raise HTTPError(httplib.NOT_FOUND)
check_access(node, user, action, key=view_only)
provider_settings = node.get_addon(provider_name)
if not provider_settings:
raise HTTPError(httplib.BAD_REQUEST)
try:
credentials = provider_settings.serialize_waterbutler_credentials()
settings = provider_settings.serialize_waterbutler_settings()
except exceptions.AddonError:
log_exception()
raise HTTPError(httplib.BAD_REQUEST)
return {
'auth': make_auth(user),
'credentials': credentials,
'settings': settings,
'callback_url': node.api_url_for(
('create_waterbutler_log' if not node.is_registration else 'registration_callbacks'),
_absolute=True,
),
}
def get_auth(**kwargs):
try:
action = request.args['action']
cookie = request.args['cookie']
node_id = request.args['nid']
provider_name = request.args['provider']
except KeyError:
raise HTTPError(httplib.BAD_REQUEST)
view_only = request.args.get('view_only')
user = get_user_from_cookie(cookie)
node = Node.load(node_id)
if not node:
raise HTTPError(httplib.NOT_FOUND)
check_access(node, user, action, key=view_only)
provider_settings = node.get_addon(provider_name)
if not provider_settings:
raise HTTPError(httplib.BAD_REQUEST)
try:
credentials = provider_settings.serialize_waterbutler_credentials()
settings = provider_settings.serialize_waterbutler_settings()
except exceptions.AddonError:
log_exception()
raise HTTPError(httplib.BAD_REQUEST)
return {
'auth':
make_auth(user),
'credentials':
credentials,
'settings':
settings,
'callback_url':
node.api_url_for(
'create_waterbutler_log',
_absolute=True,
),
}
def get_top_level_folders(self, **kwargs):
"""
Returns serialized group libraries - your personal library along with any group libraries.
This is the top-tier of "folders" in Zotero.
You can use kwargs to refine what data is returned - how to limit the number of group libraries,
whether to return the personal library alongside group_libraries, or append the total library count.
"""
# These kwargs are passed in from ZoteroViews > library_list
limit = kwargs.get('limit', None)
start = kwargs.get('start', None)
return_count = kwargs.get('return_count', False)
append_personal = kwargs.get('append_personal', True)
try:
# Fetch group libraries
libraries = self.api._fetch_libraries(limit=limit, start=start)
except zotero_errors.ResourceNotFound:
raise HTTPError(404)
except zotero_errors.UserNotAuthorised:
raise HTTPError(403)
except zotero_errors.HTTPError:
sentry.log_exception()
sentry.log_message('Unexpected Zotero Error when fetching group libraries.')
raise HTTPError(500)
# Serialize libraries
serialized = []
for library in libraries[:-1]:
data = library['data']
serialized.append(self.serialize_folder('library', data['id'], data['name'], str(data['id'])))
if return_count:
# Return total number of libraries as last item in list
serialized.append(libraries[-1])
if append_personal:
# Append personal library as option alongside group libraries
serialized.insert(0, self.serialize_folder('library', 'personal', 'My Library', 'personal'))
return serialized
def get_auth(**kwargs):
try:
action = request.args['action']
cookie = request.args['cookie']
node_id = request.args['nid']
provider_name = request.args['provider']
except KeyError:
raise HTTPError(httplib.BAD_REQUEST)
view_only = request.args.get('view_only')
user = get_user_from_cookie(cookie)
node = Node.load(node_id)
if not node:
raise HTTPError(httplib.NOT_FOUND)
check_access(node, user, action, key=view_only)
provider_settings = node.get_addon(provider_name)
if not provider_settings:
raise HTTPError(httplib.BAD_REQUEST)
try:
credentials = provider_settings.serialize_waterbutler_credentials()
settings = provider_settings.serialize_waterbutler_settings()
except exceptions.AddonError:
log_exception()
raise HTTPError(httplib.BAD_REQUEST)
return {
'auth': make_auth(user),
'credentials': credentials,
'settings': settings,
'callback_url': node.api_url_for(
'create_waterbutler_log',
_absolute=True,
),
}
def wrapped(*args, **kwargs):
try:
return func(*args, **kwargs)
except exceptions.MalformedQueryError:
raise HTTPError(http.BAD_REQUEST, data={
'message_short': 'Bad search query',
'message_long': language.SEARCH_QUERY_HELP,
})
except exceptions.SearchUnavailableError:
raise HTTPError(http.SERVICE_UNAVAILABLE, data={
'message_short': 'Search unavailable',
'message_long': ('Our search service is currently unavailable, if the issue persists, '
'please report it to <a href="mailto:[email protected]">[email protected]</a>.'),
})
except exceptions.SearchException:
# Interim fix for issue where ES fails with 500 in some settings- ensure exception is still logged until it can be better debugged. See OSF-4538
sentry.log_exception()
sentry.log_message('Elasticsearch returned an unexpected error response')
# TODO: Add a test; may need to mock out the error response due to inability to reproduce error code locally
raise HTTPError(http.BAD_REQUEST, data={
'message_short': 'Could not perform search query',
'message_long': language.SEARCH_QUERY_HELP,
})
def get_auth(auth, **kwargs):
cas_resp = None
if not auth.user:
# Central Authentication Server OAuth Bearer Token
authorization = request.headers.get('Authorization')
if authorization and authorization.startswith('Bearer '):
client = cas.get_client()
try:
access_token = cas.parse_auth_header(authorization)
cas_resp = client.profile(access_token)
except cas.CasError as err:
sentry.log_exception()
# NOTE: We assume that the request is an AJAX request
return json_renderer(err)
if cas_resp.authenticated:
auth.user = User.load(cas_resp.user)
try:
data = jwt.decode(
jwe.decrypt(request.args.get('payload', '').encode('utf-8'), WATERBUTLER_JWE_KEY),
settings.WATERBUTLER_JWT_SECRET,
options={'require_exp': True},
algorithm=settings.WATERBUTLER_JWT_ALGORITHM
)['data']
except (jwt.InvalidTokenError, KeyError):
raise HTTPError(httplib.FORBIDDEN)
if not auth.user:
auth.user = User.from_cookie(data.get('cookie', ''))
try:
action = data['action']
node_id = data['nid']
provider_name = data['provider']
except KeyError:
raise HTTPError(httplib.BAD_REQUEST)
node = Node.load(node_id)
if not node:
raise HTTPError(httplib.NOT_FOUND)
check_access(node, auth, action, cas_resp)
provider_settings = node.get_addon(provider_name)
if not provider_settings:
raise HTTPError(httplib.BAD_REQUEST)
try:
credentials = provider_settings.serialize_waterbutler_credentials()
waterbutler_settings = provider_settings.serialize_waterbutler_settings()
except exceptions.AddonError:
log_exception()
raise HTTPError(httplib.BAD_REQUEST)
return {'payload': jwe.encrypt(jwt.encode({
'exp': datetime.datetime.utcnow() + datetime.timedelta(seconds=settings.WATERBUTLER_JWT_EXPIRATION),
'data': {
'auth': make_auth(auth.user), # A waterbutler auth dict not an Auth object
'credentials': credentials,
'settings': waterbutler_settings,
'callback_url': node.api_url_for(
('create_waterbutler_log' if not node.is_registration else 'registration_callbacks'),
_absolute=True,
),
}
}, settings.WATERBUTLER_JWT_SECRET, algorithm=settings.WATERBUTLER_JWT_ALGORITHM), WATERBUTLER_JWE_KEY)}
ENGLISH_ANALYZER_PROPERTY = {'type': 'string', 'analyzer': 'english'}
INDEX = settings.ELASTIC_INDEX
try:
es = Elasticsearch(
settings.ELASTIC_URI,
request_timeout=settings.ELASTIC_TIMEOUT
)
logging.getLogger('elasticsearch').setLevel(logging.WARN)
logging.getLogger('elasticsearch.trace').setLevel(logging.WARN)
logging.getLogger('urllib3').setLevel(logging.WARN)
logging.getLogger('requests').setLevel(logging.WARN)
es.cluster.health(wait_for_status='yellow')
except ConnectionError as e:
sentry.log_exception()
sentry.log_message("The SEARCH_ENGINE setting is set to 'elastic', but there "
"was a problem starting the elasticsearch interface. Is "
"elasticsearch running?")
es = None
def requires_search(func):
def wrapped(*args, **kwargs):
if es is not None:
try:
return func(*args, **kwargs)
except ConnectionError:
raise exceptions.SearchUnavailableError('Could not connect to elasticsearch')
except NotFoundError as e:
raise exceptions.IndexNotFoundError(e.error)