def osfstorage_download(file_node, payload, node_addon, **kwargs):
# Set user ID in session data for checking if user is contributor
# to project.
user_id = payload.get('user')
if user_id:
current_session = get_session()
current_session.data['auth_user_id'] = user_id
current_session.save()
if not request.args.get('version'):
version_id = None
else:
try:
version_id = int(request.args['version'])
except ValueError:
raise make_error(httplib.BAD_REQUEST, message_short='Version must be an integer if not specified')
version = file_node.get_version(version_id, required=True)
if request.args.get('mode') not in ('render', ):
utils.update_analytics(node_addon.owner, file_node._id, int(version.identifier) - 1)
return {
'data': {
'name': file_node.name,
'path': version.location_hash,
},
'settings': {
osf_storage_settings.WATERBUTLER_RESOURCE: version.location[osf_storage_settings.WATERBUTLER_RESOURCE],
},
}
def test_migrate_download_counts(self, mock_session):
names = []
for index, num in enumerate(range(10)):
names.append('DEAR GOD$! ({})^ CARPNADOS'.format(num))
fobj, _ = oldels.OsfStorageFileRecord.get_or_create(
names[-1], self.node_settings)
for _id in range(index):
fobj.create_version(
self.user, {
'folder': '',
'bucket': '',
'service': 'buttfiles',
'object': '{}{}'.format(index, _id),
})
utils.update_analytics(self.project, fobj.path, _id + 1)
assert len(fobj.versions) == index
assert fobj.get_download_count() == index
assert len(self.node_settings.file_tree.children) == 10
migration.migrate_node_settings(self.node_settings, dry=False)
migration.migrate_children(self.node_settings, dry=False)
for index, child in enumerate(self.node_settings.root_node.children):
assert len(child.versions) == index
assert child.get_download_count() == index
for _id in range(index):
assert child.get_download_count(_id) == 1
def osfstorage_download(file_node, payload, node_addon, **kwargs):
# Set user ID in session data for checking if user is contributor
# to project.
user_id = payload.get('user')
if user_id:
current_session = get_session()
current_session.data['auth_user_id'] = user_id
if not request.args.get('version'):
version_id = None
else:
try:
version_id = int(request.args['version'])
except ValueError:
raise make_error(
httplib.BAD_REQUEST,
message_short='Version must be an integer if not specified')
version = file_node.get_version(version_id, required=True)
if request.args.get('mode') not in ('render', ):
utils.update_analytics(node_addon.owner, file_node._id,
int(version.identifier) - 1)
return {
'data': {
'name': file_node.name,
'path': version.location_hash,
},
'settings': {
osf_storage_settings.WATERBUTLER_RESOURCE:
version.location[osf_storage_settings.WATERBUTLER_RESOURCE],
},
}
def test_migrate_download_counts(self, mock_session):
names = []
for index, num in enumerate(range(10)):
names.append('DEAR GOD$! ({})^ CARPNADOS'.format(num))
fobj, _ = oldels.OsfStorageFileRecord.get_or_create(names[-1], self.node_settings)
for _id in range(index):
fobj.create_version(self.user, {
'folder': '',
'bucket': '',
'service': 'buttfiles',
'object': '{}{}'.format(index, _id),
})
utils.update_analytics(self.project, fobj.path, _id + 1)
assert len(fobj.versions) == index
assert fobj.get_download_count() == index
assert len(self.node_settings.file_tree.children) == 10
migration.migrate_node_settings(self.node_settings, dry=False)
migration.migrate_children(self.node_settings, dry=False)
for index, child in enumerate(self.node_settings.root_node.children):
assert len(child.versions) == index
assert child.get_download_count() == index
for _id in range(index):
assert child.get_download_count(_id) == 1
def test_download_count(self):
# Keen does not allow same day requests so we have to do some time traveling to my birthday
with mock.patch('django.utils.timezone.now', return_value=datetime.datetime(1991, 9, 25).replace(tzinfo=pytz.utc)):
node = ProjectFactory()
utils.update_analytics(node, 'fake id', {'contributors': node.contributors})
query_date = datetime.date(1991, 9, 25)
event = DownloadCountSummary().get_events(query_date)
assert event[0]['files']['total'] == 1
def test_download_count(self):
# Keen does not allow same day requests so we have to do some time traveling to my birthday
with mock.patch('django.utils.timezone.now', return_value=datetime.datetime(1991, 9, 25).replace(tzinfo=pytz.utc)):
node = ProjectFactory()
file = api_utils.create_test_file(
node, node.creator, filename='file_one')
utils.update_analytics(node, file, 0, 'download')
query_date = datetime.date(1991, 9, 25)
event = DownloadCountSummary().get_events(query_date)
assert event[0]['files']['total'] == 1
def test_download_count(self):
# Keen does not allow same day requests so we have to do some time traveling to my birthday
with mock.patch('django.utils.timezone.now',
return_value=datetime.datetime(
1991, 9, 25).replace(tzinfo=pytz.utc)):
node = ProjectFactory()
utils.update_analytics(node, 'fake id',
{'contributors': node.contributors})
query_date = datetime.date(1991, 9, 25)
event = DownloadCountSummary().get_events(query_date)
assert event[0]['files']['total'] == 1
def test_serialize_revision(self):
sessions.sessions[request._get_current_object()] = Session()
utils.update_analytics(self.project, self.record, 0)
utils.update_analytics(self.project, self.record, 0)
utils.update_analytics(self.project, self.record, 2)
expected = {
'index': 1,
'user': {
'name': self.user.fullname,
'url': self.user.url,
},
'date': self.versions[0].created.isoformat(),
'downloads': 2,
'md5': None,
'sha256': None,
}
observed = utils.serialize_revision(
self.project,
self.record,
self.versions[0],
0,
)
assert_equal(expected, observed)
assert_equal(self.record.get_download_count(), 3)
assert_equal(self.record.get_download_count(version=2), 1)
assert_equal(self.record.get_download_count(version=0), 2)
def test_serialize_revision(self):
sessions.sessions[request._get_current_object()] = Session()
utils.update_analytics(self.project, self.record._id, 0)
utils.update_analytics(self.project, self.record._id, 0)
utils.update_analytics(self.project, self.record._id, 2)
expected = {
'index': 1,
'user': {
'name': self.user.fullname,
'url': self.user.url,
},
'date': self.versions[0].created.isoformat(),
'downloads': 2,
'md5': None,
'sha256': None,
}
observed = utils.serialize_revision(
self.project,
self.record,
self.versions[0],
0,
)
assert_equal(expected, observed)
assert_equal(self.record.get_download_count(), 3)
assert_equal(self.record.get_download_count(version=2), 1)
assert_equal(self.record.get_download_count(version=0), 2)
def test_download_count_file(self, mock_session):
mock_session.data = {}
child = self.node_settings.get_root().append_file('Test')
utils.update_analytics(self.project, child._id, 0)
utils.update_analytics(self.project, child._id, 1)
utils.update_analytics(self.project, child._id, 2)
assert_equals(child.get_download_count(), 3)
assert_equals(child.get_download_count(0), 1)
assert_equals(child.get_download_count(1), 1)
assert_equals(child.get_download_count(2), 1)
def test_download_count_file(self, mock_session):
mock_session.data = {}
child = self.node_settings.get_root().append_file('Test')
utils.update_analytics(self.project, child._id, 0)
utils.update_analytics(self.project, child._id, 1)
utils.update_analytics(self.project, child._id, 2)
assert_equals(child.get_download_count(), 3)
assert_equals(child.get_download_count(0), 1)
assert_equals(child.get_download_count(1), 1)
assert_equals(child.get_download_count(2), 1)
def test_anon_revisions(self):
sessions.sessions[request._get_current_object()] = Session()
utils.update_analytics(self.project, self.record, 0)
utils.update_analytics(self.project, self.record, 0)
utils.update_analytics(self.project, self.record, 2)
expected = {
'index': 2,
'user': None,
'date': self.versions[0].created.isoformat(),
'downloads': 0,
'md5': None,
'sha256': None,
}
observed = utils.serialize_revision(self.project,
self.record,
self.versions[0],
1,
anon=True)
assert_equal(expected, observed)
def test_anon_revisions(self):
sessions.sessions[request._get_current_object()] = Session()
utils.update_analytics(self.project, self.record._id, 0)
utils.update_analytics(self.project, self.record._id, 0)
utils.update_analytics(self.project, self.record._id, 2)
expected = {
'index': 2,
'user': None,
'date': self.versions[0].created.isoformat(),
'downloads': 0,
'md5': None,
'sha256': None,
}
observed = utils.serialize_revision(
self.project,
self.record,
self.versions[0],
1,
anon=True
)
assert_equal(expected, observed)
def get_auth(auth, **kwargs):
cas_resp = None
if not auth.user:
# Central Authentication Server OAuth Bearer Token
authorization = request.headers.get('Authorization')
if authorization and authorization.startswith('Bearer '):
client = cas.get_client()
try:
access_token = cas.parse_auth_header(authorization)
cas_resp = client.profile(access_token)
except cas.CasError as err:
sentry.log_exception()
# NOTE: We assume that the request is an AJAX request
return json_renderer(err)
if cas_resp.authenticated:
auth.user = OSFUser.load(cas_resp.user)
try:
data = jwt.decode(jwe.decrypt(
request.args.get('payload', '').encode('utf-8'),
WATERBUTLER_JWE_KEY),
settings.WATERBUTLER_JWT_SECRET,
options={'require_exp': True},
algorithm=settings.WATERBUTLER_JWT_ALGORITHM)['data']
except (jwt.InvalidTokenError, KeyError) as err:
sentry.log_message(str(err))
raise HTTPError(http_status.HTTP_403_FORBIDDEN)
if not auth.user:
auth.user = OSFUser.from_cookie(data.get('cookie', ''))
try:
action = data['action']
node_id = data['nid']
provider_name = data['provider']
except KeyError:
raise HTTPError(http_status.HTTP_400_BAD_REQUEST)
node = AbstractNode.load(node_id) or Preprint.load(node_id)
if node and node.is_deleted:
raise HTTPError(http_status.HTTP_410_GONE)
elif not node:
raise HTTPError(http_status.HTTP_404_NOT_FOUND)
check_access(node, auth, action, cas_resp)
provider_settings = None
if hasattr(node, 'get_addon'):
provider_settings = node.get_addon(provider_name)
if not provider_settings:
raise HTTPError(http_status.HTTP_400_BAD_REQUEST)
path = data.get('path')
credentials = None
waterbutler_settings = None
fileversion = None
if provider_name == 'osfstorage':
if path:
file_id = path.strip('/')
# check to see if this is a file or a folder
filenode = OsfStorageFileNode.load(path.strip('/'))
if filenode and filenode.is_file:
# default to most recent version if none is provided in the response
version = int(data['version']) if data.get(
'version') else filenode.versions.count()
try:
fileversion = FileVersion.objects.filter(
basefilenode___id=file_id,
identifier=version).select_related('region').get()
except FileVersion.DoesNotExist:
raise HTTPError(http_status.HTTP_400_BAD_REQUEST)
if auth.user:
# mark fileversion as seen
FileVersionUserMetadata.objects.get_or_create(
user=auth.user, file_version=fileversion)
if not node.is_contributor_or_group_member(auth.user):
from_mfr = download_is_from_mfr(request, payload=data)
# version index is 0 based
version_index = version - 1
if action == 'render':
update_analytics(node, filenode, version_index, 'view')
elif action == 'download' and not from_mfr:
update_analytics(node, filenode, version_index,
'download')
if waffle.switch_is_active(features.ELASTICSEARCH_METRICS):
if isinstance(node, Preprint):
metric_class = get_metric_class_for_action(
action, from_mfr=from_mfr)
if metric_class:
sloan_flags = {
'sloan_id':
request.cookies.get(SLOAN_ID_COOKIE_NAME)
}
for flag_name in SLOAN_FLAGS:
value = request.cookies.get(
f'dwf_{flag_name}_custom_domain'
) or request.cookies.get(
f'dwf_{flag_name}')
if value:
sloan_flags[flag_name.replace(
'_display', '')] = strtobool(value)
try:
metric_class.record_for_preprint(
preprint=node,
user=auth.user,
version=fileversion.identifier
if fileversion else None,
path=path,
**sloan_flags)
except es_exceptions.ConnectionError:
log_exception()
if fileversion and provider_settings:
region = fileversion.region
credentials = region.waterbutler_credentials
waterbutler_settings = fileversion.serialize_waterbutler_settings(
node_id=provider_settings.owner._id,
root_id=provider_settings.root_node._id,
)
# If they haven't been set by version region, use the NodeSettings or Preprint directly
if not (credentials and waterbutler_settings):
credentials = node.serialize_waterbutler_credentials(provider_name)
waterbutler_settings = node.serialize_waterbutler_settings(
provider_name)
if isinstance(credentials.get('token'), bytes):
credentials['token'] = credentials.get('token').decode()
return {
'payload':
jwe.encrypt(
jwt.encode(
{
'exp':
timezone.now() + datetime.timedelta(
seconds=settings.WATERBUTLER_JWT_EXPIRATION),
'data': {
'auth':
make_auth(
auth.user
), # A waterbutler auth dict not an Auth object
'credentials':
credentials,
'settings':
waterbutler_settings,
'callback_url':
node.api_url_for(
('create_waterbutler_log'
if not getattr(node, 'is_registration', False)
else 'registration_callbacks'),
_absolute=True,
_internal=True)
}
},
settings.WATERBUTLER_JWT_SECRET,
algorithm=settings.WATERBUTLER_JWT_ALGORITHM),
WATERBUTLER_JWE_KEY).decode()
}
def get_auth(auth, **kwargs):
cas_resp = None
if not auth.user:
# Central Authentication Server OAuth Bearer Token
authorization = request.headers.get('Authorization')
if authorization and authorization.startswith('Bearer '):
client = cas.get_client()
try:
access_token = cas.parse_auth_header(authorization)
cas_resp = client.profile(access_token)
except cas.CasError as err:
sentry.log_exception()
# NOTE: We assume that the request is an AJAX request
return json_renderer(err)
if cas_resp.authenticated:
auth.user = OSFUser.load(cas_resp.user)
try:
data = jwt.decode(
jwe.decrypt(request.args.get('payload', '').encode('utf-8'), WATERBUTLER_JWE_KEY),
settings.WATERBUTLER_JWT_SECRET,
options={'require_exp': True},
algorithm=settings.WATERBUTLER_JWT_ALGORITHM
)['data']
except (jwt.InvalidTokenError, KeyError) as err:
sentry.log_message(str(err))
raise HTTPError(httplib.FORBIDDEN)
if not auth.user:
auth.user = OSFUser.from_cookie(data.get('cookie', ''))
try:
action = data['action']
node_id = data['nid']
provider_name = data['provider']
except KeyError:
raise HTTPError(httplib.BAD_REQUEST)
node = AbstractNode.load(node_id) or Preprint.load(node_id)
if not node:
raise HTTPError(httplib.NOT_FOUND)
check_access(node, auth, action, cas_resp)
provider_settings = None
if hasattr(node, 'get_addon'):
provider_settings = node.get_addon(provider_name)
if not provider_settings:
raise HTTPError(httplib.BAD_REQUEST)
path = data.get('path')
credentials = None
waterbutler_settings = None
fileversion = None
if provider_name == 'osfstorage':
if path:
file_id = path.strip('/')
# check to see if this is a file or a folder
filenode = OsfStorageFileNode.load(path.strip('/'))
if filenode and filenode.is_file:
# default to most recent version if none is provided in the response
version = int(data['version']) if data.get('version') else filenode.versions.count()
try:
fileversion = FileVersion.objects.filter(
basefilenode___id=file_id,
identifier=version
).select_related('region').get()
except FileVersion.DoesNotExist:
raise HTTPError(httplib.BAD_REQUEST)
if auth.user:
# mark fileversion as seen
FileVersionUserMetadata.objects.get_or_create(user=auth.user, file_version=fileversion)
if not node.is_contributor(auth.user):
from_mfr = download_is_from_mfr(request, payload=data)
# version index is 0 based
version_index = version - 1
if action == 'render':
update_analytics(node, file_id, version_index, 'view')
elif action == 'download' and not from_mfr:
update_analytics(node, file_id, version_index, 'download')
if waffle.switch_is_active(features.ELASTICSEARCH_METRICS):
if isinstance(node, Preprint):
metric_class = get_metric_class_for_action(action, from_mfr=from_mfr)
if metric_class:
try:
metric_class.record_for_preprint(
preprint=node,
user=auth.user,
version=fileversion.identifier if fileversion else None,
path=path
)
except es_exceptions.ConnectionError:
log_exception()
if fileversion and provider_settings:
region = fileversion.region
credentials = region.waterbutler_credentials
waterbutler_settings = fileversion.serialize_waterbutler_settings(
node_id=provider_settings.owner._id,
root_id=provider_settings.root_node._id,
)
# If they haven't been set by version region, use the NodeSettings or Preprint directly
if not (credentials and waterbutler_settings):
credentials = node.serialize_waterbutler_credentials(provider_name)
waterbutler_settings = node.serialize_waterbutler_settings(provider_name)
return {'payload': jwe.encrypt(jwt.encode({
'exp': timezone.now() + datetime.timedelta(seconds=settings.WATERBUTLER_JWT_EXPIRATION),
'data': {
'auth': make_auth(auth.user), # A waterbutler auth dict not an Auth object
'credentials': credentials,
'settings': waterbutler_settings,
'callback_url': node.api_url_for(
('create_waterbutler_log' if not getattr(node, 'is_registration', False) else 'registration_callbacks'),
_absolute=True,
_internal=True
)
}
}, settings.WATERBUTLER_JWT_SECRET, algorithm=settings.WATERBUTLER_JWT_ALGORITHM), WATERBUTLER_JWE_KEY)}
