def osfstorage_download(file_node, payload, node_addon, **kwargs): # Set user ID in session data for checking if user is contributor # to project. user_id = payload.get('user') if user_id: current_session = get_session() current_session.data['auth_user_id'] = user_id current_session.save() if not request.args.get('version'): version_id = None else: try: version_id = int(request.args['version']) except ValueError: raise make_error(httplib.BAD_REQUEST, message_short='Version must be an integer if not specified') version = file_node.get_version(version_id, required=True) if request.args.get('mode') not in ('render', ): utils.update_analytics(node_addon.owner, file_node._id, int(version.identifier) - 1) return { 'data': { 'name': file_node.name, 'path': version.location_hash, }, 'settings': { osf_storage_settings.WATERBUTLER_RESOURCE: version.location[osf_storage_settings.WATERBUTLER_RESOURCE], }, }
def test_migrate_download_counts(self, mock_session): names = [] for index, num in enumerate(range(10)): names.append('DEAR GOD$! ({})^ CARPNADOS'.format(num)) fobj, _ = oldels.OsfStorageFileRecord.get_or_create( names[-1], self.node_settings) for _id in range(index): fobj.create_version( self.user, { 'folder': '', 'bucket': '', 'service': 'buttfiles', 'object': '{}{}'.format(index, _id), }) utils.update_analytics(self.project, fobj.path, _id + 1) assert len(fobj.versions) == index assert fobj.get_download_count() == index assert len(self.node_settings.file_tree.children) == 10 migration.migrate_node_settings(self.node_settings, dry=False) migration.migrate_children(self.node_settings, dry=False) for index, child in enumerate(self.node_settings.root_node.children): assert len(child.versions) == index assert child.get_download_count() == index for _id in range(index): assert child.get_download_count(_id) == 1
def osfstorage_download(file_node, payload, node_addon, **kwargs): # Set user ID in session data for checking if user is contributor # to project. user_id = payload.get('user') if user_id: current_session = get_session() current_session.data['auth_user_id'] = user_id if not request.args.get('version'): version_id = None else: try: version_id = int(request.args['version']) except ValueError: raise make_error( httplib.BAD_REQUEST, message_short='Version must be an integer if not specified') version = file_node.get_version(version_id, required=True) if request.args.get('mode') not in ('render', ): utils.update_analytics(node_addon.owner, file_node._id, int(version.identifier) - 1) return { 'data': { 'name': file_node.name, 'path': version.location_hash, }, 'settings': { osf_storage_settings.WATERBUTLER_RESOURCE: version.location[osf_storage_settings.WATERBUTLER_RESOURCE], }, }
def test_migrate_download_counts(self, mock_session): names = [] for index, num in enumerate(range(10)): names.append('DEAR GOD$! ({})^ CARPNADOS'.format(num)) fobj, _ = oldels.OsfStorageFileRecord.get_or_create(names[-1], self.node_settings) for _id in range(index): fobj.create_version(self.user, { 'folder': '', 'bucket': '', 'service': 'buttfiles', 'object': '{}{}'.format(index, _id), }) utils.update_analytics(self.project, fobj.path, _id + 1) assert len(fobj.versions) == index assert fobj.get_download_count() == index assert len(self.node_settings.file_tree.children) == 10 migration.migrate_node_settings(self.node_settings, dry=False) migration.migrate_children(self.node_settings, dry=False) for index, child in enumerate(self.node_settings.root_node.children): assert len(child.versions) == index assert child.get_download_count() == index for _id in range(index): assert child.get_download_count(_id) == 1
def test_download_count(self): # Keen does not allow same day requests so we have to do some time traveling to my birthday with mock.patch('django.utils.timezone.now', return_value=datetime.datetime(1991, 9, 25).replace(tzinfo=pytz.utc)): node = ProjectFactory() utils.update_analytics(node, 'fake id', {'contributors': node.contributors}) query_date = datetime.date(1991, 9, 25) event = DownloadCountSummary().get_events(query_date) assert event[0]['files']['total'] == 1
def test_download_count(self): # Keen does not allow same day requests so we have to do some time traveling to my birthday with mock.patch('django.utils.timezone.now', return_value=datetime.datetime(1991, 9, 25).replace(tzinfo=pytz.utc)): node = ProjectFactory() file = api_utils.create_test_file( node, node.creator, filename='file_one') utils.update_analytics(node, file, 0, 'download') query_date = datetime.date(1991, 9, 25) event = DownloadCountSummary().get_events(query_date) assert event[0]['files']['total'] == 1
def test_download_count(self): # Keen does not allow same day requests so we have to do some time traveling to my birthday with mock.patch('django.utils.timezone.now', return_value=datetime.datetime( 1991, 9, 25).replace(tzinfo=pytz.utc)): node = ProjectFactory() utils.update_analytics(node, 'fake id', {'contributors': node.contributors}) query_date = datetime.date(1991, 9, 25) event = DownloadCountSummary().get_events(query_date) assert event[0]['files']['total'] == 1
def test_serialize_revision(self): sessions.sessions[request._get_current_object()] = Session() utils.update_analytics(self.project, self.record, 0) utils.update_analytics(self.project, self.record, 0) utils.update_analytics(self.project, self.record, 2) expected = { 'index': 1, 'user': { 'name': self.user.fullname, 'url': self.user.url, }, 'date': self.versions[0].created.isoformat(), 'downloads': 2, 'md5': None, 'sha256': None, } observed = utils.serialize_revision( self.project, self.record, self.versions[0], 0, ) assert_equal(expected, observed) assert_equal(self.record.get_download_count(), 3) assert_equal(self.record.get_download_count(version=2), 1) assert_equal(self.record.get_download_count(version=0), 2)
def test_serialize_revision(self): sessions.sessions[request._get_current_object()] = Session() utils.update_analytics(self.project, self.record._id, 0) utils.update_analytics(self.project, self.record._id, 0) utils.update_analytics(self.project, self.record._id, 2) expected = { 'index': 1, 'user': { 'name': self.user.fullname, 'url': self.user.url, }, 'date': self.versions[0].created.isoformat(), 'downloads': 2, 'md5': None, 'sha256': None, } observed = utils.serialize_revision( self.project, self.record, self.versions[0], 0, ) assert_equal(expected, observed) assert_equal(self.record.get_download_count(), 3) assert_equal(self.record.get_download_count(version=2), 1) assert_equal(self.record.get_download_count(version=0), 2)
def test_download_count_file(self, mock_session): mock_session.data = {} child = self.node_settings.get_root().append_file('Test') utils.update_analytics(self.project, child._id, 0) utils.update_analytics(self.project, child._id, 1) utils.update_analytics(self.project, child._id, 2) assert_equals(child.get_download_count(), 3) assert_equals(child.get_download_count(0), 1) assert_equals(child.get_download_count(1), 1) assert_equals(child.get_download_count(2), 1)
def test_anon_revisions(self): sessions.sessions[request._get_current_object()] = Session() utils.update_analytics(self.project, self.record, 0) utils.update_analytics(self.project, self.record, 0) utils.update_analytics(self.project, self.record, 2) expected = { 'index': 2, 'user': None, 'date': self.versions[0].created.isoformat(), 'downloads': 0, 'md5': None, 'sha256': None, } observed = utils.serialize_revision(self.project, self.record, self.versions[0], 1, anon=True) assert_equal(expected, observed)
def test_anon_revisions(self): sessions.sessions[request._get_current_object()] = Session() utils.update_analytics(self.project, self.record._id, 0) utils.update_analytics(self.project, self.record._id, 0) utils.update_analytics(self.project, self.record._id, 2) expected = { 'index': 2, 'user': None, 'date': self.versions[0].created.isoformat(), 'downloads': 0, 'md5': None, 'sha256': None, } observed = utils.serialize_revision( self.project, self.record, self.versions[0], 1, anon=True ) assert_equal(expected, observed)
def get_auth(auth, **kwargs): cas_resp = None if not auth.user: # Central Authentication Server OAuth Bearer Token authorization = request.headers.get('Authorization') if authorization and authorization.startswith('Bearer '): client = cas.get_client() try: access_token = cas.parse_auth_header(authorization) cas_resp = client.profile(access_token) except cas.CasError as err: sentry.log_exception() # NOTE: We assume that the request is an AJAX request return json_renderer(err) if cas_resp.authenticated: auth.user = OSFUser.load(cas_resp.user) try: data = jwt.decode(jwe.decrypt( request.args.get('payload', '').encode('utf-8'), WATERBUTLER_JWE_KEY), settings.WATERBUTLER_JWT_SECRET, options={'require_exp': True}, algorithm=settings.WATERBUTLER_JWT_ALGORITHM)['data'] except (jwt.InvalidTokenError, KeyError) as err: sentry.log_message(str(err)) raise HTTPError(http_status.HTTP_403_FORBIDDEN) if not auth.user: auth.user = OSFUser.from_cookie(data.get('cookie', '')) try: action = data['action'] node_id = data['nid'] provider_name = data['provider'] except KeyError: raise HTTPError(http_status.HTTP_400_BAD_REQUEST) node = AbstractNode.load(node_id) or Preprint.load(node_id) if node and node.is_deleted: raise HTTPError(http_status.HTTP_410_GONE) elif not node: raise HTTPError(http_status.HTTP_404_NOT_FOUND) check_access(node, auth, action, cas_resp) provider_settings = None if hasattr(node, 'get_addon'): provider_settings = node.get_addon(provider_name) if not provider_settings: raise HTTPError(http_status.HTTP_400_BAD_REQUEST) path = data.get('path') credentials = None waterbutler_settings = None fileversion = None if provider_name == 'osfstorage': if path: file_id = path.strip('/') # check to see if this is a file or a folder filenode = OsfStorageFileNode.load(path.strip('/')) if filenode and filenode.is_file: # default to most recent version if none is provided in the response version = int(data['version']) if data.get( 'version') else filenode.versions.count() try: fileversion = FileVersion.objects.filter( basefilenode___id=file_id, identifier=version).select_related('region').get() except FileVersion.DoesNotExist: raise HTTPError(http_status.HTTP_400_BAD_REQUEST) if auth.user: # mark fileversion as seen FileVersionUserMetadata.objects.get_or_create( user=auth.user, file_version=fileversion) if not node.is_contributor_or_group_member(auth.user): from_mfr = download_is_from_mfr(request, payload=data) # version index is 0 based version_index = version - 1 if action == 'render': update_analytics(node, filenode, version_index, 'view') elif action == 'download' and not from_mfr: update_analytics(node, filenode, version_index, 'download') if waffle.switch_is_active(features.ELASTICSEARCH_METRICS): if isinstance(node, Preprint): metric_class = get_metric_class_for_action( action, from_mfr=from_mfr) if metric_class: sloan_flags = { 'sloan_id': request.cookies.get(SLOAN_ID_COOKIE_NAME) } for flag_name in SLOAN_FLAGS: value = request.cookies.get( f'dwf_{flag_name}_custom_domain' ) or request.cookies.get( f'dwf_{flag_name}') if value: sloan_flags[flag_name.replace( '_display', '')] = strtobool(value) try: metric_class.record_for_preprint( preprint=node, user=auth.user, version=fileversion.identifier if fileversion else None, path=path, **sloan_flags) except es_exceptions.ConnectionError: log_exception() if fileversion and provider_settings: region = fileversion.region credentials = region.waterbutler_credentials waterbutler_settings = fileversion.serialize_waterbutler_settings( node_id=provider_settings.owner._id, root_id=provider_settings.root_node._id, ) # If they haven't been set by version region, use the NodeSettings or Preprint directly if not (credentials and waterbutler_settings): credentials = node.serialize_waterbutler_credentials(provider_name) waterbutler_settings = node.serialize_waterbutler_settings( provider_name) if isinstance(credentials.get('token'), bytes): credentials['token'] = credentials.get('token').decode() return { 'payload': jwe.encrypt( jwt.encode( { 'exp': timezone.now() + datetime.timedelta( seconds=settings.WATERBUTLER_JWT_EXPIRATION), 'data': { 'auth': make_auth( auth.user ), # A waterbutler auth dict not an Auth object 'credentials': credentials, 'settings': waterbutler_settings, 'callback_url': node.api_url_for( ('create_waterbutler_log' if not getattr(node, 'is_registration', False) else 'registration_callbacks'), _absolute=True, _internal=True) } }, settings.WATERBUTLER_JWT_SECRET, algorithm=settings.WATERBUTLER_JWT_ALGORITHM), WATERBUTLER_JWE_KEY).decode() }
def get_auth(auth, **kwargs): cas_resp = None if not auth.user: # Central Authentication Server OAuth Bearer Token authorization = request.headers.get('Authorization') if authorization and authorization.startswith('Bearer '): client = cas.get_client() try: access_token = cas.parse_auth_header(authorization) cas_resp = client.profile(access_token) except cas.CasError as err: sentry.log_exception() # NOTE: We assume that the request is an AJAX request return json_renderer(err) if cas_resp.authenticated: auth.user = OSFUser.load(cas_resp.user) try: data = jwt.decode( jwe.decrypt(request.args.get('payload', '').encode('utf-8'), WATERBUTLER_JWE_KEY), settings.WATERBUTLER_JWT_SECRET, options={'require_exp': True}, algorithm=settings.WATERBUTLER_JWT_ALGORITHM )['data'] except (jwt.InvalidTokenError, KeyError) as err: sentry.log_message(str(err)) raise HTTPError(httplib.FORBIDDEN) if not auth.user: auth.user = OSFUser.from_cookie(data.get('cookie', '')) try: action = data['action'] node_id = data['nid'] provider_name = data['provider'] except KeyError: raise HTTPError(httplib.BAD_REQUEST) node = AbstractNode.load(node_id) or Preprint.load(node_id) if not node: raise HTTPError(httplib.NOT_FOUND) check_access(node, auth, action, cas_resp) provider_settings = None if hasattr(node, 'get_addon'): provider_settings = node.get_addon(provider_name) if not provider_settings: raise HTTPError(httplib.BAD_REQUEST) path = data.get('path') credentials = None waterbutler_settings = None fileversion = None if provider_name == 'osfstorage': if path: file_id = path.strip('/') # check to see if this is a file or a folder filenode = OsfStorageFileNode.load(path.strip('/')) if filenode and filenode.is_file: # default to most recent version if none is provided in the response version = int(data['version']) if data.get('version') else filenode.versions.count() try: fileversion = FileVersion.objects.filter( basefilenode___id=file_id, identifier=version ).select_related('region').get() except FileVersion.DoesNotExist: raise HTTPError(httplib.BAD_REQUEST) if auth.user: # mark fileversion as seen FileVersionUserMetadata.objects.get_or_create(user=auth.user, file_version=fileversion) if not node.is_contributor(auth.user): from_mfr = download_is_from_mfr(request, payload=data) # version index is 0 based version_index = version - 1 if action == 'render': update_analytics(node, file_id, version_index, 'view') elif action == 'download' and not from_mfr: update_analytics(node, file_id, version_index, 'download') if waffle.switch_is_active(features.ELASTICSEARCH_METRICS): if isinstance(node, Preprint): metric_class = get_metric_class_for_action(action, from_mfr=from_mfr) if metric_class: try: metric_class.record_for_preprint( preprint=node, user=auth.user, version=fileversion.identifier if fileversion else None, path=path ) except es_exceptions.ConnectionError: log_exception() if fileversion and provider_settings: region = fileversion.region credentials = region.waterbutler_credentials waterbutler_settings = fileversion.serialize_waterbutler_settings( node_id=provider_settings.owner._id, root_id=provider_settings.root_node._id, ) # If they haven't been set by version region, use the NodeSettings or Preprint directly if not (credentials and waterbutler_settings): credentials = node.serialize_waterbutler_credentials(provider_name) waterbutler_settings = node.serialize_waterbutler_settings(provider_name) return {'payload': jwe.encrypt(jwt.encode({ 'exp': timezone.now() + datetime.timedelta(seconds=settings.WATERBUTLER_JWT_EXPIRATION), 'data': { 'auth': make_auth(auth.user), # A waterbutler auth dict not an Auth object 'credentials': credentials, 'settings': waterbutler_settings, 'callback_url': node.api_url_for( ('create_waterbutler_log' if not getattr(node, 'is_registration', False) else 'registration_callbacks'), _absolute=True, _internal=True ) } }, settings.WATERBUTLER_JWT_SECRET, algorithm=settings.WATERBUTLER_JWT_ALGORITHM), WATERBUTLER_JWE_KEY)}