def topten(): # Get parameters days = request.args.get('days', '1') limit = request.args.get('number', '10') data_type = request.args.get('data_type', '2') try: results = [] db_session = data_engine.db_get_session() try: # Convert params to ints days = parse_int(days) limit = parse_int(limit) data_type = parse_int(data_type) # Set options if days < 1: days = 1 if days > 30: days = 30 if limit < 10: limit = 10 if limit > 100: limit = 100 if data_type == 1: order = '-total_requests' elif data_type == 2: order = '-total_views' elif data_type == 3: order = '-total_cached_views' elif data_type == 4: order = '-total_downloads' elif data_type == 5: order = '-total_bytes' elif data_type == 6: order = '-total_seconds' elif data_type == 7: order = '-max_seconds' else: raise ValueError('Invalid data_type %d' % data_type) # Get initial stats top_stats = data_engine.summarise_image_stats( datetime.utcnow() - timedelta(days=days), datetime.utcnow(), limit=limit, order_by=order, _db_session=db_session) # Convert stats list to an image list for result in top_stats: db_image = data_engine.get_image(image_id=result[0], _db_session=db_session) if db_image: results.append({ 'id': db_image.id, 'src': db_image.src, 'requests': result[1], 'views': result[2], 'cached_views': result[3], 'downloads': result[4], 'bytes': result[5], 'seconds': result[6], 'max_seconds': result[7] }) finally: db_session.close() return render_template('reports_topten.html', days=days, data_type=data_type, number=limit, results=results) except Exception as e: log_security_error(e, request) if app.config['DEBUG']: raise raise InternalServerError(safe_error_str(e))
def test_stats_engine(self): # Clear stats - wait for previous stats to flush then delete all time.sleep(65) dm.delete_system_stats(datetime.utcnow()) dm.delete_image_stats(datetime.utcnow()) cm.clear() # Test constants IMG = 'test_images/cathedral.jpg' IMG_COPY = 'test_images/stats_test_image.jpg' IMG_LEN = 648496 # knowing length requires 'keep original' values in settings IMG_VIEWS = 5 IMG_VIEWS_NOSTATS = 3 IMG_VIEWS_304 = 8 IMG_VIEWS_COPY = 1 IMG_DOWNLOADS = 1 try: t_then = datetime.utcnow() copy_file(IMG, IMG_COPY) # View some images for _ in range(IMG_VIEWS): rv = self.app.get('/image?src='+IMG) self.assertEqual(rv.status_code, 200) self.assertEqual(len(rv.data), IMG_LEN) for _ in range(IMG_DOWNLOADS): rv = self.app.get('/original?src='+IMG) self.assertEqual(rv.status_code, 200) self.assertEqual(len(rv.data), IMG_LEN) # View some also without stats. # They should still be counted in the system stats but not the image stats. for _ in range(IMG_VIEWS_NOSTATS): rv = self.app.get('/image?src='+IMG+'&stats=0') self.assertEqual(rv.status_code, 200) self.assertEqual(len(rv.data), IMG_LEN) etag = rv.headers['ETag'] # View some that only elicit the 304 Not Modified response for _ in range(IMG_VIEWS_304): rv = self.app.get('/image?src='+IMG, headers={'If-None-Match': etag}) self.assertEqual(rv.status_code, 304) # View an image that we'll delete next for _ in range(IMG_VIEWS_COPY): rv = self.app.get('/image?src='+IMG_COPY) self.assertEqual(len(rv.data), IMG_LEN) # Get test image db record db_image = dm.get_image(src=IMG) self.assertIsNotNone(db_image) # Deleting the copied image should mean its views are counted in the system stats # but are not included in the image stats (because the image record is gone) delete_file(IMG_COPY) dm.delete_image(dm.get_image(src=IMG_COPY), purge=True) db_image_copy = dm.get_image(src=IMG_COPY) self.assertIsNone(db_image_copy) # Wait for new stats to flush time.sleep(65) # See if the system stats line up with the views t_now = datetime.utcnow() lres = dm.search_system_stats(t_then, t_now) self.assertEqual(len(lres), 1) res = lres[0] self.assertEqual(res.requests, IMG_VIEWS + IMG_VIEWS_COPY + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS + IMG_VIEWS_304) self.assertEqual(res.views, IMG_VIEWS + IMG_VIEWS_COPY + IMG_VIEWS_NOSTATS) self.assertEqual(res.cached_views, IMG_VIEWS + IMG_VIEWS_NOSTATS - 1) self.assertEqual(res.downloads, IMG_DOWNLOADS) self.assertEqual(res.total_bytes, (IMG_VIEWS + IMG_VIEWS_COPY + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS) * IMG_LEN) self.assertGreater(res.request_seconds, 0) self.assertGreater(res.max_request_seconds, 0) self.assertLess(res.max_request_seconds, res.request_seconds) self.assertGreater(res.cpu_pc, 0) self.assertGreater(res.memory_pc, 0) # See if the image stats line up with the views lres = dm.search_image_stats(t_then, t_now, db_image.id) self.assertEqual(len(lres), 1) res = lres[0] self.assertEqual(res.requests, IMG_VIEWS + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS + IMG_VIEWS_304) self.assertEqual(res.views, IMG_VIEWS) self.assertEqual(res.cached_views, IMG_VIEWS - 1) self.assertEqual(res.downloads, IMG_DOWNLOADS) self.assertEqual(res.total_bytes, (IMG_VIEWS + IMG_DOWNLOADS) * IMG_LEN) self.assertGreater(res.request_seconds, 0) self.assertGreater(res.max_request_seconds, 0) self.assertLess(res.max_request_seconds, res.request_seconds) # And the summary (reporting) data too lsummary = dm.summarise_image_stats(t_then, t_now) # lsummary [(image_id, sum_requests, sum_views, sum_cached_views, # sum_downloads, sum_bytes_served, sum_seconds, max_seconds)] self.assertEqual(len(lsummary), 1) res = lsummary[0] self.assertEqual(res[0], db_image.id) self.assertEqual(res[1], IMG_VIEWS + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS + IMG_VIEWS_304) self.assertEqual(res[2], IMG_VIEWS) self.assertEqual(res[3], IMG_VIEWS - 1) self.assertEqual(res[4], IMG_DOWNLOADS) self.assertEqual(res[5], (IMG_VIEWS + IMG_DOWNLOADS) * IMG_LEN) self.assertGreater(res[6], 0) self.assertGreater(res[7], 0) self.assertLess(res[7], res[6]) ssummary = dm.summarise_system_stats(t_then, t_now) # ssummary (sum_requests, sum_views, sum_cached_views, # sum_downloads, sum_bytes_served, sum_seconds, max_seconds) res = ssummary self.assertEqual(res[0], IMG_VIEWS + IMG_VIEWS_COPY + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS + IMG_VIEWS_304) self.assertEqual(res[1], IMG_VIEWS + IMG_VIEWS_COPY + IMG_VIEWS_NOSTATS) self.assertEqual(res[2], IMG_VIEWS + IMG_VIEWS_NOSTATS - 1) self.assertEqual(res[3], IMG_DOWNLOADS) self.assertEqual(res[4], (IMG_VIEWS + IMG_VIEWS_COPY + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS) * IMG_LEN) self.assertGreater(res[5], 0) self.assertGreater(res[6], 0) self.assertLess(res[6], res[5]) finally: delete_file(IMG_COPY)
def topten(): # Get parameters days = request.args.get('days', '1') limit = request.args.get('number', '10') data_type = request.args.get('data_type', '2') try: results = [] db_session = data_engine.db_get_session() try: # Convert params to ints days = parse_int(days) limit = parse_int(limit) data_type = parse_int(data_type) # Set options if days < 1: days = 1 if days > 30: days = 30 if limit < 10: limit = 10 if limit > 100: limit = 100 if data_type == 1: order = '-total_requests' elif data_type == 2: order = '-total_views' elif data_type == 3: order = '-total_cached_views' elif data_type == 4: order = '-total_downloads' elif data_type == 5: order = '-total_bytes' elif data_type == 6: order = '-total_seconds' elif data_type == 7: order = '-max_seconds' else: raise ValueError('Invalid data_type %d' % data_type) # Get initial stats top_stats = data_engine.summarise_image_stats( datetime.utcnow() - timedelta(days=days), datetime.utcnow(), limit=limit, order_by=order, _db_session=db_session ) # Convert stats list to an image list for result in top_stats: db_image = data_engine.get_image(image_id=result[0], _db_session=db_session) if db_image: results.append({ 'id': db_image.id, 'src': db_image.src, 'requests': result[1], 'views': result[2], 'cached_views': result[3], 'downloads': result[4], 'bytes': result[5], 'seconds': result[6], 'max_seconds': result[7] }) finally: db_session.close() return render_template( 'reports_topten.html', days=days, data_type=data_type, number=limit, results=results ) except Exception as e: log_security_error(e, request) if app.config['DEBUG']: raise raise InternalServerError(str(e))