def datafeed_image():
try:
# Get parameters
image_id = parse_long(request.args.get('id', ''))
dt_time_from = parse_iso_datetime(request.args.get('from'))
dt_time_to = parse_iso_datetime(request.args.get('to'))
data_type = parse_int(request.args.get('data_type', ''))
require_full_period = (data_type < 8)
if require_full_period:
# Stop at 'now' minus the stats gap so we don't return incomplete stats
dt_time_limit = datetime.utcnow() - timedelta(
minutes=app.config['STATS_FREQUENCY'])
if dt_time_to > dt_time_limit:
dt_time_to = dt_time_limit
# Get stats and convert to chart data
results = data_engine.search_image_stats(dt_time_from, dt_time_to,
image_id)
results = add_zero_stats(dt_time_from, dt_time_to,
app.config['STATS_FREQUENCY'], results,
ImageStats)
data = _db_results_to_flot_data(results, data_type)
return make_json_response(200,
data=data,
first=0 if len(data) == 0 else data[0][0],
last=0 if len(data) == 0 else data[-1][0])
except Exception as e:
if not log_security_error(e, request):
logger.error('Error reading image stats: ' + str(e))
if app.config['DEBUG']:
raise
return make_json_response(200, data=[], first=0, last=0)
def datafeed_image():
try:
# Get parameters
image_id = parse_long(request.args.get('id', ''))
dt_time_from = parse_iso_datetime(request.args.get('from'))
dt_time_to = parse_iso_datetime(request.args.get('to'))
data_type = parse_int(request.args.get('data_type', ''))
require_full_period = (data_type < 8)
if require_full_period:
# Stop at 'now' minus the stats gap so we don't return incomplete stats
dt_time_limit = datetime.utcnow() - timedelta(minutes=app.config['STATS_FREQUENCY'])
if dt_time_to > dt_time_limit:
dt_time_to = dt_time_limit
# Get stats and convert to chart data
results = data_engine.search_image_stats(dt_time_from, dt_time_to, image_id)
results = add_zero_stats(
dt_time_from, dt_time_to,
app.config['STATS_FREQUENCY'], results, ImageStats
)
data = _db_results_to_flot_data(results, data_type)
return make_json_response(
200,
data=data,
first=0 if len(data) == 0 else data[0][0],
last=0 if len(data) == 0 else data[-1][0]
)
except Exception as e:
if not log_security_error(e, request):
logger.error('Error reading image stats: ' + str(e))
if app.config['DEBUG']:
raise
return make_json_response(
200,
data=[],
first=0,
last=0
)
def test_stats_engine(self):
# Clear stats - wait for previous stats to flush then delete all
time.sleep(65)
dm.delete_system_stats(datetime.utcnow())
dm.delete_image_stats(datetime.utcnow())
cm.clear()
# Test constants
IMG = 'test_images/cathedral.jpg'
IMG_COPY = 'test_images/stats_test_image.jpg'
IMG_LEN = 648496 # knowing length requires 'keep original' values in settings
IMG_VIEWS = 5
IMG_VIEWS_NOSTATS = 3
IMG_VIEWS_304 = 8
IMG_VIEWS_COPY = 1
IMG_DOWNLOADS = 1
try:
t_then = datetime.utcnow()
copy_file(IMG, IMG_COPY)
# View some images
for _ in range(IMG_VIEWS):
rv = self.app.get('/image?src='+IMG)
self.assertEqual(rv.status_code, 200)
self.assertEqual(len(rv.data), IMG_LEN)
for _ in range(IMG_DOWNLOADS):
rv = self.app.get('/original?src='+IMG)
self.assertEqual(rv.status_code, 200)
self.assertEqual(len(rv.data), IMG_LEN)
# View some also without stats.
# They should still be counted in the system stats but not the image stats.
for _ in range(IMG_VIEWS_NOSTATS):
rv = self.app.get('/image?src='+IMG+'&stats=0')
self.assertEqual(rv.status_code, 200)
self.assertEqual(len(rv.data), IMG_LEN)
etag = rv.headers['ETag']
# View some that only elicit the 304 Not Modified response
for _ in range(IMG_VIEWS_304):
rv = self.app.get('/image?src='+IMG, headers={'If-None-Match': etag})
self.assertEqual(rv.status_code, 304)
# View an image that we'll delete next
for _ in range(IMG_VIEWS_COPY):
rv = self.app.get('/image?src='+IMG_COPY)
self.assertEqual(len(rv.data), IMG_LEN)
# Get test image db record
db_image = dm.get_image(src=IMG)
self.assertIsNotNone(db_image)
# Deleting the copied image should mean its views are counted in the system stats
# but are not included in the image stats (because the image record is gone)
delete_file(IMG_COPY)
dm.delete_image(dm.get_image(src=IMG_COPY), purge=True)
db_image_copy = dm.get_image(src=IMG_COPY)
self.assertIsNone(db_image_copy)
# Wait for new stats to flush
time.sleep(65)
# See if the system stats line up with the views
t_now = datetime.utcnow()
lres = dm.search_system_stats(t_then, t_now)
self.assertEqual(len(lres), 1)
res = lres[0]
self.assertEqual(res.requests, IMG_VIEWS + IMG_VIEWS_COPY + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS + IMG_VIEWS_304)
self.assertEqual(res.views, IMG_VIEWS + IMG_VIEWS_COPY + IMG_VIEWS_NOSTATS)
self.assertEqual(res.cached_views, IMG_VIEWS + IMG_VIEWS_NOSTATS - 1)
self.assertEqual(res.downloads, IMG_DOWNLOADS)
self.assertEqual(res.total_bytes, (IMG_VIEWS + IMG_VIEWS_COPY + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS) * IMG_LEN)
self.assertGreater(res.request_seconds, 0)
self.assertGreater(res.max_request_seconds, 0)
self.assertLess(res.max_request_seconds, res.request_seconds)
self.assertGreater(res.cpu_pc, 0)
self.assertGreater(res.memory_pc, 0)
# See if the image stats line up with the views
lres = dm.search_image_stats(t_then, t_now, db_image.id)
self.assertEqual(len(lres), 1)
res = lres[0]
self.assertEqual(res.requests, IMG_VIEWS + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS + IMG_VIEWS_304)
self.assertEqual(res.views, IMG_VIEWS)
self.assertEqual(res.cached_views, IMG_VIEWS - 1)
self.assertEqual(res.downloads, IMG_DOWNLOADS)
self.assertEqual(res.total_bytes, (IMG_VIEWS + IMG_DOWNLOADS) * IMG_LEN)
self.assertGreater(res.request_seconds, 0)
self.assertGreater(res.max_request_seconds, 0)
self.assertLess(res.max_request_seconds, res.request_seconds)
# And the summary (reporting) data too
lsummary = dm.summarise_image_stats(t_then, t_now)
# lsummary [(image_id, sum_requests, sum_views, sum_cached_views,
# sum_downloads, sum_bytes_served, sum_seconds, max_seconds)]
self.assertEqual(len(lsummary), 1)
res = lsummary[0]
self.assertEqual(res[0], db_image.id)
self.assertEqual(res[1], IMG_VIEWS + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS + IMG_VIEWS_304)
self.assertEqual(res[2], IMG_VIEWS)
self.assertEqual(res[3], IMG_VIEWS - 1)
self.assertEqual(res[4], IMG_DOWNLOADS)
self.assertEqual(res[5], (IMG_VIEWS + IMG_DOWNLOADS) * IMG_LEN)
self.assertGreater(res[6], 0)
self.assertGreater(res[7], 0)
self.assertLess(res[7], res[6])
ssummary = dm.summarise_system_stats(t_then, t_now)
# ssummary (sum_requests, sum_views, sum_cached_views,
# sum_downloads, sum_bytes_served, sum_seconds, max_seconds)
res = ssummary
self.assertEqual(res[0], IMG_VIEWS + IMG_VIEWS_COPY + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS + IMG_VIEWS_304)
self.assertEqual(res[1], IMG_VIEWS + IMG_VIEWS_COPY + IMG_VIEWS_NOSTATS)
self.assertEqual(res[2], IMG_VIEWS + IMG_VIEWS_NOSTATS - 1)
self.assertEqual(res[3], IMG_DOWNLOADS)
self.assertEqual(res[4], (IMG_VIEWS + IMG_VIEWS_COPY + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS) * IMG_LEN)
self.assertGreater(res[5], 0)
self.assertGreater(res[6], 0)
self.assertLess(res[6], res[5])
finally:
delete_file(IMG_COPY)
