def test_task_server(self): # Create some stats t_now = datetime.utcnow() dm.save_object(SystemStats( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, t_now - timedelta(minutes=5), t_now )) # We should now have stats sys_stats = dm.search_system_stats(t_now - timedelta(minutes=60), t_now) self.assertGreater(len(sys_stats), 0) # Check current task list entries tasks = dm.list_objects(Task, order_field=Task.id) task_list_len = len(tasks) # Post a background task to purge the stats KEEP_SECS = 10 task_obj = tm.add_task( None, 'Purge system statistics', 'purge_system_stats', {'before_time': t_now}, Task.PRIORITY_NORMAL, 'info', 'error', KEEP_SECS ) self.assertIsNotNone(task_obj) # Check it really got added tasks = dm.list_objects(Task, order_field=Task.id) self.assertEqual(len(tasks), task_list_len + 1) task_list_len = len(tasks) # Post it again, make sure there is no dupe added dupe_task_obj = tm.add_task( None, 'Purge system statistics', 'purge_system_stats', {'before_time': t_now}, Task.PRIORITY_NORMAL, 'info', 'error', KEEP_SECS ) self.assertIsNone(dupe_task_obj) # Check it really didn't get re-added tasks = dm.list_objects(Task, order_field=Task.id) self.assertEqual(len(tasks), task_list_len) task = tasks[-1] self.assertEqual(task.id, task_obj.id) # Wait for task completion tm.wait_for_task(task_obj.id, 20) # We should now have no stats t_now = datetime.utcnow() sys_stats = dm.search_system_stats(t_now - timedelta(minutes=60), t_now) self.assertEqual(len(sys_stats), 0) # The completed task should only be removed after the delay we specified task = dm.get_object(Task, task_obj.id) self.assertIsNotNone(task) self.assertEqual(task.status, Task.STATUS_COMPLETE) # Wait for keep time + task server's poll time time.sleep(KEEP_SECS + 10) # Should now be gone task = dm.get_object(Task, task_obj.id) self.assertIsNone(task)
def test_task_server(self): # Create some stats t_now = datetime.utcnow() dm.save_object( SystemStats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, t_now - timedelta(minutes=5), t_now)) # We should now have stats sys_stats = dm.search_system_stats(t_now - timedelta(minutes=60), t_now) self.assertGreater(len(sys_stats), 0) # Check current task list entries tasks = dm.list_objects(Task, order_field=Task.id) task_list_len = len(tasks) # Post a background task to purge the stats KEEP_SECS = 10 task_obj = tm.add_task(None, 'Purge system statistics', 'purge_system_stats', {'before_time': t_now}, Task.PRIORITY_NORMAL, 'info', 'error', KEEP_SECS) self.assertIsNotNone(task_obj) # Check it really got added tasks = dm.list_objects(Task, order_field=Task.id) self.assertEqual(len(tasks), task_list_len + 1) task_list_len = len(tasks) # Post it again, make sure there is no dupe added dupe_task_obj = tm.add_task(None, 'Purge system statistics', 'purge_system_stats', {'before_time': t_now}, Task.PRIORITY_NORMAL, 'info', 'error', KEEP_SECS) self.assertIsNone(dupe_task_obj) # Check it really didn't get re-added tasks = dm.list_objects(Task, order_field=Task.id) self.assertEqual(len(tasks), task_list_len) task = tasks[-1] self.assertEqual(task.id, task_obj.id) # Wait for task completion tm.wait_for_task(task_obj.id, 10) # We should now have no stats t_now = datetime.utcnow() sys_stats = dm.search_system_stats(t_now - timedelta(minutes=60), t_now) self.assertEqual(len(sys_stats), 0) # The completed task should only be removed after the delay we specified task = dm.get_object(Task, task_obj.id) self.assertIsNotNone(task) self.assertEqual(task.status, Task.STATUS_COMPLETE) # Wait for keep time + task server's poll time time.sleep(KEEP_SECS + 10) # Should now be gone task = dm.get_object(Task, task_obj.id) self.assertIsNone(task)
def datafeed_system(): try: # Get parameters dt_time_from = parse_iso_datetime(request.args.get('from')) dt_time_to = parse_iso_datetime(request.args.get('to')) data_type = parse_int(request.args.get('data_type', '')) require_full_period = (data_type < 8) if require_full_period: # Stop at 'now' minus the stats gap so we don't return incomplete stats dt_time_limit = datetime.utcnow() - timedelta( minutes=app.config['STATS_FREQUENCY']) if dt_time_to > dt_time_limit: dt_time_to = dt_time_limit # Get stats and convert to chart data results = data_engine.search_system_stats(dt_time_from, dt_time_to) results = add_zero_stats(dt_time_from, dt_time_to, app.config['STATS_FREQUENCY'], results, SystemStats) data = _db_results_to_flot_data(results, data_type) return make_json_response(200, data=data, first=0 if len(data) == 0 else data[0][0], last=0 if len(data) == 0 else data[-1][0]) except Exception as e: if not log_security_error(e, request): logger.error('Error reading system stats: ' + str(e)) if app.config['DEBUG']: raise return make_json_response(200, data=[], first=0, last=0)
def datafeed_system(): try: # Get parameters dt_time_from = parse_iso_datetime(request.args.get('from')) dt_time_to = parse_iso_datetime(request.args.get('to')) data_type = parse_int(request.args.get('data_type', '')) require_full_period = (data_type < 8) if require_full_period: # Stop at 'now' minus the stats gap so we don't return incomplete stats dt_time_limit = datetime.utcnow() - timedelta(minutes=app.config['STATS_FREQUENCY']) if dt_time_to > dt_time_limit: dt_time_to = dt_time_limit # Get stats and convert to chart data results = data_engine.search_system_stats(dt_time_from, dt_time_to) results = add_zero_stats( dt_time_from, dt_time_to, app.config['STATS_FREQUENCY'], results, SystemStats ) data = _db_results_to_flot_data(results, data_type) return make_json_response( 200, data=data, first=0 if len(data) == 0 else data[0][0], last=0 if len(data) == 0 else data[-1][0] ) except Exception as e: if not log_security_error(e, request): logger.error('Error reading system stats: ' + str(e)) if app.config['DEBUG']: raise return make_json_response( 200, data=[], first=0, last=0 )
def test_stats_engine(self): # Clear stats - wait for previous stats to flush then delete all time.sleep(65) dm.delete_system_stats(datetime.utcnow()) dm.delete_image_stats(datetime.utcnow()) cm.clear() # Test constants IMG = 'test_images/cathedral.jpg' IMG_COPY = 'test_images/stats_test_image.jpg' IMG_LEN = 648496 # knowing length requires 'keep original' values in settings IMG_VIEWS = 5 IMG_VIEWS_NOSTATS = 3 IMG_VIEWS_304 = 8 IMG_VIEWS_COPY = 1 IMG_DOWNLOADS = 1 try: t_then = datetime.utcnow() copy_file(IMG, IMG_COPY) # View some images for _ in range(IMG_VIEWS): rv = self.app.get('/image?src='+IMG) self.assertEqual(rv.status_code, 200) self.assertEqual(len(rv.data), IMG_LEN) for _ in range(IMG_DOWNLOADS): rv = self.app.get('/original?src='+IMG) self.assertEqual(rv.status_code, 200) self.assertEqual(len(rv.data), IMG_LEN) # View some also without stats. # They should still be counted in the system stats but not the image stats. for _ in range(IMG_VIEWS_NOSTATS): rv = self.app.get('/image?src='+IMG+'&stats=0') self.assertEqual(rv.status_code, 200) self.assertEqual(len(rv.data), IMG_LEN) etag = rv.headers['ETag'] # View some that only elicit the 304 Not Modified response for _ in range(IMG_VIEWS_304): rv = self.app.get('/image?src='+IMG, headers={'If-None-Match': etag}) self.assertEqual(rv.status_code, 304) # View an image that we'll delete next for _ in range(IMG_VIEWS_COPY): rv = self.app.get('/image?src='+IMG_COPY) self.assertEqual(len(rv.data), IMG_LEN) # Get test image db record db_image = dm.get_image(src=IMG) self.assertIsNotNone(db_image) # Deleting the copied image should mean its views are counted in the system stats # but are not included in the image stats (because the image record is gone) delete_file(IMG_COPY) dm.delete_image(dm.get_image(src=IMG_COPY), purge=True) db_image_copy = dm.get_image(src=IMG_COPY) self.assertIsNone(db_image_copy) # Wait for new stats to flush time.sleep(65) # See if the system stats line up with the views t_now = datetime.utcnow() lres = dm.search_system_stats(t_then, t_now) self.assertEqual(len(lres), 1) res = lres[0] self.assertEqual(res.requests, IMG_VIEWS + IMG_VIEWS_COPY + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS + IMG_VIEWS_304) self.assertEqual(res.views, IMG_VIEWS + IMG_VIEWS_COPY + IMG_VIEWS_NOSTATS) self.assertEqual(res.cached_views, IMG_VIEWS + IMG_VIEWS_NOSTATS - 1) self.assertEqual(res.downloads, IMG_DOWNLOADS) self.assertEqual(res.total_bytes, (IMG_VIEWS + IMG_VIEWS_COPY + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS) * IMG_LEN) self.assertGreater(res.request_seconds, 0) self.assertGreater(res.max_request_seconds, 0) self.assertLess(res.max_request_seconds, res.request_seconds) self.assertGreater(res.cpu_pc, 0) self.assertGreater(res.memory_pc, 0) # See if the image stats line up with the views lres = dm.search_image_stats(t_then, t_now, db_image.id) self.assertEqual(len(lres), 1) res = lres[0] self.assertEqual(res.requests, IMG_VIEWS + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS + IMG_VIEWS_304) self.assertEqual(res.views, IMG_VIEWS) self.assertEqual(res.cached_views, IMG_VIEWS - 1) self.assertEqual(res.downloads, IMG_DOWNLOADS) self.assertEqual(res.total_bytes, (IMG_VIEWS + IMG_DOWNLOADS) * IMG_LEN) self.assertGreater(res.request_seconds, 0) self.assertGreater(res.max_request_seconds, 0) self.assertLess(res.max_request_seconds, res.request_seconds) # And the summary (reporting) data too lsummary = dm.summarise_image_stats(t_then, t_now) # lsummary [(image_id, sum_requests, sum_views, sum_cached_views, # sum_downloads, sum_bytes_served, sum_seconds, max_seconds)] self.assertEqual(len(lsummary), 1) res = lsummary[0] self.assertEqual(res[0], db_image.id) self.assertEqual(res[1], IMG_VIEWS + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS + IMG_VIEWS_304) self.assertEqual(res[2], IMG_VIEWS) self.assertEqual(res[3], IMG_VIEWS - 1) self.assertEqual(res[4], IMG_DOWNLOADS) self.assertEqual(res[5], (IMG_VIEWS + IMG_DOWNLOADS) * IMG_LEN) self.assertGreater(res[6], 0) self.assertGreater(res[7], 0) self.assertLess(res[7], res[6]) ssummary = dm.summarise_system_stats(t_then, t_now) # ssummary (sum_requests, sum_views, sum_cached_views, # sum_downloads, sum_bytes_served, sum_seconds, max_seconds) res = ssummary self.assertEqual(res[0], IMG_VIEWS + IMG_VIEWS_COPY + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS + IMG_VIEWS_304) self.assertEqual(res[1], IMG_VIEWS + IMG_VIEWS_COPY + IMG_VIEWS_NOSTATS) self.assertEqual(res[2], IMG_VIEWS + IMG_VIEWS_NOSTATS - 1) self.assertEqual(res[3], IMG_DOWNLOADS) self.assertEqual(res[4], (IMG_VIEWS + IMG_VIEWS_COPY + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS) * IMG_LEN) self.assertGreater(res[5], 0) self.assertGreater(res[6], 0) self.assertLess(res[6], res[5]) finally: delete_file(IMG_COPY)