def image_stats(): # Get parameters image_id = request.args.get('id', '') time_from = request.args.get('from', '') time_to = request.args.get('to', '') data_type = request.args.get('data_type', '1') embed = request.args.get('embed', '') try: # Validate params, get iso and datetime versions if image_id == '': raise ValueError('No image was specified.') image_id = parse_long(image_id) (iso_time_from, iso_time_to) = process_time_parameters(time_from, time_to) dt_time_from = parse_iso_datetime(iso_time_from) dt_time_to = parse_iso_datetime(iso_time_to) embed = parse_boolean(embed) return render_template( 'reports_image_stats.html', timezone=get_timezone_code(), timezone_seconds=get_timezone_offset(), time_from=dt_time_from, time_to=dt_time_to, data_type=data_type, db_image=data_engine.get_image(image_id=image_id), embed=embed) except Exception as e: log_security_error(e, request) if app.config['DEBUG']: raise raise InternalServerError(safe_error_str(e))
def image_stats(): # Get parameters image_id = request.args.get('id', '') time_from = request.args.get('from', '') time_to = request.args.get('to', '') data_type = request.args.get('data_type', '1') embed = request.args.get('embed', '') try: # Validate params, get iso and datetime versions if image_id == '': raise ValueError('No image was specified.') image_id = parse_long(image_id) (iso_time_from, iso_time_to) = process_time_parameters(time_from, time_to) dt_time_from = parse_iso_datetime(iso_time_from) dt_time_to = parse_iso_datetime(iso_time_to) embed = parse_boolean(embed) return render_template( 'reports_image_stats.html', timezone=get_timezone_code(), timezone_seconds=get_timezone_offset(), time_from=dt_time_from, time_to=dt_time_to, data_type=data_type, db_image=data_engine.get_image(image_id=image_id), embed=embed ) except Exception as e: log_security_error(e, request) if app.config['DEBUG']: raise raise InternalServerError(str(e))
def get(self, image_id): db_img = data_engine.get_image(image_id=image_id) if not db_img: raise DoesNotExistError(str(image_id)) else: # Require view permission or file admin permissions_engine.ensure_folder_permitted( db_img.folder, FolderPermission.ACCESS_VIEW, get_session_user()) return make_api_success_response( object_to_dict(_prep_image_object(db_img)))
def put(self, image_id): params = self._get_validated_object_parameters(request.form) # Get image and update it db_img = data_engine.get_image(image_id=image_id) if not db_img: raise DoesNotExistError(str(image_id)) # Require edit permission or file admin permissions_engine.ensure_folder_permitted( db_img.folder, FolderPermission.ACCESS_EDIT, get_session_user() ) old_title = db_img.title old_description = db_img.description db_img.title = params['title'] db_img.description = params['description'] data_engine.save_object(db_img) # Get text changes. Max info length = # 100 + 200 + len('()' + '()' + 'Title: ' + ' / ' + 'Description: ') ==> 327 title_diff = get_string_changes(old_title, params['title'], char_limit=100).strip() if not title_diff: # Try for deletions from title title_diff = get_string_changes(params['title'], old_title, char_limit=100).strip() if title_diff: title_diff = '(' + title_diff + ')' desc_diff = get_string_changes( old_description, params['description'], char_limit=200 ).strip() if not desc_diff: # Try for deletions from description desc_diff = get_string_changes( params['description'], old_description, char_limit=200 ).strip() if desc_diff: desc_diff = '(' + desc_diff + ')' info = '' if title_diff: info += 'Title: ' + title_diff if info and desc_diff: info += ' / ' if desc_diff: info += 'Description: ' + desc_diff # Add change history data_engine.add_image_history( db_img, get_session_user(), ImageHistory.ACTION_EDITED, info ) return make_api_success_response(object_to_dict(db_img))
def get(self, image_id): db_img = data_engine.get_image(image_id=image_id) if not db_img: raise DoesNotExistError(str(image_id)) else: # Require view permission or file admin permissions_engine.ensure_folder_permitted( db_img.folder, FolderPermission.ACCESS_VIEW, get_session_user() ) return make_api_success_response(object_to_dict(db_img))
def put(self, image_id): params = self._get_validated_object_parameters(request.form) # Get image and update it db_img = data_engine.get_image(image_id=image_id) if not db_img: raise DoesNotExistError(str(image_id)) # Require edit permission or file admin permissions_engine.ensure_folder_permitted( db_img.folder, FolderPermission.ACCESS_EDIT, get_session_user()) old_title = db_img.title old_description = db_img.description db_img.title = params['title'] db_img.description = params['description'] data_engine.save_object(db_img) # Get text changes. Max info length = # 100 + 200 + len('()' + '()' + 'Title: ' + ' / ' + 'Description: ') ==> 327 title_diff = get_string_changes(old_title, params['title'], char_limit=100).strip() if not title_diff: # Try for deletions from title title_diff = get_string_changes(params['title'], old_title, char_limit=100).strip() if title_diff: title_diff = '(' + title_diff + ')' desc_diff = get_string_changes(old_description, params['description'], char_limit=200).strip() if not desc_diff: # Try for deletions from description desc_diff = get_string_changes(params['description'], old_description, char_limit=200).strip() if desc_diff: desc_diff = '(' + desc_diff + ')' info = '' if title_diff: info += 'Title: ' + title_diff if info and desc_diff: info += ' / ' if desc_diff: info += 'Description: ' + desc_diff # Add change history data_engine.add_image_history(db_img, get_session_user(), ImageHistory.ACTION_EDITED, info) return make_api_success_response( object_to_dict(_prep_image_object(db_img)))
def post(self, folio_id): db_session = data_engine.db_get_session() try: # Get the portfolio folio = data_engine.get_portfolio(folio_id, _db_session=db_session) if folio is None: raise DoesNotExistError(str(folio_id)) # Check portfolio permissions permissions_engine.ensure_portfolio_permitted( folio, FolioPermission.ACCESS_EDIT, get_session_user()) # Get the image by either ID or src params = self._get_validated_object_parameters(request.form, True) if 'image_id' in params: image = data_engine.get_image(params['image_id'], _db_session=db_session) if image is None: raise DoesNotExistError(str(params['image_id'])) else: image = auto_sync_file(params['image_src'], data_engine, task_engine, anon_history=True, burst_pdf=False, _db_session=db_session) if image is None or image.status == Image.STATUS_DELETED: raise DoesNotExistError(params['image_src']) # Check image permissions permissions_engine.ensure_folder_permitted( image.folder, FolderPermission.ACCESS_VIEW, get_session_user(), False) # Add history first so that we only commit once at the end data_engine.add_portfolio_history(folio, get_session_user(), FolioHistory.ACTION_IMAGE_CHANGE, '%s added' % image.src, _db_session=db_session, _commit=False) # Flag that exported zips are now out of date folio.last_updated = datetime.utcnow() # Add the image and commit changes db_folio_image = data_engine.save_object(FolioImage( folio, image, params['image_parameters'], params['filename'], params['index']), refresh=True, _db_session=db_session, _commit=True) return make_api_success_response( object_to_dict(_prep_folioimage_object(db_folio_image), _omit_fields + ['portfolio'])) finally: db_session.close()
def delete(self, image_id): """ Deletes a file from disk """ # Get image data db_img = data_engine.get_image(image_id=image_id) if not db_img: raise DoesNotExistError(str(image_id)) # v4.1 #10 delete_file() doesn't care whether the file exists, but we # want the API to return a "not found" if the file doesn't exist # (and as long as the database is already in sync with that) if not path_exists( db_img.src, require_file=True) and db_img.status == Image.STATUS_DELETED: raise DoesNotExistError(db_img.src) # Delete db_img = delete_file(db_img, get_session_user(), data_engine, permissions_engine) # Remove cached images for old path image_engine._uncache_image_id(db_img.id) # Return updated image return make_api_success_response( object_to_dict(_prep_image_object(db_img)))
def put(self, image_id): """ Moves or renames a file on disk """ params = self._get_validated_parameters(request.form) # Get image data db_img = data_engine.get_image(image_id=image_id) if not db_img: raise DoesNotExistError(str(image_id)) # Move try: db_img = move_file(db_img, params['path'], get_session_user(), data_engine, permissions_engine) except ValueError as e: if type(e) is ValueError: raise ParameterError(str(e)) else: raise # Sub-classes of ValueError # Remove cached images for the old path image_engine._uncache_image_id(db_img.id) # Return updated image return make_api_success_response( object_to_dict(_prep_image_object(db_img)))
def test_stats_engine(self): # Clear stats - wait for previous stats to flush then delete all time.sleep(65) dm.delete_system_stats(datetime.utcnow()) dm.delete_image_stats(datetime.utcnow()) cm.clear() # Test constants IMG = 'test_images/cathedral.jpg' IMG_COPY = 'test_images/stats_test_image.jpg' IMG_LEN = 648496 # knowing length requires 'keep original' values in settings IMG_VIEWS = 5 IMG_VIEWS_NOSTATS = 3 IMG_VIEWS_304 = 8 IMG_VIEWS_COPY = 1 IMG_DOWNLOADS = 1 try: t_then = datetime.utcnow() copy_file(IMG, IMG_COPY) # View some images for _ in range(IMG_VIEWS): rv = self.app.get('/image?src='+IMG) self.assertEqual(rv.status_code, 200) self.assertEqual(len(rv.data), IMG_LEN) for _ in range(IMG_DOWNLOADS): rv = self.app.get('/original?src='+IMG) self.assertEqual(rv.status_code, 200) self.assertEqual(len(rv.data), IMG_LEN) # View some also without stats. # They should still be counted in the system stats but not the image stats. for _ in range(IMG_VIEWS_NOSTATS): rv = self.app.get('/image?src='+IMG+'&stats=0') self.assertEqual(rv.status_code, 200) self.assertEqual(len(rv.data), IMG_LEN) etag = rv.headers['ETag'] # View some that only elicit the 304 Not Modified response for _ in range(IMG_VIEWS_304): rv = self.app.get('/image?src='+IMG, headers={'If-None-Match': etag}) self.assertEqual(rv.status_code, 304) # View an image that we'll delete next for _ in range(IMG_VIEWS_COPY): rv = self.app.get('/image?src='+IMG_COPY) self.assertEqual(len(rv.data), IMG_LEN) # Get test image db record db_image = dm.get_image(src=IMG) self.assertIsNotNone(db_image) # Deleting the copied image should mean its views are counted in the system stats # but are not included in the image stats (because the image record is gone) delete_file(IMG_COPY) dm.delete_image(dm.get_image(src=IMG_COPY), purge=True) db_image_copy = dm.get_image(src=IMG_COPY) self.assertIsNone(db_image_copy) # Wait for new stats to flush time.sleep(65) # See if the system stats line up with the views t_now = datetime.utcnow() lres = dm.search_system_stats(t_then, t_now) self.assertEqual(len(lres), 1) res = lres[0] self.assertEqual(res.requests, IMG_VIEWS + IMG_VIEWS_COPY + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS + IMG_VIEWS_304) self.assertEqual(res.views, IMG_VIEWS + IMG_VIEWS_COPY + IMG_VIEWS_NOSTATS) self.assertEqual(res.cached_views, IMG_VIEWS + IMG_VIEWS_NOSTATS - 1) self.assertEqual(res.downloads, IMG_DOWNLOADS) self.assertEqual(res.total_bytes, (IMG_VIEWS + IMG_VIEWS_COPY + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS) * IMG_LEN) self.assertGreater(res.request_seconds, 0) self.assertGreater(res.max_request_seconds, 0) self.assertLess(res.max_request_seconds, res.request_seconds) self.assertGreater(res.cpu_pc, 0) self.assertGreater(res.memory_pc, 0) # See if the image stats line up with the views lres = dm.search_image_stats(t_then, t_now, db_image.id) self.assertEqual(len(lres), 1) res = lres[0] self.assertEqual(res.requests, IMG_VIEWS + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS + IMG_VIEWS_304) self.assertEqual(res.views, IMG_VIEWS) self.assertEqual(res.cached_views, IMG_VIEWS - 1) self.assertEqual(res.downloads, IMG_DOWNLOADS) self.assertEqual(res.total_bytes, (IMG_VIEWS + IMG_DOWNLOADS) * IMG_LEN) self.assertGreater(res.request_seconds, 0) self.assertGreater(res.max_request_seconds, 0) self.assertLess(res.max_request_seconds, res.request_seconds) # And the summary (reporting) data too lsummary = dm.summarise_image_stats(t_then, t_now) # lsummary [(image_id, sum_requests, sum_views, sum_cached_views, # sum_downloads, sum_bytes_served, sum_seconds, max_seconds)] self.assertEqual(len(lsummary), 1) res = lsummary[0] self.assertEqual(res[0], db_image.id) self.assertEqual(res[1], IMG_VIEWS + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS + IMG_VIEWS_304) self.assertEqual(res[2], IMG_VIEWS) self.assertEqual(res[3], IMG_VIEWS - 1) self.assertEqual(res[4], IMG_DOWNLOADS) self.assertEqual(res[5], (IMG_VIEWS + IMG_DOWNLOADS) * IMG_LEN) self.assertGreater(res[6], 0) self.assertGreater(res[7], 0) self.assertLess(res[7], res[6]) ssummary = dm.summarise_system_stats(t_then, t_now) # ssummary (sum_requests, sum_views, sum_cached_views, # sum_downloads, sum_bytes_served, sum_seconds, max_seconds) res = ssummary self.assertEqual(res[0], IMG_VIEWS + IMG_VIEWS_COPY + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS + IMG_VIEWS_304) self.assertEqual(res[1], IMG_VIEWS + IMG_VIEWS_COPY + IMG_VIEWS_NOSTATS) self.assertEqual(res[2], IMG_VIEWS + IMG_VIEWS_NOSTATS - 1) self.assertEqual(res[3], IMG_DOWNLOADS) self.assertEqual(res[4], (IMG_VIEWS + IMG_VIEWS_COPY + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS) * IMG_LEN) self.assertGreater(res[5], 0) self.assertGreater(res[6], 0) self.assertLess(res[6], res[5]) finally: delete_file(IMG_COPY)
def topten(): # Get parameters days = request.args.get('days', '1') limit = request.args.get('number', '10') data_type = request.args.get('data_type', '2') try: results = [] db_session = data_engine.db_get_session() try: # Convert params to ints days = parse_int(days) limit = parse_int(limit) data_type = parse_int(data_type) # Set options if days < 1: days = 1 if days > 30: days = 30 if limit < 10: limit = 10 if limit > 100: limit = 100 if data_type == 1: order = '-total_requests' elif data_type == 2: order = '-total_views' elif data_type == 3: order = '-total_cached_views' elif data_type == 4: order = '-total_downloads' elif data_type == 5: order = '-total_bytes' elif data_type == 6: order = '-total_seconds' elif data_type == 7: order = '-max_seconds' else: raise ValueError('Invalid data_type %d' % data_type) # Get initial stats top_stats = data_engine.summarise_image_stats( datetime.utcnow() - timedelta(days=days), datetime.utcnow(), limit=limit, order_by=order, _db_session=db_session) # Convert stats list to an image list for result in top_stats: db_image = data_engine.get_image(image_id=result[0], _db_session=db_session) if db_image: results.append({ 'id': db_image.id, 'src': db_image.src, 'requests': result[1], 'views': result[2], 'cached_views': result[3], 'downloads': result[4], 'bytes': result[5], 'seconds': result[6], 'max_seconds': result[7] }) finally: db_session.close() return render_template('reports_topten.html', days=days, data_type=data_type, number=limit, results=results) except Exception as e: log_security_error(e, request) if app.config['DEBUG']: raise raise InternalServerError(safe_error_str(e))
def topten(): # Get parameters days = request.args.get('days', '1') limit = request.args.get('number', '10') data_type = request.args.get('data_type', '2') try: results = [] db_session = data_engine.db_get_session() try: # Convert params to ints days = parse_int(days) limit = parse_int(limit) data_type = parse_int(data_type) # Set options if days < 1: days = 1 if days > 30: days = 30 if limit < 10: limit = 10 if limit > 100: limit = 100 if data_type == 1: order = '-total_requests' elif data_type == 2: order = '-total_views' elif data_type == 3: order = '-total_cached_views' elif data_type == 4: order = '-total_downloads' elif data_type == 5: order = '-total_bytes' elif data_type == 6: order = '-total_seconds' elif data_type == 7: order = '-max_seconds' else: raise ValueError('Invalid data_type %d' % data_type) # Get initial stats top_stats = data_engine.summarise_image_stats( datetime.utcnow() - timedelta(days=days), datetime.utcnow(), limit=limit, order_by=order, _db_session=db_session ) # Convert stats list to an image list for result in top_stats: db_image = data_engine.get_image(image_id=result[0], _db_session=db_session) if db_image: results.append({ 'id': db_image.id, 'src': db_image.src, 'requests': result[1], 'views': result[2], 'cached_views': result[3], 'downloads': result[4], 'bytes': result[5], 'seconds': result[6], 'max_seconds': result[7] }) finally: db_session.close() return render_template( 'reports_topten.html', days=days, data_type=data_type, number=limit, results=results ) except Exception as e: log_security_error(e, request) if app.config['DEBUG']: raise raise InternalServerError(str(e))