def image_stats():
# Get parameters
image_id = request.args.get('id', '')
time_from = request.args.get('from', '')
time_to = request.args.get('to', '')
data_type = request.args.get('data_type', '1')
embed = request.args.get('embed', '')
try:
# Validate params, get iso and datetime versions
if image_id == '':
raise ValueError('No image was specified.')
image_id = parse_long(image_id)
(iso_time_from,
iso_time_to) = process_time_parameters(time_from, time_to)
dt_time_from = parse_iso_datetime(iso_time_from)
dt_time_to = parse_iso_datetime(iso_time_to)
embed = parse_boolean(embed)
return render_template(
'reports_image_stats.html',
timezone=get_timezone_code(),
timezone_seconds=get_timezone_offset(),
time_from=dt_time_from,
time_to=dt_time_to,
data_type=data_type,
db_image=data_engine.get_image(image_id=image_id),
embed=embed)
except Exception as e:
log_security_error(e, request)
if app.config['DEBUG']:
raise
raise InternalServerError(safe_error_str(e))
def image_stats():
# Get parameters
image_id = request.args.get('id', '')
time_from = request.args.get('from', '')
time_to = request.args.get('to', '')
data_type = request.args.get('data_type', '1')
embed = request.args.get('embed', '')
try:
# Validate params, get iso and datetime versions
if image_id == '':
raise ValueError('No image was specified.')
image_id = parse_long(image_id)
(iso_time_from, iso_time_to) = process_time_parameters(time_from, time_to)
dt_time_from = parse_iso_datetime(iso_time_from)
dt_time_to = parse_iso_datetime(iso_time_to)
embed = parse_boolean(embed)
return render_template(
'reports_image_stats.html',
timezone=get_timezone_code(),
timezone_seconds=get_timezone_offset(),
time_from=dt_time_from,
time_to=dt_time_to,
data_type=data_type,
db_image=data_engine.get_image(image_id=image_id),
embed=embed
)
except Exception as e:
log_security_error(e, request)
if app.config['DEBUG']:
raise
raise InternalServerError(str(e))
def get(self, image_id):
db_img = data_engine.get_image(image_id=image_id)
if not db_img:
raise DoesNotExistError(str(image_id))
else:
# Require view permission or file admin
permissions_engine.ensure_folder_permitted(
db_img.folder, FolderPermission.ACCESS_VIEW,
get_session_user())
return make_api_success_response(
object_to_dict(_prep_image_object(db_img)))
def put(self, image_id):
params = self._get_validated_object_parameters(request.form)
# Get image and update it
db_img = data_engine.get_image(image_id=image_id)
if not db_img:
raise DoesNotExistError(str(image_id))
# Require edit permission or file admin
permissions_engine.ensure_folder_permitted(
db_img.folder,
FolderPermission.ACCESS_EDIT,
get_session_user()
)
old_title = db_img.title
old_description = db_img.description
db_img.title = params['title']
db_img.description = params['description']
data_engine.save_object(db_img)
# Get text changes. Max info length =
# 100 + 200 + len('()' + '()' + 'Title: ' + ' / ' + 'Description: ') ==> 327
title_diff = get_string_changes(old_title, params['title'], char_limit=100).strip()
if not title_diff:
# Try for deletions from title
title_diff = get_string_changes(params['title'], old_title, char_limit=100).strip()
if title_diff:
title_diff = '(' + title_diff + ')'
desc_diff = get_string_changes(
old_description, params['description'], char_limit=200
).strip()
if not desc_diff:
# Try for deletions from description
desc_diff = get_string_changes(
params['description'], old_description, char_limit=200
).strip()
if desc_diff:
desc_diff = '(' + desc_diff + ')'
info = ''
if title_diff:
info += 'Title: ' + title_diff
if info and desc_diff:
info += ' / '
if desc_diff:
info += 'Description: ' + desc_diff
# Add change history
data_engine.add_image_history(
db_img,
get_session_user(),
ImageHistory.ACTION_EDITED,
info
)
return make_api_success_response(object_to_dict(db_img))
def get(self, image_id):
db_img = data_engine.get_image(image_id=image_id)
if not db_img:
raise DoesNotExistError(str(image_id))
else:
# Require view permission or file admin
permissions_engine.ensure_folder_permitted(
db_img.folder,
FolderPermission.ACCESS_VIEW,
get_session_user()
)
return make_api_success_response(object_to_dict(db_img))
def put(self, image_id):
params = self._get_validated_object_parameters(request.form)
# Get image and update it
db_img = data_engine.get_image(image_id=image_id)
if not db_img:
raise DoesNotExistError(str(image_id))
# Require edit permission or file admin
permissions_engine.ensure_folder_permitted(
db_img.folder, FolderPermission.ACCESS_EDIT, get_session_user())
old_title = db_img.title
old_description = db_img.description
db_img.title = params['title']
db_img.description = params['description']
data_engine.save_object(db_img)
# Get text changes. Max info length =
# 100 + 200 + len('()' + '()' + 'Title: ' + ' / ' + 'Description: ') ==> 327
title_diff = get_string_changes(old_title,
params['title'],
char_limit=100).strip()
if not title_diff:
# Try for deletions from title
title_diff = get_string_changes(params['title'],
old_title,
char_limit=100).strip()
if title_diff:
title_diff = '(' + title_diff + ')'
desc_diff = get_string_changes(old_description,
params['description'],
char_limit=200).strip()
if not desc_diff:
# Try for deletions from description
desc_diff = get_string_changes(params['description'],
old_description,
char_limit=200).strip()
if desc_diff:
desc_diff = '(' + desc_diff + ')'
info = ''
if title_diff:
info += 'Title: ' + title_diff
if info and desc_diff:
info += ' / '
if desc_diff:
info += 'Description: ' + desc_diff
# Add change history
data_engine.add_image_history(db_img, get_session_user(),
ImageHistory.ACTION_EDITED, info)
return make_api_success_response(
object_to_dict(_prep_image_object(db_img)))
def post(self, folio_id):
db_session = data_engine.db_get_session()
try:
# Get the portfolio
folio = data_engine.get_portfolio(folio_id, _db_session=db_session)
if folio is None:
raise DoesNotExistError(str(folio_id))
# Check portfolio permissions
permissions_engine.ensure_portfolio_permitted(
folio, FolioPermission.ACCESS_EDIT, get_session_user())
# Get the image by either ID or src
params = self._get_validated_object_parameters(request.form, True)
if 'image_id' in params:
image = data_engine.get_image(params['image_id'],
_db_session=db_session)
if image is None:
raise DoesNotExistError(str(params['image_id']))
else:
image = auto_sync_file(params['image_src'],
data_engine,
task_engine,
anon_history=True,
burst_pdf=False,
_db_session=db_session)
if image is None or image.status == Image.STATUS_DELETED:
raise DoesNotExistError(params['image_src'])
# Check image permissions
permissions_engine.ensure_folder_permitted(
image.folder, FolderPermission.ACCESS_VIEW, get_session_user(),
False)
# Add history first so that we only commit once at the end
data_engine.add_portfolio_history(folio,
get_session_user(),
FolioHistory.ACTION_IMAGE_CHANGE,
'%s added' % image.src,
_db_session=db_session,
_commit=False)
# Flag that exported zips are now out of date
folio.last_updated = datetime.utcnow()
# Add the image and commit changes
db_folio_image = data_engine.save_object(FolioImage(
folio, image, params['image_parameters'], params['filename'],
params['index']),
refresh=True,
_db_session=db_session,
_commit=True)
return make_api_success_response(
object_to_dict(_prep_folioimage_object(db_folio_image),
_omit_fields + ['portfolio']))
finally:
db_session.close()
def delete(self, image_id):
""" Deletes a file from disk """
# Get image data
db_img = data_engine.get_image(image_id=image_id)
if not db_img:
raise DoesNotExistError(str(image_id))
# v4.1 #10 delete_file() doesn't care whether the file exists, but we
# want the API to return a "not found" if the file doesn't exist
# (and as long as the database is already in sync with that)
if not path_exists(
db_img.src,
require_file=True) and db_img.status == Image.STATUS_DELETED:
raise DoesNotExistError(db_img.src)
# Delete
db_img = delete_file(db_img, get_session_user(), data_engine,
permissions_engine)
# Remove cached images for old path
image_engine._uncache_image_id(db_img.id)
# Return updated image
return make_api_success_response(
object_to_dict(_prep_image_object(db_img)))
def put(self, image_id):
""" Moves or renames a file on disk """
params = self._get_validated_parameters(request.form)
# Get image data
db_img = data_engine.get_image(image_id=image_id)
if not db_img:
raise DoesNotExistError(str(image_id))
# Move
try:
db_img = move_file(db_img, params['path'], get_session_user(),
data_engine, permissions_engine)
except ValueError as e:
if type(e) is ValueError:
raise ParameterError(str(e))
else:
raise # Sub-classes of ValueError
# Remove cached images for the old path
image_engine._uncache_image_id(db_img.id)
# Return updated image
return make_api_success_response(
object_to_dict(_prep_image_object(db_img)))
def test_stats_engine(self):
# Clear stats - wait for previous stats to flush then delete all
time.sleep(65)
dm.delete_system_stats(datetime.utcnow())
dm.delete_image_stats(datetime.utcnow())
cm.clear()
# Test constants
IMG = 'test_images/cathedral.jpg'
IMG_COPY = 'test_images/stats_test_image.jpg'
IMG_LEN = 648496 # knowing length requires 'keep original' values in settings
IMG_VIEWS = 5
IMG_VIEWS_NOSTATS = 3
IMG_VIEWS_304 = 8
IMG_VIEWS_COPY = 1
IMG_DOWNLOADS = 1
try:
t_then = datetime.utcnow()
copy_file(IMG, IMG_COPY)
# View some images
for _ in range(IMG_VIEWS):
rv = self.app.get('/image?src='+IMG)
self.assertEqual(rv.status_code, 200)
self.assertEqual(len(rv.data), IMG_LEN)
for _ in range(IMG_DOWNLOADS):
rv = self.app.get('/original?src='+IMG)
self.assertEqual(rv.status_code, 200)
self.assertEqual(len(rv.data), IMG_LEN)
# View some also without stats.
# They should still be counted in the system stats but not the image stats.
for _ in range(IMG_VIEWS_NOSTATS):
rv = self.app.get('/image?src='+IMG+'&stats=0')
self.assertEqual(rv.status_code, 200)
self.assertEqual(len(rv.data), IMG_LEN)
etag = rv.headers['ETag']
# View some that only elicit the 304 Not Modified response
for _ in range(IMG_VIEWS_304):
rv = self.app.get('/image?src='+IMG, headers={'If-None-Match': etag})
self.assertEqual(rv.status_code, 304)
# View an image that we'll delete next
for _ in range(IMG_VIEWS_COPY):
rv = self.app.get('/image?src='+IMG_COPY)
self.assertEqual(len(rv.data), IMG_LEN)
# Get test image db record
db_image = dm.get_image(src=IMG)
self.assertIsNotNone(db_image)
# Deleting the copied image should mean its views are counted in the system stats
# but are not included in the image stats (because the image record is gone)
delete_file(IMG_COPY)
dm.delete_image(dm.get_image(src=IMG_COPY), purge=True)
db_image_copy = dm.get_image(src=IMG_COPY)
self.assertIsNone(db_image_copy)
# Wait for new stats to flush
time.sleep(65)
# See if the system stats line up with the views
t_now = datetime.utcnow()
lres = dm.search_system_stats(t_then, t_now)
self.assertEqual(len(lres), 1)
res = lres[0]
self.assertEqual(res.requests, IMG_VIEWS + IMG_VIEWS_COPY + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS + IMG_VIEWS_304)
self.assertEqual(res.views, IMG_VIEWS + IMG_VIEWS_COPY + IMG_VIEWS_NOSTATS)
self.assertEqual(res.cached_views, IMG_VIEWS + IMG_VIEWS_NOSTATS - 1)
self.assertEqual(res.downloads, IMG_DOWNLOADS)
self.assertEqual(res.total_bytes, (IMG_VIEWS + IMG_VIEWS_COPY + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS) * IMG_LEN)
self.assertGreater(res.request_seconds, 0)
self.assertGreater(res.max_request_seconds, 0)
self.assertLess(res.max_request_seconds, res.request_seconds)
self.assertGreater(res.cpu_pc, 0)
self.assertGreater(res.memory_pc, 0)
# See if the image stats line up with the views
lres = dm.search_image_stats(t_then, t_now, db_image.id)
self.assertEqual(len(lres), 1)
res = lres[0]
self.assertEqual(res.requests, IMG_VIEWS + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS + IMG_VIEWS_304)
self.assertEqual(res.views, IMG_VIEWS)
self.assertEqual(res.cached_views, IMG_VIEWS - 1)
self.assertEqual(res.downloads, IMG_DOWNLOADS)
self.assertEqual(res.total_bytes, (IMG_VIEWS + IMG_DOWNLOADS) * IMG_LEN)
self.assertGreater(res.request_seconds, 0)
self.assertGreater(res.max_request_seconds, 0)
self.assertLess(res.max_request_seconds, res.request_seconds)
# And the summary (reporting) data too
lsummary = dm.summarise_image_stats(t_then, t_now)
# lsummary [(image_id, sum_requests, sum_views, sum_cached_views,
# sum_downloads, sum_bytes_served, sum_seconds, max_seconds)]
self.assertEqual(len(lsummary), 1)
res = lsummary[0]
self.assertEqual(res[0], db_image.id)
self.assertEqual(res[1], IMG_VIEWS + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS + IMG_VIEWS_304)
self.assertEqual(res[2], IMG_VIEWS)
self.assertEqual(res[3], IMG_VIEWS - 1)
self.assertEqual(res[4], IMG_DOWNLOADS)
self.assertEqual(res[5], (IMG_VIEWS + IMG_DOWNLOADS) * IMG_LEN)
self.assertGreater(res[6], 0)
self.assertGreater(res[7], 0)
self.assertLess(res[7], res[6])
ssummary = dm.summarise_system_stats(t_then, t_now)
# ssummary (sum_requests, sum_views, sum_cached_views,
# sum_downloads, sum_bytes_served, sum_seconds, max_seconds)
res = ssummary
self.assertEqual(res[0], IMG_VIEWS + IMG_VIEWS_COPY + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS + IMG_VIEWS_304)
self.assertEqual(res[1], IMG_VIEWS + IMG_VIEWS_COPY + IMG_VIEWS_NOSTATS)
self.assertEqual(res[2], IMG_VIEWS + IMG_VIEWS_NOSTATS - 1)
self.assertEqual(res[3], IMG_DOWNLOADS)
self.assertEqual(res[4], (IMG_VIEWS + IMG_VIEWS_COPY + IMG_DOWNLOADS + IMG_VIEWS_NOSTATS) * IMG_LEN)
self.assertGreater(res[5], 0)
self.assertGreater(res[6], 0)
self.assertLess(res[6], res[5])
finally:
delete_file(IMG_COPY)
def topten():
# Get parameters
days = request.args.get('days', '1')
limit = request.args.get('number', '10')
data_type = request.args.get('data_type', '2')
try:
results = []
db_session = data_engine.db_get_session()
try:
# Convert params to ints
days = parse_int(days)
limit = parse_int(limit)
data_type = parse_int(data_type)
# Set options
if days < 1:
days = 1
if days > 30:
days = 30
if limit < 10:
limit = 10
if limit > 100:
limit = 100
if data_type == 1:
order = '-total_requests'
elif data_type == 2:
order = '-total_views'
elif data_type == 3:
order = '-total_cached_views'
elif data_type == 4:
order = '-total_downloads'
elif data_type == 5:
order = '-total_bytes'
elif data_type == 6:
order = '-total_seconds'
elif data_type == 7:
order = '-max_seconds'
else:
raise ValueError('Invalid data_type %d' % data_type)
# Get initial stats
top_stats = data_engine.summarise_image_stats(
datetime.utcnow() - timedelta(days=days),
datetime.utcnow(),
limit=limit,
order_by=order,
_db_session=db_session)
# Convert stats list to an image list
for result in top_stats:
db_image = data_engine.get_image(image_id=result[0],
_db_session=db_session)
if db_image:
results.append({
'id': db_image.id,
'src': db_image.src,
'requests': result[1],
'views': result[2],
'cached_views': result[3],
'downloads': result[4],
'bytes': result[5],
'seconds': result[6],
'max_seconds': result[7]
})
finally:
db_session.close()
return render_template('reports_topten.html',
days=days,
data_type=data_type,
number=limit,
results=results)
except Exception as e:
log_security_error(e, request)
if app.config['DEBUG']:
raise
raise InternalServerError(safe_error_str(e))
def topten():
# Get parameters
days = request.args.get('days', '1')
limit = request.args.get('number', '10')
data_type = request.args.get('data_type', '2')
try:
results = []
db_session = data_engine.db_get_session()
try:
# Convert params to ints
days = parse_int(days)
limit = parse_int(limit)
data_type = parse_int(data_type)
# Set options
if days < 1:
days = 1
if days > 30:
days = 30
if limit < 10:
limit = 10
if limit > 100:
limit = 100
if data_type == 1:
order = '-total_requests'
elif data_type == 2:
order = '-total_views'
elif data_type == 3:
order = '-total_cached_views'
elif data_type == 4:
order = '-total_downloads'
elif data_type == 5:
order = '-total_bytes'
elif data_type == 6:
order = '-total_seconds'
elif data_type == 7:
order = '-max_seconds'
else:
raise ValueError('Invalid data_type %d' % data_type)
# Get initial stats
top_stats = data_engine.summarise_image_stats(
datetime.utcnow() - timedelta(days=days),
datetime.utcnow(),
limit=limit,
order_by=order,
_db_session=db_session
)
# Convert stats list to an image list
for result in top_stats:
db_image = data_engine.get_image(image_id=result[0], _db_session=db_session)
if db_image:
results.append({
'id': db_image.id,
'src': db_image.src,
'requests': result[1],
'views': result[2],
'cached_views': result[3],
'downloads': result[4],
'bytes': result[5],
'seconds': result[6],
'max_seconds': result[7]
})
finally:
db_session.close()
return render_template(
'reports_topten.html',
days=days,
data_type=data_type,
number=limit,
results=results
)
except Exception as e:
log_security_error(e, request)
if app.config['DEBUG']:
raise
raise InternalServerError(str(e))
