async def delete(self, uid, variation_name):
"""Remove the variation from the asset"""
asset = self.get_asset(uid,
projection={
'expires': True,
'name': True,
'secure': True,
'uid': True,
f'variations.{variation_name}': {
'$sub': Variation
}
})
variation = self.get_variation(asset, variation_name)
# Remove file for the variation
backend = self.get_backend(asset.secure)
await backend.async_delete(variation.get_store_key(
asset, variation_name),
loop=asyncio.get_event_loop())
# Remove the variation from the asset
Asset.get_collection().update(
(Q._id == asset._id).to_dict(),
{'$unset': {
f'variations.{variation_name}': ''
}})
# Update the asset stats
Stats.inc(self.account, today_tz(tz=self.config['TIMEZONE']), {
'variations': -1,
'length': -variation.meta['length']
})
self.set_status(204)
self.finish()
def get_highlights(state):
# Get the stats for the account
api_call_keys = Stats.get_key_range(today_tz(), 'days', 7, 'api_calls')
state.has_stats = False
stats = Stats.one(Q.scope == state.account,
projection={
'values.all.assets': True,
'values.all.length': True,
'values.all.variations': True,
**{f'values.{k}': True
for k in api_call_keys}
})
if stats:
state.has_stats = True
state.highlights = {
'api_calls': stats.sum_stats(api_call_keys),
'assets': stats.get_stat('all.assets'),
'length': stats.get_stat('all.length'),
'variations': stats.get_stat('all.variations')
}
def get_activity(state):
# Determine the period to view
unit = 'days'
length = 90
state.period = '90d'
if request.args.get('period') == '12m':
unit = 'months'
length = 12
state.period = '12m'
elif request.args.get('period') == '10y':
unit = 'years'
length = 10
state.period = '10y'
# Build the keys required for the period
today = today_tz()
api_call_keys = Stats.get_key_range(today, unit, length, 'api_calls')
asset_keys = Stats.get_key_range(today, unit, length, 'assets')
length_keys = Stats.get_key_range(today, unit, length, 'length')
variations_keys = Stats.get_key_range(today, unit, length, 'variations')
# Get the stats for the account
state.has_stats = False
stats = Stats.one(Q.scope == 'all',
projection={
**{f'values.{k}': True
for k in api_call_keys},
**{f'values.{k}': True
for k in asset_keys},
**{f'values.{k}': True
for k in length_keys},
**{f'values.{k}': True
for k in variations_keys}
})
if stats:
state.has_stats = True
# Build the period totals
state.totals = {
'api_calls': stats.sum_stats(api_call_keys),
'assets': stats.sum_stats(asset_keys),
'length': stats.sum_stats(length_keys),
'variations': stats.sum_stats(variations_keys)
}
# Build the chart data
color = {
'backgroundColor': '#67B3DA64',
'borderColor': '#67B3DA',
'pointBackgroundColor': '#67B3DA64',
'pointBorderColor': '#67B3DA'
}
state.data_series = {
'api_calls': {
'datasets': [{
'data': stats.get_series(api_call_keys),
**color
}],
'labels': stats.get_series_labels(api_call_keys)
},
'assets': {
'datasets': [{
'data': stats.get_series(asset_keys),
**color
}],
'labels': stats.get_series_labels(asset_keys)
},
'length': {
'datasets': [{
'data': [
round(v / 1000000, 2)
for v in stats.get_series(length_keys)
],
**color
}],
'labels':
stats.get_series_labels(length_keys)
},
'variations': {
'datasets': [{
'data': stats.get_series(variations_keys),
**color
}],
'labels': stats.get_series_labels(variations_keys)
}
}
def _store_variation(
self,
config,
asset,
variation_name,
versioned,
ext,
meta,
file
):
"""
Store a new variation of the asset. This method both stores the
variation (and removes any existing variation with the same name) as
well as updating the asset's `variations` field with details of the
new variation.
"""
# Get the account and backend associated with the asset
account = Account.by_id(
asset.account,
projection={
'public_backend_settings': True,
'secure_backend_settings': True
}
)
if asset.secure:
backend = account.secure_backend
else:
backend = account.public_backend
assert backend, 'No backend configured for the asset'
# Ensure the asset's variation value is a dictionary (in case it's
# never been set before).
if not asset.variations:
asset.variations = {}
# Create the new variation
old_variation = asset.variations.get(variation_name)
new_variation = Variation(
content_type=mimetypes.guess_type(f'f.{ext}')[0] if ext else '',
ext=ext,
meta=meta,
version=(
Variation.next_version(
old_variation.version if old_variation else None
)
if versioned else None
)
)
# Store the new variation
new_store_key = new_variation.get_store_key(asset, variation_name)
file.seek(0)
backend.store(file, new_store_key)
# Add the new variation's details to the asset's `variations` field
asset.variations[variation_name] = new_variation
asset.modified = datetime.utcnow()
# Apply the updated to the database
Asset.get_collection().update(
(Q._id == asset._id).to_dict(),
{
'$set': {
f'variations.{variation_name}': \
new_variation.to_json_type(),
'modified': asset.modified
}
}
)
# Remove the existing variation
if old_variation:
old_store_key = old_variation.get_store_key(asset, variation_name)
if new_store_key != old_store_key:
backend.delete(old_store_key)
# Update the asset stats
new_length = new_variation.meta['length']
old_length = 0
if old_variation:
old_length = old_variation.meta['length']
Stats.inc(
account,
today_tz(tz=config['TIMEZONE']),
{
'variations': 0 if old_variation else 1,
'length': new_length - old_length
}
)
async def put(self):
"""Store the uploaded file as an asset"""
# Make sure a file was received
files = self.request.files.get('file')
if not files:
raise APIError(
'invalid_request',
arg_errors={'file': ['No file received.']}
)
file = files[0]
# Validate the arguments
form = PutForm(to_multi_dict(self.request.body_arguments))
if not form.validate():
raise APIError(
'invalid_request',
arg_errors=form.errors
)
if self.config['ANTI_VIRUS_ENABLED']:
# Check the file for viruses
av_client = clamd.ClamdUnixSocket(
self.config['ANTI_VIRUS_CLAMD_PATH']
)
av_scan_result = av_client.instream(io.BytesIO(file.body))
if av_scan_result['stream'][0] == 'FOUND':
raise APIError(
'invalid_request',
arg_errors={
'file': ['File appears to be a virus.']
}
)
form_data = form.data
# Create a name for the asset
fname, fext = os.path.splitext(file.filename)
name = slugify(
form_data['name'] or fname,
regex_pattern=ALLOWED_SLUGIFY_CHARACTERS,
max_length=200
)
# Determine the files extension
ext = fext[1:] if fext else imghdr.what(file.filename, file.body)
# Determine the asset type/content type for the image
content_type = mimetypes.guess_type(f'f.{ext}')[0] \
or 'application/octet-stream'
asset_type = self.config['CONTENT_TYPE_TO_TYPES'].get(
content_type,
'file'
)
# Build the meta data for the asset
meta = {
'filename': file.filename,
'length': len(file.body)
}
if asset_type == 'audio':
try:
au_file = io.BytesIO(file.body)
au_file.name = file.filename
au = mutagen.File(au_file)
except:
raise APIError(
'invalid_request',
arg_errors={
'file': ['Unable to open the file as an audio file.']
}
)
if au is not None:
meta['audio'] = {
'channels': getattr(au.info, 'channels', -1),
'length': getattr(au.info, 'length', -1),
'mode': {
0: 'stereo',
1: 'joint_stereo',
2: 'dual_channel',
3: 'mono'
}.get(getattr(au.info, 'mode', ''), ''),
'sample_rate': getattr(au.info, 'sample_rate', -1)
}
if asset_type == 'image':
im = None
try:
im = Image.open(io.BytesIO(file.body))
meta['image'] = {
'mode': im.mode,
'size': im.size
}
except:
raise APIError(
'invalid_request',
arg_errors={
'file': ['Unable to open the file as an image.']
}
)
finally:
if im:
im.close()
# Create the asset
asset = Asset(
uid=Asset.generate_uid(),
account=self.account,
secure=form_data['secure'],
name=name,
ext=ext,
type=asset_type,
content_type=content_type,
expires=(time.time() + form_data['expire'])
if form_data['expire'] else None,
meta=meta
)
# Store the file
backend = self.get_backend(asset.secure)
await backend.async_store(
io.BytesIO(file.body),
asset.store_key,
loop=asyncio.get_event_loop()
)
# Save the asset
asset.insert()
# Update the asset stats
Stats.inc(
self.account,
today_tz(tz=self.config['TIMEZONE']),
{
'assets': 1,
'length': asset.meta['length']
}
)
self.write(asset.to_json_type())
async def prepare(self):
from blueprints.accounts.models import Account, Stats
# (Re)Set the account attribute
self._account = None
# (Re)Set the write log
self._write_log = ''
# (Re)Set the call timer
self._call_timer = time.time()
# Check a valid API key has been provided
api_key = self.request.headers.get('X-H51-APIKey')
if not api_key:
raise APIError('unauthorized', 'No authorization key provided.')
# Find the calling account
account = Account.one(
Q.api_key == api_key,
projection={
'api_allowed_ip_addresses': True,
'api_rate_limit_per_second': True
}
)
# Store a reference to the account document against the request handler
self._account = account
if not account:
raise APIError('unauthorized', 'API key not recognized.')
if account.api_allowed_ip_addresses:
# Check the caller's IP address is allowed
ip_address = self.request.headers.get('X-Real-Ip', '')
if ip_address not in account.api_allowed_ip_addresses:
raise APIError(
'forbidden',
(
f'The IP address {ip_address} is not allowed to call '
'the API for this account.'
)
)
# Record this request
rate_key = account.get_rate_key()
ttl = await self.redis.pttl(rate_key)
if ttl > 0:
await self.redis.incr(rate_key)
else:
multi = self.redis.multi_exec()
multi.incr(rate_key)
multi.expire(rate_key, 1)
await multi.execute()
# Apply rate limit
request_count = int((await self.redis.get(rate_key)) or 0)
rate_limit = account.api_rate_limit_per_second \
or self.config['API_RATE_LIMIT_PER_SECOND']
if request_count > rate_limit:
raise APIError('request_limit_exceeded')
# Set the remaining requests allowed this second in the response
# headers.
rate_key_reset = max(0, ttl / 1000.0)
rate_key_reset += time.time()
self.set_header('X-H51-RateLimit-Limit', str(rate_limit))
self.set_header(
'X-H51-RateLimit-Remaining',
str(rate_limit - request_count)
)
self.set_header('X-H51-RateLimit-Reset', str(rate_key_reset))
# Update the API call stats
Stats.inc(
account,
today_tz(tz=self.config['TIMEZONE']),
{'api_calls': 1}
)
def purge():
"""Purge assets that have expired"""
# Get expired assets
now = time.time()
max_delete_period = 48 * 60 * 60
assets = Asset.many(And(
Q.expires <= now,
Q.expires > now - max_delete_period,
),
projection={
'expires': True,
'ext': True,
'meta.length': True,
'name': True,
'secure': True,
'uid': True,
'variations': {
'$sub.': Variation
},
'account': {
'$ref': Account,
'public_backend_settings': True,
'secure_backend_settings': True
}
})
# Delete the assets
for asset in assets:
variation_count = 0
length = asset.meta['length']
# Remove all stored files for the asset
if asset.secure:
backend = asset.account.secure_backend
else:
backend = asset.account.public_backend
if backend:
if asset.variations:
# Count the variations for the asset
variation_count = len(asset.variations)
for variation_name, variation in asset.variations.items():
# Tally up the assets total footprint
length += variation.meta['length']
# Remove variation files
backend.delete(
variation.get_store_key(asset, variation_name))
# Remove the asset file
backend.delete(asset.store_key)
# Delete the asset from the database
asset.delete()
# Update the asset stats
Stats.inc(asset.account, today_tz(), {
'assets': -1,
'variations': -variation_count,
'length': -length
})