def post(self):
"""
Removes any existing timeout for one or more existing assets making
the assets persistent.
"""
assets = self.get_assets(projection={'uid': True})
# Clear the expires timestamp (if there is one) for the assets
Asset.get_collection().update(In(Q._id,
[a._id for a in assets]).to_dict(), {
'$set': {
'modified': datetime.utcnow()
},
'$unset': {
'expires': ''
}
},
multi=True)
self.write(
{'results': [{
'uid': a.uid,
'expires': None
} for a in assets]})
def delete_document(state):
Asset.get_collection().update_many(
(Q.account == state.account._id).to_dict(),
{
'$set': {
'expires': time.time() - 1,
'modified': datetime.utcnow()
}
}
)
generic.delete['post'].super(state)
def post(self):
"""
Removes any existing timeout for one or more existing assets making
the assets persistent.
Set a timeout for one or more assets. After the timeout has expired
the assets will be automatically deleted.
NOTE: The files associated with expired assets are delete periodically
and therefore may still temporarily be available after the asset has
expired and is not longer available via the API.
"""
assets = self.get_assets(projection={'uid': True})
# Validate the arguments
form = ExpireForm(to_multi_dict(self.request.body_arguments))
if not form.validate():
raise APIError(
'invalid_request',
arg_errors=form.errors
)
form_data = form.data
# Update the expires timestamp for the asset
expires = time.time() + form_data['seconds']
Asset.get_collection().update(
In(Q._id, [a._id for a in assets]).to_dict(),
{
'$set': {
'expires': time.time() + form_data['seconds'],
'modified': datetime.utcnow()
}
},
multi=True
)
self.write({
'results': [
{
'uid': a.uid,
'expires': expires
}
for a in assets
]
})
def uid_to_object_id(uid):
"""Coerce a UID for an asset to the asset's ObjectId"""
ids = Asset.ids(Q.uid == uid)
if not ids:
raise ValueError('Not a valid uid.')
return ids[0]
def post(self, uid):
"""
Removes any existing timeout for an asset making the asset persistent.
"""
asset = self.get_asset(uid, projection={'uid': True, 'expires': True})
# Clear the expires timestamp (if there is one)
Asset.get_collection().update({'_id': asset._id}, {
'$set': {
'modified': datetime.utcnow()
},
'$unset': {
'expires': ''
}
})
self.write({'uid': asset.uid, 'expires': None})
def _add_to_meta(self, asset, data):
"""Add the specified data to the asset's meta"""
# Modify the asset instance
if self.asset_type not in asset.meta:
asset.meta[self.asset_type] = {}
asset.meta[self.asset_type][self.name] = data
asset.modified = datetime.utcnow()
# Apply the updated to the database
Asset.get_collection().update(
(Q._id == asset._id).to_dict(),
{
'$set': {
f'meta.{self.asset_type}.{self.name}': data,
'modified': asset.modified
}
}
)
async def delete(self, uid, variation_name):
"""Remove the variation from the asset"""
asset = self.get_asset(uid,
projection={
'expires': True,
'name': True,
'secure': True,
'uid': True,
f'variations.{variation_name}': {
'$sub': Variation
}
})
variation = self.get_variation(asset, variation_name)
# Remove file for the variation
backend = self.get_backend(asset.secure)
await backend.async_delete(variation.get_store_key(
asset, variation_name),
loop=asyncio.get_event_loop())
# Remove the variation from the asset
Asset.get_collection().update(
(Q._id == asset._id).to_dict(),
{'$unset': {
f'variations.{variation_name}': ''
}})
# Update the asset stats
Stats.inc(self.account, today_tz(tz=self.config['TIMEZONE']), {
'variations': -1,
'length': -variation.meta['length']
})
self.set_status(204)
self.finish()
def not_expired(**view_args):
"""
This action is only available against assets that have not expired.
"""
asset_id = view_args.get('asset', request.args.get('asset'))
if asset_id:
asset = Asset.by_id(ObjectId(asset_id), projection={'expires': True})
if asset:
return not asset.expired
return False
def get_asset(self, uid, projection=None):
"""
Get the asset for the given `uid` and raise an error if no asset is
found.
"""
assert not projection or 'expires' in projection, \
'`expires` must be included by the projection'
# Fetch the asset
asset = Asset.one(And(Q.account == self.account, Q.uid == uid),
projection=(projection or self.DEFAULT_PROJECTION))
if not asset or asset.expired:
raise APIError('not_found',
hint=f'Asset not found for uid: {uid}.')
return asset
def get_assets(self, projection=None):
"""
Return a list of assets given (as JSON) in the form argument `assets`.
"""
uids = self.get_body_arguments('uids')
if not uids:
raise APIError(
'invalid_request',
hint=f'No uids provided.'
)
# Fetch the asset
return Asset.many(
And(
Q.account == self.account,
In(Q.uid, uids),
Not(Q.expires <= time.time())
),
projection=(projection or self.DEFAULT_PROJECTION)
)
async def put(self):
"""Generate variations for one or more assets"""
assets = self.get_assets(projection={
'_id': True,
'type': True,
'uid': True
})
# Extract the variations from the variations argument
try:
raw_variations = json.loads(self.get_body_argument('variations'))
except ValueError:
raise APIError('invalid_request',
hint='Variations argument is not valid JSON.')
# Peek to determine if the user wants the variations applied globally
# or locally.
if self.get_body_argument('local', False):
# Variations must be defined for each uid
uids = set(self.get_body_arguments('uids'))
variation_keys = set(list(raw_variations.keys()))
if uids != variation_keys:
raise APIError('invalid_request',
hint='Each uid must be assigned a variation.')
variations = {
a.uid: self.validate_variations(a.type, raw_variations[a.uid])
for a in assets
}
else:
# Global application
# Ensure all assets are of the same type
asset_types = set([a.type for a in assets])
if len(asset_types) > 1:
raise APIError('invalid_request',
hint='All assets must be of the same type.')
local_variations = self.validate_variations(
asset_types.pop(), raw_variations)
variations = {a.uid: local_variations for a in assets}
# Add a set of tasks to generate the asset variations
notification_url = self.get_body_argument('notification_url', None)
tasks = []
task_names = []
for asset in assets:
local_variations = variations[asset.uid]
for variation_name, transforms in local_variations.items():
task = GenerateVariationTask(self.account._id, asset._id,
variation_name, transforms,
notification_url)
if notification_url:
tasks.append(self.add_task_and_forget(task))
else:
tasks.append(self.add_task_and_wait(task))
task_names.append(f'{asset.uid}:{variation_name}')
if notification_url:
# Fire and forget
await asyncio.gather(*tasks)
self.finish()
else:
# Wait for response
events = await asyncio.gather(*tasks)
# Collect any errors
errors = {}
for i, event in enumerate(events):
if not event:
errors[task_names[i]] = ['Connection lost']
elif event.type == 'task_error':
errors[task_names[i]] = [event.reason]
if errors:
raise APIError('error', arg_errors=errors)
# Fetch the asset again now the variations have been generated
with Asset.with_options(read_preference=ReadPreference.PRIMARY):
assets = self.get_assets(
projection={
'uid': True,
'expires': True,
'ext': True,
'meta': True,
'name': True,
'variations': {
'$sub.': Variation
}
})
results = [a.to_json_type() for a in assets]
self.write({
'results': [{
'uid': r['uid'],
'variations': r['variations']
} for r in results]
})
async def post(self, uid):
"""Analyze the asset for additional meta data"""
asset = self.get_asset(
uid,
projection={
'_id': True,
'type': True,
'expires': True
}
)
# Extract the analyzers from the request body
try:
raw_analyzers = json.loads(self.get_body_argument('analyzers'))
except:
raise APIError(
'invalid_request',
hint='Analyzers argument is not valid JSON.'
)
analyzers = self.validate_analyzers(asset.type, raw_analyzers)
# Add a task to perform the asset analysis
notification_url = self.get_body_argument('notification_url', None)
task = AnalyzeTask(
self.account._id,
asset._id,
analyzers,
notification_url
)
if notification_url:
# Fire and forget
await self.add_task_and_forget(task)
self.finish()
else:
# Wait for response
event = await self.add_task_and_wait(task)
if not event:
raise APIError('error', 'Connection lost')
elif event.type == 'task_error':
raise APIError('error', event.reason)
# Fetch the asset again now the analysis is complete
with Asset.with_options(read_preference=ReadPreference.PRIMARY):
asset = self.get_asset(
uid,
projection={
'uid': True,
'expires': True,
'ext': True,
'meta': True,
'name': True
}
)
# Handle image expiry
if not asset:
raise APIError(
'not_found',
hint='Asset expired whilst being analyzed'
)
json_type = asset.to_json_type()
self.write({
'uid': json_type['uid'],
'meta': json_type['meta']
})
async def post(self):
"""Analyze one for more asset for additional meta data"""
assets = self.get_assets(
projection={
'_id': True,
'type': True,
'expires': True,
'uid': True
}
)
# Extract the analyzers from the request body
try:
raw_analyzers = json.loads(self.get_body_argument('analyzers'))
except:
raise APIError(
'invalid_request',
hint='Analyzers argument is not valid JSON.'
)
# Peek to determine if the user wants the variations applied globally
# or locally.
if self.get_body_argument('local', False):
# Variations must be defined for each uid
uids = set(self.get_body_arguments('uids'))
analyzer_keys = set(list(raw_analyzers.keys()))
if uids != analyzer_keys:
raise APIError(
'invalid_request',
hint='Each uid must be assigned a list of analyzers.'
)
analyzers = {
a.uid: self.validate_analyzers(a.type, raw_analyzers[a.uid])
for a in assets
}
else:
# Global application
# Ensure all assets are the same type / base type (file)
asset_types = set([a.type for a in assets if a.type != 'file'])
if len(asset_types) > 1:
raise APIError(
'invalid_request',
hint=(
'All assets must be of the same type / base type '
'(file)'
)
)
local_analyzers = self.validate_analyzers(
asset_types.pop(),
raw_analyzers
)
analyzers = {a.uid: local_analyzers for a in assets}
# Add a set of tasks to generate the asset variations
notification_url = self.get_body_argument('notification_url', None)
tasks = []
task_names = []
for asset in assets:
task = AnalyzeTask(
self.account._id,
asset._id,
analyzers[asset.uid],
notification_url
)
if notification_url:
tasks.append(self.add_task_and_forget(task))
else:
tasks.append(self.add_task_and_wait(task))
task_names.append(asset.uid)
if notification_url:
# Fire and forget
await self.add_task_and_forget(task)
self.finish()
else:
# Wait for response
events = await asyncio.gather(*tasks)
# Collect any errors
errors = {}
for i, event in enumerate(events):
if not event:
errors[task_names[i]] = ['Connection lost']
elif event.type == 'task_error':
errors[task_names[i]] = [event.reason]
if errors:
raise APIError('error', arg_errors=errors)
# Fetch the assets again now the analysis is complete
with Asset.with_options(read_preference=ReadPreference.PRIMARY):
assets = self.get_assets(
projection={
'uid': True,
'expires': True,
'ext': True,
'meta': True,
'name': True
}
)
# Handle image expiry
if not asset:
raise APIError(
'not_found',
hint='Asset expired whilst being analyzed'
)
results = [a.to_json_type() for a in assets]
self.write({
'results': [
{
'uid': r['uid'],
'meta': r['meta']
}
for r in results
]
})
async def put(self):
"""Store the uploaded file as an asset"""
# Make sure a file was received
files = self.request.files.get('file')
if not files:
raise APIError(
'invalid_request',
arg_errors={'file': ['No file received.']}
)
file = files[0]
# Validate the arguments
form = PutForm(to_multi_dict(self.request.body_arguments))
if not form.validate():
raise APIError(
'invalid_request',
arg_errors=form.errors
)
if self.config['ANTI_VIRUS_ENABLED']:
# Check the file for viruses
av_client = clamd.ClamdUnixSocket(
self.config['ANTI_VIRUS_CLAMD_PATH']
)
av_scan_result = av_client.instream(io.BytesIO(file.body))
if av_scan_result['stream'][0] == 'FOUND':
raise APIError(
'invalid_request',
arg_errors={
'file': ['File appears to be a virus.']
}
)
form_data = form.data
# Create a name for the asset
fname, fext = os.path.splitext(file.filename)
name = slugify(
form_data['name'] or fname,
regex_pattern=ALLOWED_SLUGIFY_CHARACTERS,
max_length=200
)
# Determine the files extension
ext = fext[1:] if fext else imghdr.what(file.filename, file.body)
# Determine the asset type/content type for the image
content_type = mimetypes.guess_type(f'f.{ext}')[0] \
or 'application/octet-stream'
asset_type = self.config['CONTENT_TYPE_TO_TYPES'].get(
content_type,
'file'
)
# Build the meta data for the asset
meta = {
'filename': file.filename,
'length': len(file.body)
}
if asset_type == 'audio':
try:
au_file = io.BytesIO(file.body)
au_file.name = file.filename
au = mutagen.File(au_file)
except:
raise APIError(
'invalid_request',
arg_errors={
'file': ['Unable to open the file as an audio file.']
}
)
if au is not None:
meta['audio'] = {
'channels': getattr(au.info, 'channels', -1),
'length': getattr(au.info, 'length', -1),
'mode': {
0: 'stereo',
1: 'joint_stereo',
2: 'dual_channel',
3: 'mono'
}.get(getattr(au.info, 'mode', ''), ''),
'sample_rate': getattr(au.info, 'sample_rate', -1)
}
if asset_type == 'image':
im = None
try:
im = Image.open(io.BytesIO(file.body))
meta['image'] = {
'mode': im.mode,
'size': im.size
}
except:
raise APIError(
'invalid_request',
arg_errors={
'file': ['Unable to open the file as an image.']
}
)
finally:
if im:
im.close()
# Create the asset
asset = Asset(
uid=Asset.generate_uid(),
account=self.account,
secure=form_data['secure'],
name=name,
ext=ext,
type=asset_type,
content_type=content_type,
expires=(time.time() + form_data['expire'])
if form_data['expire'] else None,
meta=meta
)
# Store the file
backend = self.get_backend(asset.secure)
await backend.async_store(
io.BytesIO(file.body),
asset.store_key,
loop=asyncio.get_event_loop()
)
# Save the asset
asset.insert()
# Update the asset stats
Stats.inc(
self.account,
today_tz(tz=self.config['TIMEZONE']),
{
'assets': 1,
'length': asset.meta['length']
}
)
self.write(asset.to_json_type())
def get_asset(self, projection=None):
"""Get the asset the task will be run against"""
return Asset.by_id(self.asset_id, projection=projection)
def _store_variation(
self,
config,
asset,
variation_name,
versioned,
ext,
meta,
file
):
"""
Store a new variation of the asset. This method both stores the
variation (and removes any existing variation with the same name) as
well as updating the asset's `variations` field with details of the
new variation.
"""
# Get the account and backend associated with the asset
account = Account.by_id(
asset.account,
projection={
'public_backend_settings': True,
'secure_backend_settings': True
}
)
if asset.secure:
backend = account.secure_backend
else:
backend = account.public_backend
assert backend, 'No backend configured for the asset'
# Ensure the asset's variation value is a dictionary (in case it's
# never been set before).
if not asset.variations:
asset.variations = {}
# Create the new variation
old_variation = asset.variations.get(variation_name)
new_variation = Variation(
content_type=mimetypes.guess_type(f'f.{ext}')[0] if ext else '',
ext=ext,
meta=meta,
version=(
Variation.next_version(
old_variation.version if old_variation else None
)
if versioned else None
)
)
# Store the new variation
new_store_key = new_variation.get_store_key(asset, variation_name)
file.seek(0)
backend.store(file, new_store_key)
# Add the new variation's details to the asset's `variations` field
asset.variations[variation_name] = new_variation
asset.modified = datetime.utcnow()
# Apply the updated to the database
Asset.get_collection().update(
(Q._id == asset._id).to_dict(),
{
'$set': {
f'variations.{variation_name}': \
new_variation.to_json_type(),
'modified': asset.modified
}
}
)
# Remove the existing variation
if old_variation:
old_store_key = old_variation.get_store_key(asset, variation_name)
if new_store_key != old_store_key:
backend.delete(old_store_key)
# Update the asset stats
new_length = new_variation.meta['length']
old_length = 0
if old_variation:
old_length = old_variation.meta['length']
Stats.inc(
account,
today_tz(tz=config['TIMEZONE']),
{
'variations': 0 if old_variation else 1,
'length': new_length - old_length
}
)
def purge():
"""Purge assets that have expired"""
# Get expired assets
now = time.time()
max_delete_period = 48 * 60 * 60
assets = Asset.many(And(
Q.expires <= now,
Q.expires > now - max_delete_period,
),
projection={
'expires': True,
'ext': True,
'meta.length': True,
'name': True,
'secure': True,
'uid': True,
'variations': {
'$sub.': Variation
},
'account': {
'$ref': Account,
'public_backend_settings': True,
'secure_backend_settings': True
}
})
# Delete the assets
for asset in assets:
variation_count = 0
length = asset.meta['length']
# Remove all stored files for the asset
if asset.secure:
backend = asset.account.secure_backend
else:
backend = asset.account.public_backend
if backend:
if asset.variations:
# Count the variations for the asset
variation_count = len(asset.variations)
for variation_name, variation in asset.variations.items():
# Tally up the assets total footprint
length += variation.meta['length']
# Remove variation files
backend.delete(
variation.get_store_key(asset, variation_name))
# Remove the asset file
backend.delete(asset.store_key)
# Delete the asset from the database
asset.delete()
# Update the asset stats
Stats.inc(asset.account, today_tz(), {
'assets': -1,
'variations': -variation_count,
'length': -length
})
async def put(self, uid):
"""Generate variations for an asset"""
asset = self.get_asset(uid,
projection={
'_id': True,
'type': True,
'expires': True
})
# Extract the variations from the variations argument
try:
raw_variations = json.loads(self.get_body_argument('variations'))
except ValueError:
raise APIError('invalid_request',
hint='Variations argument is not valid JSON.')
variations = self.validate_variations(asset.type, raw_variations)
# Add a set of tasks to generate the asset variations
notification_url = self.get_body_argument('notification_url', None)
tasks = []
task_names = []
for variation_name, transforms in variations.items():
task = GenerateVariationTask(self.account._id, asset._id,
variation_name, transforms,
notification_url)
if notification_url:
tasks.append(self.add_task_and_forget(task))
else:
tasks.append(self.add_task_and_wait(task))
task_names.append(variation_name)
if notification_url:
# Fire and forget
await asyncio.gather(*tasks)
self.finish()
else:
# Wait for response
events = await asyncio.gather(*tasks)
# Collect any errors
errors = {}
for i, event in enumerate(events):
if not event:
errors[task_names[i]] = ['Connection lost']
elif event.type == 'task_error':
errors[task_names[i]] = [event.reason]
if errors:
raise APIError('error', arg_errors=errors)
# Fetch the asset again now the variations have been generated
with Asset.with_options(read_preference=ReadPreference.PRIMARY):
asset = self.get_asset(uid,
projection={
'uid': True,
'expires': True,
'ext': True,
'meta': True,
'name': True,
'variations': {
'$sub.': Variation
}
})
# Handle image expiry
if not asset:
raise APIError(
'not_found',
hint=('Asset expired whilst variations where being '
'generated.'))
json_type = asset.to_json_type()
self.write({
'uid': json_type['uid'],
'variations': json_type['variations']
})
