def post(self): """ Removes any existing timeout for one or more existing assets making the assets persistent. """ assets = self.get_assets(projection={'uid': True}) # Clear the expires timestamp (if there is one) for the assets Asset.get_collection().update(In(Q._id, [a._id for a in assets]).to_dict(), { '$set': { 'modified': datetime.utcnow() }, '$unset': { 'expires': '' } }, multi=True) self.write( {'results': [{ 'uid': a.uid, 'expires': None } for a in assets]})
def delete_document(state): Asset.get_collection().update_many( (Q.account == state.account._id).to_dict(), { '$set': { 'expires': time.time() - 1, 'modified': datetime.utcnow() } } ) generic.delete['post'].super(state)
def post(self): """ Removes any existing timeout for one or more existing assets making the assets persistent. Set a timeout for one or more assets. After the timeout has expired the assets will be automatically deleted. NOTE: The files associated with expired assets are delete periodically and therefore may still temporarily be available after the asset has expired and is not longer available via the API. """ assets = self.get_assets(projection={'uid': True}) # Validate the arguments form = ExpireForm(to_multi_dict(self.request.body_arguments)) if not form.validate(): raise APIError( 'invalid_request', arg_errors=form.errors ) form_data = form.data # Update the expires timestamp for the asset expires = time.time() + form_data['seconds'] Asset.get_collection().update( In(Q._id, [a._id for a in assets]).to_dict(), { '$set': { 'expires': time.time() + form_data['seconds'], 'modified': datetime.utcnow() } }, multi=True ) self.write({ 'results': [ { 'uid': a.uid, 'expires': expires } for a in assets ] })
def uid_to_object_id(uid): """Coerce a UID for an asset to the asset's ObjectId""" ids = Asset.ids(Q.uid == uid) if not ids: raise ValueError('Not a valid uid.') return ids[0]
def post(self, uid): """ Removes any existing timeout for an asset making the asset persistent. """ asset = self.get_asset(uid, projection={'uid': True, 'expires': True}) # Clear the expires timestamp (if there is one) Asset.get_collection().update({'_id': asset._id}, { '$set': { 'modified': datetime.utcnow() }, '$unset': { 'expires': '' } }) self.write({'uid': asset.uid, 'expires': None})
def _add_to_meta(self, asset, data): """Add the specified data to the asset's meta""" # Modify the asset instance if self.asset_type not in asset.meta: asset.meta[self.asset_type] = {} asset.meta[self.asset_type][self.name] = data asset.modified = datetime.utcnow() # Apply the updated to the database Asset.get_collection().update( (Q._id == asset._id).to_dict(), { '$set': { f'meta.{self.asset_type}.{self.name}': data, 'modified': asset.modified } } )
async def delete(self, uid, variation_name): """Remove the variation from the asset""" asset = self.get_asset(uid, projection={ 'expires': True, 'name': True, 'secure': True, 'uid': True, f'variations.{variation_name}': { '$sub': Variation } }) variation = self.get_variation(asset, variation_name) # Remove file for the variation backend = self.get_backend(asset.secure) await backend.async_delete(variation.get_store_key( asset, variation_name), loop=asyncio.get_event_loop()) # Remove the variation from the asset Asset.get_collection().update( (Q._id == asset._id).to_dict(), {'$unset': { f'variations.{variation_name}': '' }}) # Update the asset stats Stats.inc(self.account, today_tz(tz=self.config['TIMEZONE']), { 'variations': -1, 'length': -variation.meta['length'] }) self.set_status(204) self.finish()
def not_expired(**view_args): """ This action is only available against assets that have not expired. """ asset_id = view_args.get('asset', request.args.get('asset')) if asset_id: asset = Asset.by_id(ObjectId(asset_id), projection={'expires': True}) if asset: return not asset.expired return False
def get_asset(self, uid, projection=None): """ Get the asset for the given `uid` and raise an error if no asset is found. """ assert not projection or 'expires' in projection, \ '`expires` must be included by the projection' # Fetch the asset asset = Asset.one(And(Q.account == self.account, Q.uid == uid), projection=(projection or self.DEFAULT_PROJECTION)) if not asset or asset.expired: raise APIError('not_found', hint=f'Asset not found for uid: {uid}.') return asset
def get_assets(self, projection=None): """ Return a list of assets given (as JSON) in the form argument `assets`. """ uids = self.get_body_arguments('uids') if not uids: raise APIError( 'invalid_request', hint=f'No uids provided.' ) # Fetch the asset return Asset.many( And( Q.account == self.account, In(Q.uid, uids), Not(Q.expires <= time.time()) ), projection=(projection or self.DEFAULT_PROJECTION) )
async def put(self): """Generate variations for one or more assets""" assets = self.get_assets(projection={ '_id': True, 'type': True, 'uid': True }) # Extract the variations from the variations argument try: raw_variations = json.loads(self.get_body_argument('variations')) except ValueError: raise APIError('invalid_request', hint='Variations argument is not valid JSON.') # Peek to determine if the user wants the variations applied globally # or locally. if self.get_body_argument('local', False): # Variations must be defined for each uid uids = set(self.get_body_arguments('uids')) variation_keys = set(list(raw_variations.keys())) if uids != variation_keys: raise APIError('invalid_request', hint='Each uid must be assigned a variation.') variations = { a.uid: self.validate_variations(a.type, raw_variations[a.uid]) for a in assets } else: # Global application # Ensure all assets are of the same type asset_types = set([a.type for a in assets]) if len(asset_types) > 1: raise APIError('invalid_request', hint='All assets must be of the same type.') local_variations = self.validate_variations( asset_types.pop(), raw_variations) variations = {a.uid: local_variations for a in assets} # Add a set of tasks to generate the asset variations notification_url = self.get_body_argument('notification_url', None) tasks = [] task_names = [] for asset in assets: local_variations = variations[asset.uid] for variation_name, transforms in local_variations.items(): task = GenerateVariationTask(self.account._id, asset._id, variation_name, transforms, notification_url) if notification_url: tasks.append(self.add_task_and_forget(task)) else: tasks.append(self.add_task_and_wait(task)) task_names.append(f'{asset.uid}:{variation_name}') if notification_url: # Fire and forget await asyncio.gather(*tasks) self.finish() else: # Wait for response events = await asyncio.gather(*tasks) # Collect any errors errors = {} for i, event in enumerate(events): if not event: errors[task_names[i]] = ['Connection lost'] elif event.type == 'task_error': errors[task_names[i]] = [event.reason] if errors: raise APIError('error', arg_errors=errors) # Fetch the asset again now the variations have been generated with Asset.with_options(read_preference=ReadPreference.PRIMARY): assets = self.get_assets( projection={ 'uid': True, 'expires': True, 'ext': True, 'meta': True, 'name': True, 'variations': { '$sub.': Variation } }) results = [a.to_json_type() for a in assets] self.write({ 'results': [{ 'uid': r['uid'], 'variations': r['variations'] } for r in results] })
async def post(self, uid): """Analyze the asset for additional meta data""" asset = self.get_asset( uid, projection={ '_id': True, 'type': True, 'expires': True } ) # Extract the analyzers from the request body try: raw_analyzers = json.loads(self.get_body_argument('analyzers')) except: raise APIError( 'invalid_request', hint='Analyzers argument is not valid JSON.' ) analyzers = self.validate_analyzers(asset.type, raw_analyzers) # Add a task to perform the asset analysis notification_url = self.get_body_argument('notification_url', None) task = AnalyzeTask( self.account._id, asset._id, analyzers, notification_url ) if notification_url: # Fire and forget await self.add_task_and_forget(task) self.finish() else: # Wait for response event = await self.add_task_and_wait(task) if not event: raise APIError('error', 'Connection lost') elif event.type == 'task_error': raise APIError('error', event.reason) # Fetch the asset again now the analysis is complete with Asset.with_options(read_preference=ReadPreference.PRIMARY): asset = self.get_asset( uid, projection={ 'uid': True, 'expires': True, 'ext': True, 'meta': True, 'name': True } ) # Handle image expiry if not asset: raise APIError( 'not_found', hint='Asset expired whilst being analyzed' ) json_type = asset.to_json_type() self.write({ 'uid': json_type['uid'], 'meta': json_type['meta'] })
async def post(self): """Analyze one for more asset for additional meta data""" assets = self.get_assets( projection={ '_id': True, 'type': True, 'expires': True, 'uid': True } ) # Extract the analyzers from the request body try: raw_analyzers = json.loads(self.get_body_argument('analyzers')) except: raise APIError( 'invalid_request', hint='Analyzers argument is not valid JSON.' ) # Peek to determine if the user wants the variations applied globally # or locally. if self.get_body_argument('local', False): # Variations must be defined for each uid uids = set(self.get_body_arguments('uids')) analyzer_keys = set(list(raw_analyzers.keys())) if uids != analyzer_keys: raise APIError( 'invalid_request', hint='Each uid must be assigned a list of analyzers.' ) analyzers = { a.uid: self.validate_analyzers(a.type, raw_analyzers[a.uid]) for a in assets } else: # Global application # Ensure all assets are the same type / base type (file) asset_types = set([a.type for a in assets if a.type != 'file']) if len(asset_types) > 1: raise APIError( 'invalid_request', hint=( 'All assets must be of the same type / base type ' '(file)' ) ) local_analyzers = self.validate_analyzers( asset_types.pop(), raw_analyzers ) analyzers = {a.uid: local_analyzers for a in assets} # Add a set of tasks to generate the asset variations notification_url = self.get_body_argument('notification_url', None) tasks = [] task_names = [] for asset in assets: task = AnalyzeTask( self.account._id, asset._id, analyzers[asset.uid], notification_url ) if notification_url: tasks.append(self.add_task_and_forget(task)) else: tasks.append(self.add_task_and_wait(task)) task_names.append(asset.uid) if notification_url: # Fire and forget await self.add_task_and_forget(task) self.finish() else: # Wait for response events = await asyncio.gather(*tasks) # Collect any errors errors = {} for i, event in enumerate(events): if not event: errors[task_names[i]] = ['Connection lost'] elif event.type == 'task_error': errors[task_names[i]] = [event.reason] if errors: raise APIError('error', arg_errors=errors) # Fetch the assets again now the analysis is complete with Asset.with_options(read_preference=ReadPreference.PRIMARY): assets = self.get_assets( projection={ 'uid': True, 'expires': True, 'ext': True, 'meta': True, 'name': True } ) # Handle image expiry if not asset: raise APIError( 'not_found', hint='Asset expired whilst being analyzed' ) results = [a.to_json_type() for a in assets] self.write({ 'results': [ { 'uid': r['uid'], 'meta': r['meta'] } for r in results ] })
async def put(self): """Store the uploaded file as an asset""" # Make sure a file was received files = self.request.files.get('file') if not files: raise APIError( 'invalid_request', arg_errors={'file': ['No file received.']} ) file = files[0] # Validate the arguments form = PutForm(to_multi_dict(self.request.body_arguments)) if not form.validate(): raise APIError( 'invalid_request', arg_errors=form.errors ) if self.config['ANTI_VIRUS_ENABLED']: # Check the file for viruses av_client = clamd.ClamdUnixSocket( self.config['ANTI_VIRUS_CLAMD_PATH'] ) av_scan_result = av_client.instream(io.BytesIO(file.body)) if av_scan_result['stream'][0] == 'FOUND': raise APIError( 'invalid_request', arg_errors={ 'file': ['File appears to be a virus.'] } ) form_data = form.data # Create a name for the asset fname, fext = os.path.splitext(file.filename) name = slugify( form_data['name'] or fname, regex_pattern=ALLOWED_SLUGIFY_CHARACTERS, max_length=200 ) # Determine the files extension ext = fext[1:] if fext else imghdr.what(file.filename, file.body) # Determine the asset type/content type for the image content_type = mimetypes.guess_type(f'f.{ext}')[0] \ or 'application/octet-stream' asset_type = self.config['CONTENT_TYPE_TO_TYPES'].get( content_type, 'file' ) # Build the meta data for the asset meta = { 'filename': file.filename, 'length': len(file.body) } if asset_type == 'audio': try: au_file = io.BytesIO(file.body) au_file.name = file.filename au = mutagen.File(au_file) except: raise APIError( 'invalid_request', arg_errors={ 'file': ['Unable to open the file as an audio file.'] } ) if au is not None: meta['audio'] = { 'channels': getattr(au.info, 'channels', -1), 'length': getattr(au.info, 'length', -1), 'mode': { 0: 'stereo', 1: 'joint_stereo', 2: 'dual_channel', 3: 'mono' }.get(getattr(au.info, 'mode', ''), ''), 'sample_rate': getattr(au.info, 'sample_rate', -1) } if asset_type == 'image': im = None try: im = Image.open(io.BytesIO(file.body)) meta['image'] = { 'mode': im.mode, 'size': im.size } except: raise APIError( 'invalid_request', arg_errors={ 'file': ['Unable to open the file as an image.'] } ) finally: if im: im.close() # Create the asset asset = Asset( uid=Asset.generate_uid(), account=self.account, secure=form_data['secure'], name=name, ext=ext, type=asset_type, content_type=content_type, expires=(time.time() + form_data['expire']) if form_data['expire'] else None, meta=meta ) # Store the file backend = self.get_backend(asset.secure) await backend.async_store( io.BytesIO(file.body), asset.store_key, loop=asyncio.get_event_loop() ) # Save the asset asset.insert() # Update the asset stats Stats.inc( self.account, today_tz(tz=self.config['TIMEZONE']), { 'assets': 1, 'length': asset.meta['length'] } ) self.write(asset.to_json_type())
def get_asset(self, projection=None): """Get the asset the task will be run against""" return Asset.by_id(self.asset_id, projection=projection)
def _store_variation( self, config, asset, variation_name, versioned, ext, meta, file ): """ Store a new variation of the asset. This method both stores the variation (and removes any existing variation with the same name) as well as updating the asset's `variations` field with details of the new variation. """ # Get the account and backend associated with the asset account = Account.by_id( asset.account, projection={ 'public_backend_settings': True, 'secure_backend_settings': True } ) if asset.secure: backend = account.secure_backend else: backend = account.public_backend assert backend, 'No backend configured for the asset' # Ensure the asset's variation value is a dictionary (in case it's # never been set before). if not asset.variations: asset.variations = {} # Create the new variation old_variation = asset.variations.get(variation_name) new_variation = Variation( content_type=mimetypes.guess_type(f'f.{ext}')[0] if ext else '', ext=ext, meta=meta, version=( Variation.next_version( old_variation.version if old_variation else None ) if versioned else None ) ) # Store the new variation new_store_key = new_variation.get_store_key(asset, variation_name) file.seek(0) backend.store(file, new_store_key) # Add the new variation's details to the asset's `variations` field asset.variations[variation_name] = new_variation asset.modified = datetime.utcnow() # Apply the updated to the database Asset.get_collection().update( (Q._id == asset._id).to_dict(), { '$set': { f'variations.{variation_name}': \ new_variation.to_json_type(), 'modified': asset.modified } } ) # Remove the existing variation if old_variation: old_store_key = old_variation.get_store_key(asset, variation_name) if new_store_key != old_store_key: backend.delete(old_store_key) # Update the asset stats new_length = new_variation.meta['length'] old_length = 0 if old_variation: old_length = old_variation.meta['length'] Stats.inc( account, today_tz(tz=config['TIMEZONE']), { 'variations': 0 if old_variation else 1, 'length': new_length - old_length } )
def purge(): """Purge assets that have expired""" # Get expired assets now = time.time() max_delete_period = 48 * 60 * 60 assets = Asset.many(And( Q.expires <= now, Q.expires > now - max_delete_period, ), projection={ 'expires': True, 'ext': True, 'meta.length': True, 'name': True, 'secure': True, 'uid': True, 'variations': { '$sub.': Variation }, 'account': { '$ref': Account, 'public_backend_settings': True, 'secure_backend_settings': True } }) # Delete the assets for asset in assets: variation_count = 0 length = asset.meta['length'] # Remove all stored files for the asset if asset.secure: backend = asset.account.secure_backend else: backend = asset.account.public_backend if backend: if asset.variations: # Count the variations for the asset variation_count = len(asset.variations) for variation_name, variation in asset.variations.items(): # Tally up the assets total footprint length += variation.meta['length'] # Remove variation files backend.delete( variation.get_store_key(asset, variation_name)) # Remove the asset file backend.delete(asset.store_key) # Delete the asset from the database asset.delete() # Update the asset stats Stats.inc(asset.account, today_tz(), { 'assets': -1, 'variations': -variation_count, 'length': -length })
async def put(self, uid): """Generate variations for an asset""" asset = self.get_asset(uid, projection={ '_id': True, 'type': True, 'expires': True }) # Extract the variations from the variations argument try: raw_variations = json.loads(self.get_body_argument('variations')) except ValueError: raise APIError('invalid_request', hint='Variations argument is not valid JSON.') variations = self.validate_variations(asset.type, raw_variations) # Add a set of tasks to generate the asset variations notification_url = self.get_body_argument('notification_url', None) tasks = [] task_names = [] for variation_name, transforms in variations.items(): task = GenerateVariationTask(self.account._id, asset._id, variation_name, transforms, notification_url) if notification_url: tasks.append(self.add_task_and_forget(task)) else: tasks.append(self.add_task_and_wait(task)) task_names.append(variation_name) if notification_url: # Fire and forget await asyncio.gather(*tasks) self.finish() else: # Wait for response events = await asyncio.gather(*tasks) # Collect any errors errors = {} for i, event in enumerate(events): if not event: errors[task_names[i]] = ['Connection lost'] elif event.type == 'task_error': errors[task_names[i]] = [event.reason] if errors: raise APIError('error', arg_errors=errors) # Fetch the asset again now the variations have been generated with Asset.with_options(read_preference=ReadPreference.PRIMARY): asset = self.get_asset(uid, projection={ 'uid': True, 'expires': True, 'ext': True, 'meta': True, 'name': True, 'variations': { '$sub.': Variation } }) # Handle image expiry if not asset: raise APIError( 'not_found', hint=('Asset expired whilst variations where being ' 'generated.')) json_type = asset.to_json_type() self.write({ 'uid': json_type['uid'], 'variations': json_type['variations'] })