def analyze(self, task):
"""Analyze an asset"""
asset = task.get_asset(
projection={
'variations': {'$sub.': Variation}
}
)
file = task.get_file()
history = []
for analyzer in task.get_analyzers(asset):
analyzer.analyze(self.config, asset, file, history)
history.append(analyzer)
if task.notification_url:
# POST the result to the notification URL
account = Account.by_id(
asset.account,
projection={'api_key': True}
)
task.post_notification(
account.api_key,
json.dumps(asset.to_json_type())
)
return {}
def trim_api_logs():
"""
Ensure the API logs for all accounts do not contain entries that exceed the
maximum retention period.
CRON: This task should be run daily during less active periods.
"""
trim_after = datetime.utcnow() \
- current_app.config['API_LOG_RETENTION_PERIOD']
for account_id in Account.ids():
keys = [f'log:succeeded:{account_id}', f'log:failed:{account_id}']
for key in keys:
log_entries = current_app.redis.lrange(key, 0, -1)
log_entries = [json.loads(e) for e in log_entries]
for i, log_entry in enumerate(log_entries):
called = datetime.utcfromtimestamp(log_entry['called'])
if called <= trim_after:
current_app.redis.ltrim(key, 0, i - 1)
break
def generate_variation(self, task):
"""Generate variation for the asset"""
asset = task.get_asset(
projection={
'variations': {
'$sub.': Variation
}
}
)
file = task.get_file()
native_file = None
history = []
for transform in task.get_transforms(asset):
native_file = transform.transform(
self.config,
asset,
file,
task.variation_name,
native_file,
history
)
history.append(transform)
if task.notification_url:
# POST the result to the notification URL
account = Account.by_id(
asset.account,
projection={'api_key': True}
)
task.post_notification(
account.api_key,
json.dumps(asset.to_json_type())
)
return {}
def get_backend(self, secure=False):
"""
Return the storage backend for the account. Raise an error if the
request backend isn't configured.
"""
backend_type = 'secure' if secure else 'public'
account = Account.by_id(
self.account._id,
projection={f'{backend_type}_backend_settings': True}
)
backend = getattr(account, f'{backend_type}_backend', None)
if not backend:
raise APIError(
'invalid_request',
hint=f'No backend configured for {backend_type} storage.'
)
return backend
def get_file(self):
"""Get the file for the asset the task will be run against"""
account = Account.by_id(self.account_id,
projection={
'public_backend_settings': True,
'secure_backend_settings': True
})
asset = self.get_asset(projection={
'secure': True,
'name': True,
'uid': True,
'ext': True
})
if asset.secure:
backend = account.secure_backend
else:
backend = account.public_backend
assert backend, 'No backend configured for the asset'
return backend.retrieve(asset.store_key)
def _store_variation(
self,
config,
asset,
variation_name,
versioned,
ext,
meta,
file
):
"""
Store a new variation of the asset. This method both stores the
variation (and removes any existing variation with the same name) as
well as updating the asset's `variations` field with details of the
new variation.
"""
# Get the account and backend associated with the asset
account = Account.by_id(
asset.account,
projection={
'public_backend_settings': True,
'secure_backend_settings': True
}
)
if asset.secure:
backend = account.secure_backend
else:
backend = account.public_backend
assert backend, 'No backend configured for the asset'
# Ensure the asset's variation value is a dictionary (in case it's
# never been set before).
if not asset.variations:
asset.variations = {}
# Create the new variation
old_variation = asset.variations.get(variation_name)
new_variation = Variation(
content_type=mimetypes.guess_type(f'f.{ext}')[0] if ext else '',
ext=ext,
meta=meta,
version=(
Variation.next_version(
old_variation.version if old_variation else None
)
if versioned else None
)
)
# Store the new variation
new_store_key = new_variation.get_store_key(asset, variation_name)
file.seek(0)
backend.store(file, new_store_key)
# Add the new variation's details to the asset's `variations` field
asset.variations[variation_name] = new_variation
asset.modified = datetime.utcnow()
# Apply the updated to the database
Asset.get_collection().update(
(Q._id == asset._id).to_dict(),
{
'$set': {
f'variations.{variation_name}': \
new_variation.to_json_type(),
'modified': asset.modified
}
}
)
# Remove the existing variation
if old_variation:
old_store_key = old_variation.get_store_key(asset, variation_name)
if new_store_key != old_store_key:
backend.delete(old_store_key)
# Update the asset stats
new_length = new_variation.meta['length']
old_length = 0
if old_variation:
old_length = old_variation.meta['length']
Stats.inc(
account,
today_tz(tz=config['TIMEZONE']),
{
'variations': 0 if old_variation else 1,
'length': new_length - old_length
}
)
async def prepare(self):
from blueprints.accounts.models import Account, Stats
# (Re)Set the account attribute
self._account = None
# (Re)Set the write log
self._write_log = ''
# (Re)Set the call timer
self._call_timer = time.time()
# Check a valid API key has been provided
api_key = self.request.headers.get('X-H51-APIKey')
if not api_key:
raise APIError('unauthorized', 'No authorization key provided.')
# Find the calling account
account = Account.one(
Q.api_key == api_key,
projection={
'api_allowed_ip_addresses': True,
'api_rate_limit_per_second': True
}
)
# Store a reference to the account document against the request handler
self._account = account
if not account:
raise APIError('unauthorized', 'API key not recognized.')
if account.api_allowed_ip_addresses:
# Check the caller's IP address is allowed
ip_address = self.request.headers.get('X-Real-Ip', '')
if ip_address not in account.api_allowed_ip_addresses:
raise APIError(
'forbidden',
(
f'The IP address {ip_address} is not allowed to call '
'the API for this account.'
)
)
# Record this request
rate_key = account.get_rate_key()
ttl = await self.redis.pttl(rate_key)
if ttl > 0:
await self.redis.incr(rate_key)
else:
multi = self.redis.multi_exec()
multi.incr(rate_key)
multi.expire(rate_key, 1)
await multi.execute()
# Apply rate limit
request_count = int((await self.redis.get(rate_key)) or 0)
rate_limit = account.api_rate_limit_per_second \
or self.config['API_RATE_LIMIT_PER_SECOND']
if request_count > rate_limit:
raise APIError('request_limit_exceeded')
# Set the remaining requests allowed this second in the response
# headers.
rate_key_reset = max(0, ttl / 1000.0)
rate_key_reset += time.time()
self.set_header('X-H51-RateLimit-Limit', str(rate_limit))
self.set_header(
'X-H51-RateLimit-Remaining',
str(rate_limit - request_count)
)
self.set_header('X-H51-RateLimit-Reset', str(rate_key_reset))
# Update the API call stats
Stats.inc(
account,
today_tz(tz=self.config['TIMEZONE']),
{'api_calls': 1}
)