class ContentReaperFilter(Filter):
"""Filter that deletes chunks on content deletion events"""
def init(self):
self.handlers = {
"plain": self._handle_rawx,
"ec": self._handle_rawx,
"backblaze": self._handle_b2,
}
self.blob_client = BlobClient(self.conf)
def _handle_rawx(self, url, chunks, content_headers, storage_method,
reqid):
cid = url.get('id')
headers = {REQID_HEADER: reqid, 'Connection': 'close'}
resps = self.blob_client.chunk_delete_many(chunks,
cid=cid,
headers=headers,
timeout=5.0)
for resp in resps:
if isinstance(resp, Exception):
self.logger.warn('failed to delete chunk %s (%s)',
resp.chunk.get('real_url', resp.chunk['url']),
resp)
elif resp.status not in (204, 404):
self.logger.warn('failed to delete chunk %s (HTTP %s)',
resp.chunk.get('real_url', resp.chunk['url']),
resp.status)
def _handle_b2(self, url, chunks, headers, storage_method, reqid):
meta = {'container_id': url['id']}
key_file = self.conf.get('key_file')
b2_creds = BackblazeUtils.get_credentials(storage_method, key_file)
try:
BackblazeDeleteHandler(meta, chunks, b2_creds).delete()
except OioException as exc:
self.logger.warn('delete failed: %s' % str(exc))
def _load_handler(self, chunk_method):
storage_method = STORAGE_METHODS.load(chunk_method)
handler = self.handlers.get(storage_method.type)
if not handler:
raise OioException("No handler found for chunk method [%s]" %
chunk_method)
return handler, storage_method
def process(self, env, cb):
event = Event(env)
if event.event_type == EventTypes.CONTENT_DELETED:
url = event.env.get('url')
chunks = []
content_headers = list()
for item in event.data:
if item.get('type') == 'chunks':
# The event contains "id" whereas the API uses "url".
item['url'] = item['id']
chunks.append(item)
if item.get("type") == 'contents_headers':
content_headers.append(item)
if len(chunks):
reqid = event.reqid or request_id('content-cleaner-')
if not content_headers:
chunk_method = guess_storage_method(chunks[0]['id']) + '/'
else:
chunk_method = content_headers[0]['chunk-method']
handler, storage_method = self._load_handler(chunk_method)
handler(url, chunks, content_headers, storage_method, reqid)
return self.app(env, cb)
return self.app(env, cb)
class ContentReaperFilter(Filter):
"""Filter that deletes chunks on content deletion events"""
def init(self):
self.handlers = {
"plain": self._handle_rawx,
"ec": self._handle_rawx,
}
kwargs = {
k: v
for k, v in self.conf.items() if k in URLLIB3_POOLMANAGER_KWARGS
}
self.blob_client = BlobClient(self.conf, logger=self.logger, **kwargs)
self.chunk_concurrency = int(self.conf.get('concurrency', 3))
self.chunk_timeout = float(self.conf.get('timeout', 5.0))
def _handle_rawx(self, url, chunks, content_headers, storage_method,
reqid):
cid = url.get('id')
headers = {REQID_HEADER: reqid, 'Connection': 'close'}
resps = self.blob_client.chunk_delete_many(
chunks,
cid=cid,
headers=headers,
concurrency=self.chunk_concurrency,
timeout=self.chunk_timeout)
for resp in resps:
if isinstance(resp, Exception):
self.logger.warn('failed to delete chunk %s (%s)',
resp.chunk.get('real_url', resp.chunk['url']),
resp)
elif resp.status not in (204, 404):
self.logger.warn('failed to delete chunk %s (HTTP %s)',
resp.chunk.get('real_url', resp.chunk['url']),
resp.status)
def _load_handler(self, chunk_method):
storage_method = STORAGE_METHODS.load(chunk_method)
handler = self.handlers.get(storage_method.type)
if not handler:
raise OioException("No handler found for chunk method [%s]" %
chunk_method)
return handler, storage_method
def process(self, env, beanstalkd, cb):
event = Event(env)
if event.event_type == EventTypes.CONTENT_DELETED:
url = event.env.get('url')
chunks = []
content_headers = list()
for item in event.data:
if item.get('type') == 'chunks':
# The event contains "id" whereas the API uses "url".
item['url'] = item['id']
chunks.append(item)
if item.get("type") == 'contents_headers':
content_headers.append(item)
if len(chunks):
reqid = event.reqid or request_id('content-cleaner-')
if not content_headers:
chunk_method = guess_storage_method(chunks[0]['id']) + '/'
else:
chunk_method = content_headers[0]['chunk-method']
handler, storage_method = self._load_handler(chunk_method)
handler(url, chunks, content_headers, storage_method, reqid)
return self.app(env, beanstalkd, cb)
return self.app(env, beanstalkd, cb)
