def process(self, env, cb):
event = Event(env)
if event.event_type in CONTAINER_EVENTS:
mtime = event.when / 1000000.0 # convert to seconds
data = event.data
url = event.env.get('url')
body = dict()
if event.event_type == EventTypes.CONTAINER_STATE:
body['bytes'] = data.get('bytes-count', 0)
body['objects'] = data.get('object-count', 0)
body['mtime'] = mtime
elif event.event_type == EventTypes.CONTAINER_DELETED:
body['dtime'] = mtime
elif event.event_type == EventTypes.CONTAINER_NEW:
body['mtime'] = mtime
try:
with Timeout(ACCOUNT_TIMEOUT):
self.account.container_update(url.get('account'),
url.get('user'), body)
except Timeout as exc:
msg = 'account update failure: %s' % str(exc)
resp = EventError(event=Event(env), body=msg)
return resp(env, cb)
except ClientException as exc:
if (exc.http_status == 409
and "No update needed" in exc.message):
self.logger.info("Discarding event %s (%s): %s",
event.job_id, event.event_type,
exc.message)
else:
msg = 'account update failure: %s' % str(exc)
resp = EventError(event=Event(env), body=msg)
return resp(env, cb)
return self.app(env, cb)
def process(self, env, cb):
event = Event(env)
if event.event_type in CONTAINER_EVENTS:
mtime = event.when / 1000000.0 # convert to seconds
data = event.data
url = event.env.get('url')
body = dict()
if event.event_type == EventTypes.CONTAINER_STATE:
body['bytes'] = data.get('bytes-count', 0)
body['objects'] = data.get('object-count', 0)
body['mtime'] = mtime
elif event.event_type == EventTypes.CONTAINER_NEW:
body['mtime'] = mtime
try:
self.account.container_update(url.get('account'),
url.get('user'),
body,
read_timeout=ACCOUNT_TIMEOUT)
except OioTimeout as exc:
msg = 'account update failure: %s' % str(exc)
resp = EventError(event=Event(env), body=msg)
return resp(env, cb)
except ClientException as exc:
if (exc.http_status == 409
and "No update needed" in exc.message):
self.logger.info("Discarding event %s (%s): %s",
event.job_id, event.event_type,
exc.message)
else:
msg = 'account update failure: %s' % str(exc)
resp = EventError(event=Event(env), body=msg)
return resp(env, cb)
elif event.event_type == EventTypes.ACCOUNT_SERVICES:
url = event.env.get('url')
if isinstance(event.data, list):
# Legacy format: list of services
new_services = event.data
else:
# New format: dictionary with new and deleted services
new_services = event.data.get('services') or list()
m2_services = [x for x in new_services if x.get('type') == 'meta2']
if not m2_services:
# No service in charge, container has been deleted
self.account.container_update(
url.get('account'),
url.get('user'), {'dtime': event.when / 1000000.0},
read_timeout=ACCOUNT_TIMEOUT)
else:
self.account.account_create(url.get('account'),
read_timeout=ACCOUNT_TIMEOUT)
return self.app(env, cb)
def __call__(self, env, cb):
event = Event(env)
try:
res = self.process(event)
return res(env, cb)
except StopServe:
self.logger.info('Job %s not handled: the process is stopping',
event.job_id)
res = EventError(event=event, body='Process is stopping')
except Exception as err:
self.logger.exception('Job %s not handled: %s', event.job_id, err)
res = EventError(event=event, body='An error ocurred')
return res(env, cb)
def process(self, env, cb):
event = Event(env)
if event.event_type in CONTAINER_EVENTS:
uri = '/v1.0/account/container/update'
mtime = event.when / 1000000.0 # convert to seconds
data = event.data
url = event.env.get('url')
name = url.get('user')
account = url.get('account')
body = {'name': name}
if event.event_type == EventTypes.CONTAINER_STATE:
body['bytes'] = data.get('bytes-count', 0)
body['objects'] = data.get('object-count', 0)
body['mtime'] = mtime
elif event.event_type == EventTypes.CONTAINER_DELETED:
body['dtime'] = mtime
elif event.event_type == EventTypes.CONTAINER_NEW:
body['mtime'] = mtime
query = urlencode({'id': account})
p = urlparse('http://' + self.app.app.acct_addr)
try:
with Timeout(ACCOUNT_TIMEOUT):
resp, body = http_request(p.netloc, 'POST', uri,
query_string=query, body=body)
except Timeout as e:
msg = 'account update failure: %s' % str(e)
resp = EventError(event=Event(env), body=msg)
return resp(env, cb)
return self.app(env, cb)
def process(self, env, cb):
event = Event(env)
if event.event_type in CONTAINER_EVENTS:
mtime = event.when / 1000000.0 # convert to seconds
data = event.data
url = event.env.get('url')
body = {'name': url.get('user')}
if event.event_type == EventTypes.CONTAINER_STATE:
body['bytes'] = data.get('bytes-count', 0)
body['objects'] = data.get('object-count', 0)
body['mtime'] = mtime
elif event.event_type == EventTypes.CONTAINER_DELETED:
body['dtime'] = mtime
elif event.event_type == EventTypes.CONTAINER_NEW:
body['mtime'] = mtime
query = urlencode({'id': url.get('account')})
try:
with Timeout(ACCOUNT_TIMEOUT):
# TODO(FVE): fix and use AccountClient
_, _ = http_request(self.app_env['acct_addr'](),
'POST',
'/v1.0/account/container/update',
query_string=query,
body=body)
except Timeout as exc:
msg = 'account update failure: %s' % str(exc)
resp = EventError(event=Event(env), body=msg)
return resp(env, cb)
return self.app(env, cb)
def process(self, env, beanstalkd, cb):
event = Event(env)
url = env['url']
alias, content_header, properties = extract_data_from_event(env)
data = {
'id': url['content'],
'account': url['account'],
'container': url['user'],
'name': url['path'],
}
if all((alias, content_header)) \
and event.event_type == EventTypes.CONTENT_NEW:
data.update({
'md5Hash': content_header['hash'],
'contentType': content_header['mime-type'],
'policy': content_header['policy'],
'chunkMethod': content_header['chunk-method'],
'size': content_header['size'],
'creationTime': alias['ctime'],
'modificationTime': alias['mtime'],
'version': alias['version'],
'metadata': properties,
})
elif event.event_type not in (EventTypes.CONTENT_DELETED,
EventTypes.CONTENT_APPEND):
all_properties = self.container_client.content_get_properties(
account=url['account'],
reference=url['user'],
content=url['content'])
data.update({
'md5Hash': all_properties['hash'],
'contentType': all_properties['mime_type'],
'policy': all_properties['policy'],
'chunkMethod': all_properties['chunk_method'],
'size': all_properties['length'],
'creationTime': all_properties['ctime'],
'modificationTime': all_properties['mtime'],
'version': all_properties['version'],
'metadata': all_properties['properties'],
})
body = {
'eventId': env['job_id'],
'eventType': env['event'],
'timestamp': env['when'],
'data': data
}
try:
data = json.dumps(body)
self._request(data)
except exceptions.OioException as exc:
return EventError(event=event,
body='webhook error: %s' % exc)(env, beanstalkd,
cb)
return self.app(env, beanstalkd, cb)
def process(self, env, cb):
data = json.dumps(env)
try:
self.beanstalk.put(data)
except BeanstalkError as e:
msg = 'notify failure: %s' % str(e)
resp = EventError(event=Event(env), body=msg)
return resp(env, cb)
return self.app(env, cb)
def __call__(self, env, cb):
event = Event(env)
try:
res = self.process(event)
return res(env, cb)
except:
self.logger.exception('Error: An error occured')
res = EventError(event=event, body='An error ocurred')
return res(env, cb)
def process(self, env, cb):
data = json.dumps(env)
try:
# TODO we could retry the put
self.beanstalk.put(data)
except ConnectionError:
self.logger.warn("beanstalk notify failed")
except Exception as e:
self.logger.warn("failed to notify event: %s" % str(e))
return EventError(event=Event(env))(env, cb)
return self.app(env, cb)
def process(self, env, cb):
event = Event(env)
if self.should_notify(event):
try:
data = json.dumps(env)
self.beanstalk.put(data)
except BeanstalkError as err:
msg = 'notify failure: %s' % str(err)
resp = EventError(event=Event(env), body=msg)
return resp(env, cb)
return self.app(env, cb)
def process(self, env, cb):
event = Event(env)
if self._should_replicate(event.url.get('account'),
event.url.get('user')):
try:
data = json.dumps(env)
self.beanstalk.put(data)
except BeanstalkError as e:
msg = 'notify failure: %s' % str(e)
resp = EventError(event=Event(env), body=msg)
return resp(env, cb)
return self.app(env, cb)
def process(self, env, beanstalkd, cb):
event = Event(env)
if self.should_notify(event):
try:
# Encode without whitespace to make sure not
# to exceed the maximum size of the event (default: 65535)
data = json.dumps(env,
separators=(',', ':')) # compact encoding
self.beanstalk.put(data)
except BeanstalkError as err:
msg = 'notify failure: %s' % str(err)
resp = EventError(event=Event(env), body=msg)
return resp(env, beanstalkd, cb)
return self.app(env, beanstalkd, cb)
def process(self, env, beanstalkd, cb):
event = Event(env)
mtime = event.when // 1000000 # seconds
if event.event_type in CHUNK_EVENTS:
data = event.data
volume_id = data.get('volume_service_id') or data.get('volume_id')
container_id = data.get('container_id')
content_id = data.get('content_id')
chunk_id = data.get('chunk_id')
try:
if event.event_type == EventTypes.CHUNK_DELETED:
self._chunk_delete(event.reqid, volume_id, container_id,
content_id, chunk_id)
else:
args = {'mtime': mtime}
self._chunk_push(event.reqid, volume_id, container_id,
content_id, chunk_id, args)
except OioException as exc:
resp = EventError(event=event,
body="rdir update error: %s" % exc)
return resp(env, beanstalkd, cb)
elif event.event_type in SERVICE_EVENTS:
container_id = event.url['id']
container_url = '/'.join(
(event.url['ns'], event.url['account'], event.url['user']))
if event.event_type == EventTypes.ACCOUNT_SERVICES:
peers = event.data
for peer in peers:
self._service_push(event.reqid, peer['type'], peer['host'],
container_url, container_id, mtime)
elif event.event_type == EventTypes.META2_DELETED:
peer = event.data['peer']
self._service_delete(event.reqid, 'meta2', peer, container_url,
container_id)
elif event.event_type == EventTypes.CONTAINER_DELETED:
# TODO(adu): Delete when it will no longer be used
peers = event.data.get('peers') or list()
for peer in peers:
self._service_delete(event.reqid, 'meta2', peer,
container_url, container_id)
return self.app(env, beanstalkd, cb)
def process(self, env, cb):
event = Event(env)
if event.event_type in CHUNK_EVENTS:
data = event.data
volume_id = data.get('volume_id')
container_id = data.get('container_id')
content_id = data.get('content_id')
chunk_id = data.get('chunk_id')
try:
if event.event_type == EventTypes.CHUNK_DELETED:
self._chunk_delete(
volume_id, container_id, content_id, chunk_id)
else:
args = {
'mtime': event.when / 1000000, # seconds
}
self._chunk_push(
volume_id, container_id, content_id, chunk_id, args)
except OioException as exc:
resp = EventError(event=event,
body="rdir update error: %s" % exc)
return resp(env, cb)
return self.app(env, cb)
def process(self, env, cb):
event = Event(env)
headers = {
REQID_HEADER: event.reqid or request_id('account-update-')
}
try:
if event.event_type in CONTAINER_EVENTS:
mtime = event.when / 1000000.0 # convert to seconds
data = event.data
url = event.env.get('url')
body = dict()
if event.event_type == EventTypes.CONTAINER_STATE:
body['objects'] = data.get('object-count', 0)
body['bytes'] = data.get('bytes-count', 0)
body['damaged_objects'] = data.get('damaged-objects', 0)
body['missing_chunks'] = data.get('missing-chunks', 0)
body['mtime'] = mtime
elif event.event_type == EventTypes.CONTAINER_NEW:
body['mtime'] = mtime
self.account.container_update(
url.get('account'), url.get('user'), body,
connection_timeout=self.connection_timeout,
read_timeout=self.read_timeout, headers=headers)
elif event.event_type == EventTypes.ACCOUNT_SERVICES:
url = event.env.get('url')
if isinstance(event.data, list):
# Legacy format: list of services
new_services = event.data
else:
# New format: dictionary with new and deleted services
new_services = event.data.get('services') or list()
m2_services = [x for x in new_services
if x.get('type') == 'meta2']
if not m2_services:
# No service in charge, container has been deleted
self.account.container_update(
url.get('account'), url.get('user'),
{'dtime': event.when / 1000000.0},
connection_timeout=self.connection_timeout,
read_timeout=self.read_timeout, headers=headers)
else:
try:
self.account.account_create(
url.get('account'),
connection_timeout=self.connection_timeout,
read_timeout=self.read_timeout, headers=headers)
except OioTimeout as exc:
# The account will be autocreated by the next event,
# just warn and continue.
self.logger.warn(
'Failed to create account %s (reqid=%s): %s',
url.get('account'), headers[REQID_HEADER], exc)
except OioTimeout as exc:
msg = 'account update failure: %s' % str(exc)
resp = EventError(event=Event(env), body=msg)
return resp(env, cb)
except ClientException as exc:
if (exc.http_status == 409 and
"No update needed" in exc.message):
self.logger.info(
"Discarding event %s (job_id=%s, reqid=%s): %s",
event.job_id, headers[REQID_HEADER],
event.event_type, exc.message)
else:
msg = 'account update failure: %s' % str(exc)
resp = EventError(event=Event(env), body=msg)
return resp(env, cb)
return self.app(env, cb)
