def on_target_completed(self, target_type, target, delivery, start, end, target_ok, target_self_info, err_info=None):
now = datetime.utcnow().isoformat()
task_id = delivery['task_id']
lock_name = '{}{}'.format(KVDB.LOCK_DELIVERY, task_id)
total_time = str(end - start)
with Lock(lock_name, self.delivery_lock_timeout, LOCK_TIMEOUT, self.kvdb.conn):
entry_ctx = dumps({
'target_self_info': target_self_info,
'total_time': total_time
})
with closing(self.odb.session()) as session:
delivery = session.merge(self.get_delivery(task_id))
delivery.state = DELIVERY_STATE.IN_PROGRESS_TARGET_OK if target_ok else DELIVERY_STATE.IN_PROGRESS_TARGET_FAILURE
delivery.last_used = now
delivery.definition.last_used = now
delivery.target_count += 1
history = DeliveryHistory()
history.task_id = task_id
history.entry_type = DELIVERY_HISTORY_ENTRY.TARGET_OK if target_ok else DELIVERY_HISTORY_ENTRY.TARGET_FAILURE
history.entry_time = now
history.entry_ctx = entry_ctx
history.delivery = delivery
history.resubmit_count = delivery.resubmit_count
session.add(delivery)
session.add(history)
session.commit()
self._invoke_callbacks(target, target_type, delivery, target_ok, False, DELIVERY_CALLBACK_INVOKER.TARGET)
def _on_in_doubt(self, item, delivery, now):
""" Delivery enteres an in-doubt state.
"""
with closing(self.odb.session()) as session:
delivery = session.merge(delivery)
delivery.state = DELIVERY_STATE.IN_DOUBT
delivery.last_used = now
delivery.definition.last_used = now
history = DeliveryHistory()
history.task_id = delivery.task_id
history.entry_type = DELIVERY_HISTORY_ENTRY.ENTERED_IN_DOUBT
history.entry_time = now
history.entry_ctx = DELIVERY_HISTORY_ENTRY.NONE
history.delivery = delivery
history.resubmit_count = delivery.resubmit_count
session.add(delivery)
session.add(history)
session.commit()
self._invoke_callbacks(item.target, item.target_type, delivery, False, True, DELIVERY_CALLBACK_INVOKER.SOURCE)
msg = 'Delivery [%s] is in-doubt (source/target %s/%s)'
self.logger.warn(msg, item.log_name, delivery.source_count, delivery.target_count)
def on_target_completed(self, target_type, target, delivery, start, end, target_ok, target_self_info, err_info=None):
now = datetime.utcnow().isoformat()
task_id = delivery['task_id']
lock_name = '{}{}'.format(KVDB.LOCK_DELIVERY, task_id)
total_time = str(end - start)
with Lock(lock_name, self.delivery_lock_timeout, LOCK_TIMEOUT, self.kvdb.conn):
entry_ctx = dumps({
'target_self_info': target_self_info,
'total_time': total_time
})
with closing(self.odb.session()) as session:
delivery = session.merge(self.get_delivery(task_id))
delivery.state = DELIVERY_STATE.IN_PROGRESS_TARGET_OK if target_ok else DELIVERY_STATE.IN_PROGRESS_TARGET_FAILURE
delivery.last_used = now
delivery.definition.last_used = now
delivery.target_count += 1
history = DeliveryHistory()
history.task_id = task_id
history.entry_type = DELIVERY_HISTORY_ENTRY.TARGET_OK if target_ok else DELIVERY_HISTORY_ENTRY.TARGET_FAILURE
history.entry_time = now
history.entry_ctx = entry_ctx
history.delivery = delivery
history.resubmit_count = delivery.resubmit_count
session.add(delivery)
session.add(history)
session.commit()
self._invoke_callbacks(target, target_type, delivery, target_ok, False, DELIVERY_CALLBACK_INVOKER.TARGET)
def _on_in_doubt(self, item, delivery, now):
""" Delivery enteres an in-doubt state.
"""
with closing(self.odb.session()) as session:
delivery = session.merge(delivery)
delivery.state = DELIVERY_STATE.IN_DOUBT
delivery.last_used = now
delivery.definition.last_used = now
history = DeliveryHistory()
history.task_id = delivery.task_id
history.entry_type = DELIVERY_HISTORY_ENTRY.ENTERED_IN_DOUBT
history.entry_time = now
history.entry_ctx = DELIVERY_HISTORY_ENTRY.NONE
history.delivery = delivery
history.resubmit_count = delivery.resubmit_count
session.add(delivery)
session.add(history)
session.commit()
self._invoke_callbacks(item.target, item.target_type, delivery, False, True, DELIVERY_CALLBACK_INVOKER.SOURCE)
msg = 'Delivery [%s] is in-doubt (source/target %s/%s)'
self.logger.warn(msg, item.log_name, delivery.source_count, delivery.target_count)
def _history_from_source(self, delivery, item, now, entry_type):
history = DeliveryHistory()
history.task_id = item.task_id
history.entry_type = entry_type
history.entry_time = now
history.entry_ctx = DELIVERY_HISTORY_ENTRY.NONE
history.delivery = delivery
history.resubmit_count = delivery.resubmit_count
return history
def _history_from_source(self, delivery, item, now, entry_type):
history = DeliveryHistory()
history.task_id = item.task_id
history.entry_type = entry_type
history.entry_time = now
history.entry_ctx = DELIVERY_HISTORY_ENTRY.NONE
history.delivery = delivery
history.resubmit_count = delivery.resubmit_count
return history
def finish_delivery(self, delivery, target_ok, now_dt, item):
""" Called after running out of attempts to deliver the payload.
"""
with closing(self.odb.session()) as session:
delivery = session.merge(delivery)
if target_ok:
msg_prefix = 'Confirmed delivery'
log_func = self.logger.info
expires = delivery.definition.expire_arch_succ_after
delivery_state = DELIVERY_STATE.CONFIRMED
history_entry_type = DELIVERY_HISTORY_ENTRY.ENTERED_CONFIRMED
else:
msg_prefix = 'Delivery failed'
log_func = self.logger.warn
expires = delivery.definition.expire_arch_fail_after
delivery_state = DELIVERY_STATE.FAILED
history_entry_type = DELIVERY_HISTORY_ENTRY.ENTERED_FAILED
delivery.state = delivery_state
delivery.last_used = now_dt
delivery.definition.last_used = now_dt
history = DeliveryHistory()
history.task_id = delivery.task_id
history.entry_type = history_entry_type
history.entry_time = now_dt
history.entry_ctx = DELIVERY_HISTORY_ENTRY.NONE
history.delivery = delivery
history.resubmit_count = delivery.resubmit_count
session.add(delivery)
session.add(history)
session.commit()
self._invoke_callbacks(item.target, item.target_type, delivery,
target_ok, False,
DELIVERY_CALLBACK_INVOKER.SOURCE)
msg = '{} [{}] after {}/{} attempts, archive expires in {} hour(s) ({} UTC)'.format(
msg_prefix, item.log_name, delivery.source_count,
delivery.definition.retry_repeats, expires,
now_dt + timedelta(hours=expires))
log_func(msg)
def finish_delivery(self, delivery, target_ok, now_dt, item):
""" Called after running out of attempts to deliver the payload.
"""
with closing(self.odb.session()) as session:
delivery = session.merge(delivery)
if target_ok:
msg_prefix = 'Confirmed delivery'
log_func = self.logger.info
expires = delivery.definition.expire_arch_succ_after
delivery_state = DELIVERY_STATE.CONFIRMED
history_entry_type = DELIVERY_HISTORY_ENTRY.ENTERED_CONFIRMED
else:
msg_prefix = 'Delivery failed'
log_func = self.logger.warn
expires = delivery.definition.expire_arch_fail_after
delivery_state = DELIVERY_STATE.FAILED
history_entry_type = DELIVERY_HISTORY_ENTRY.ENTERED_FAILED
delivery.state = delivery_state
delivery.last_used = now_dt
delivery.definition.last_used = now_dt
history = DeliveryHistory()
history.task_id = delivery.task_id
history.entry_type = history_entry_type
history.entry_time = now_dt
history.entry_ctx = DELIVERY_HISTORY_ENTRY.NONE
history.delivery = delivery
history.resubmit_count = delivery.resubmit_count
session.add(delivery)
session.add(history)
session.commit()
self._invoke_callbacks(item.target, item.target_type, delivery, target_ok, False, DELIVERY_CALLBACK_INVOKER.SOURCE)
msg = '{} [{}] after {}/{} attempts, archive expires in {} hour(s) ({} UTC)'.format(
msg_prefix, item.log_name, delivery.source_count, delivery.definition.retry_repeats,
expires, now_dt + timedelta(hours=expires))
log_func(msg)
def update_delivery(self, task_id, payload, args, kwargs):
with closing(self.odb.session()) as session:
now = datetime.utcnow().isoformat()
delivery = session.merge(self.get_delivery(task_id))
old_payload = delivery.payload.payload
old_args = delivery.args
old_kwargs = delivery.kwargs
delivery.payload.payload = payload
delivery.args = args
delivery.kwargs = kwargs
delivery.definition.last_used = now
delivery.target_count += 1
entry_ctx = dumps({
'old_payload': old_payload,
'old_args': old_args,
'old_kwargs': old_kwargs,
})
history = DeliveryHistory()
history.task_id = task_id
history.entry_type = DELIVERY_HISTORY_ENTRY.UPDATED
history.entry_time = now
history.entry_ctx = entry_ctx
history.delivery = delivery
history.resubmit_count = delivery.resubmit_count
session.add(delivery)
session.add(history)
session.commit()
self.logger.info('Updated delivery [%s], history.id [%s]', task_id,
history.id)
def update_delivery(self, task_id, payload, args, kwargs):
with closing(self.odb.session()) as session:
now = datetime.utcnow().isoformat()
delivery = session.merge(self.get_delivery(task_id))
old_payload = delivery.payload.payload
old_args = delivery.args
old_kwargs = delivery.kwargs
delivery.payload.payload = payload
delivery.args = args
delivery.kwargs = kwargs
delivery.definition.last_used = now
delivery.target_count += 1
entry_ctx = dumps({
'old_payload': old_payload,
'old_args': old_args,
'old_kwargs': old_kwargs,
})
history = DeliveryHistory()
history.task_id = task_id
history.entry_type = DELIVERY_HISTORY_ENTRY.UPDATED
history.entry_time = now
history.entry_ctx = entry_ctx
history.delivery = delivery
history.resubmit_count = delivery.resubmit_count
session.add(delivery)
session.add(history)
session.commit()
self.logger.info('Updated delivery [%s], history.id [%s]', task_id, history.id)