def send_update(update, is_final):
try:
return rem_proxy.update_graph(
opts.task_id,
rpc_server_addr,
update,
is_final
)
except Exception as e:
if isinstance(e, socket.error):
logging.warning("on_notifier_fail: %s" % e)
else:
logging.exception("on_notifier_fail")
if is_xmlrpc_exception(e, WrongTaskIdError):
try:
on_notifier_fail()
except:
logging.exception("on_notifier_fail")
raise RuntimeError("Failed to send data to rem server: %s" % e.faultString)
else:
# FIXME Actually if isinstance(e, xmlrpclib.Fault) then not retriable
# but not fatal as WrongTaskIdError
raise RemNotifier.RetriableError(str(e))
def _do_stop_packet(self, pck):
task_id = pck._sandbox_task_id
stop_mode = pck._target_stop_mode
def reschedule_if_need():
if pck._state != RemotePacketState.STARTED:
return
if pck._target_stop_mode > stop_mode:
return
if pck._sched:
# TODO Assert that we have another _do_stop_packet in pck._sched
return
self._schedule(
pck,
self._start_packet_stop,
timeout=self._RPC_RESEND_INTERVAL)
with pck._lock:
if not pck._peer_addr: # FIXME _do_stop_packet must not be called on this condition
reschedule_if_need()
return
proxy = self._create_packet_rpc_proxy(pck)
logging.debug('_do_stop_packet(%s, stop_mode=%s' % (pck, stop_mode))
try:
if stop_mode == StopMode.CANCEL:
proxy.cancel(task_id)
else:
kill_jobs = stop_mode == StopMode.STOP
proxy.stop(task_id, kill_jobs)
except Exception as e:
logging.warning("Failed to send stop to packet %s: %s" % (pck, e))
if is_xmlrpc_exception(e, WrongTaskIdError) \
or isinstance(e, socket.error) and e.errno == errno.ECONNREFUSED:
return # FIXME Is enough? # STOP/SUSPEND->EXECUTING
with pck._lock:
reschedule_if_need()
else:
with pck._lock:
#assert pck._state == RemotePacketState.STARTED # FIXME XXX
if pck._state != RemotePacketState.STARTED:
return
if pck._sent_stop_mode < stop_mode:
pck._sent_stop_mode = stop_mode
