def on_future_error(e):
if isinstance(
e,
KafkaError) and e.retriable and state['delay'] <= 30:
_get_logger().debug(
"Got Kafka retriable error in future, will retry in %ds: %s"
% (state['delay'], repr(e)))
time.sleep(state['delay'])
state['delay'] *= 2
future = f(*args, **kwargs)
future.add_callback(callback)
future.add_errback(on_future_error)
else:
if errback:
_get_logger().error(
"Got Kafka fatal error or too many retries for '%s', passing along: %r"
% (f.__name__, e))
errback(e)
else:
from six.moves import _thread
_get_logger().error(
"Got Kafka fatal error or too many retries in future, exiting: %r"
% e)
_thread.interrupt_main(
) # simulate Ctrl-C in main thread - FIXME: hides real exception
def start_network(config):
global PREDICTOR_NETWORK
try:
PREDICTOR_NETWORK = PredictorNetwork(config)
except Exception as e:
# An error occurred loading the model; interrupt the whole server.
tf.logging.error(e)
_thread.interrupt_main()
def check_status(self):
join_requested = False
while not join_requested:
status_response = (
# 'or False' because this could return None.
self._interface.send_status_request(check_stop_req=True)
or False)
if status_response.run_should_stop:
thread.interrupt_main()
return
join_requested = self._join_event.wait(self._polling_interval)
def _threaded_worker(self, worker_number):
log.info('upload worker {} starting.'.format(worker_number))
try:
while True:
filename = self._queue.get(block=True)
msg = 'upload worker {} handling {}'
msg = msg.format(worker_number, filename)
log.info(msg)
self._synchronous_push(filename)
self._queue.task_done()
except Exception as e:
log.exception(e)
# when a worker fails, we fail the entire process.
interrupt_main()
def _threaded_worker(self, worker_number):
log.info('upload worker {} starting.'.format(worker_number))
try:
while True:
filename = self._queue.get(block=True)
msg = 'upload worker {} handling {}'
msg = msg.format(worker_number, filename)
log.info(msg)
self._synchronous_push(filename)
self._queue.task_done()
except Exception as e:
log.exception(e)
# when a worker fails, we fail the entire process.
interrupt_main()
def run(self):
"""Process from queue until it is empty."""
try:
while not self.stopped:
scraperobj = jobs.get(False)
self.name = scraperobj.name
try:
self.getStrips(scraperobj)
finally:
jobs.task_done()
self.name = self.origname
except Empty:
pass
except KeyboardInterrupt:
_thread.interrupt_main()
def run(self):
"""Process from queue until it is empty."""
try:
while not self.stopped:
scraperobj = jobs.get(False)
self.name = scraperobj.name
try:
self.getStrips(scraperobj)
finally:
jobs.task_done()
self.name = self.origname
except Empty:
pass
except KeyboardInterrupt:
_thread.interrupt_main()
def inference(self, model, input_dir, score_threshold):
config = get_checkpoint_config(network, prompt=False)
if not config:
return flask.jsonify({'error': 'network not found'})
try:
predictor = PredictorNetwork(config)
paths = os.listdir(input_dir) if os.path.isdir(input_dir) else [
input_dir
]
for image_file in paths:
image = Image.open(image_file).conver('RGB')
predictor.predict_image(image)
except Exception as e:
# An error occurred loading the model; interrupt the whole server.
_thread.interrupt_main()
def _reader(self):
"""Loop forever, processing received serial data in terminal mode"""
try:
# Try to read as many bytes as possible at once, and use a short
# timeout to avoid blocking for more data
self._port.timeout = 0.050
while self._resume:
data = self._port.read(4096)
if data:
sys.stdout.write(data.decode('utf8'))
sys.stdout.flush()
except KeyboardInterrupt:
return
except Exception as e:
print_("Exception: %s" % e)
if self._debug:
import traceback
print_(traceback.format_exc(), file=sys.stderr)
from six.moves import _thread
_thread.interrupt_main()
def _reader(self):
"""Loop forever, processing received serial data in terminal mode"""
try:
# Try to read as many bytes as possible at once, and use a short
# timeout to avoid blocking for more data
self._port.timeout = 0.050
while self._resume:
data = self._port.read(4096)
if data:
sys.stdout.write(data.decode('utf8'))
sys.stdout.flush()
except KeyboardInterrupt:
return
except Exception as e:
print_("Exception: %s" % e)
if self._debug:
import traceback
print_(traceback.format_exc(), file=sys.stderr)
from six.moves import _thread
_thread.interrupt_main()
def stop_code(self):
"""
Attempt to stop the running code by raising a keyboard interrupt in
the main thread
"""
self._stop = True #stops the code if paused
if self._paused is True:
self.prompt()
self._resume_event.set()
#try a keyboard interrupt - this will not work for the internal engine
# as the error is raised here instead of the running code, hence put in
#try clause.
try:
thread.interrupt_main()
except:
pass
# send the notification, the debugger may stop after the notification
dp.send('debugger.ended')
def time_limit(seconds):
if hasattr(signal, "SIGALRM"):
def signal_handler(signum, frame):
raise TimeoutException("Timeout after {} seconds.".format(seconds))
signal.signal(signal.SIGALRM, signal_handler)
signal.alarm(seconds)
try:
yield
finally:
signal.alarm(0)
else:
timer = threading.Timer(seconds, lambda: _thread.interrupt_main())
timer.start()
try:
yield
except KeyboardInterrupt:
raise TimeoutException("Timeout after {} seconds.".format(seconds))
finally:
timer.cancel()
def time_limit(seconds):
if hasattr(signal, "SIGALRM"):
# for Linux
def signal_handler(signum, frame):
raise TimeoutException("Timeout after {} seconds.".format(seconds))
signal.signal(signal.SIGALRM, signal_handler)
signal.alarm(seconds)
try:
yield
finally:
signal.alarm(0)
else:
# for Windows
timer = threading.Timer(seconds, lambda: _thread.interrupt_main())
timer.start()
try:
yield
except KeyboardInterrupt:
raise TimeoutException("Timeout after {} seconds.".format(seconds))
finally:
timer.cancel()
def _run(self):
try:
with self._lock:
_wakeup_fd = self._wakeup_fds[0]
conn = None
self.stream_id = None
next_connect_deadline = 0
CONNECT_TIMEOUT = 2
CONNECT_RETRY_INTERVAL = 2
while True:
to_write = set()
to_read = set([_wakeup_fd])
with self._lock:
if not self._started:
break
if self._new_master != self._master:
if conn is not None:
conn.close()
conn = None
self.stream_id = None
next_connect_deadline = (time.time() +
CONNECT_RETRY_INTERVAL)
self._master = self._new_master
if (conn is None and self._master is not None
and time.time() > next_connect_deadline):
conn = Connection(self._master, self)
next_connect_deadline = time.time()
if conn is not None:
if conn.want_write():
to_write.add(conn.fileno())
to_read.add(conn.fileno())
now = time.time()
if next_connect_deadline > now:
timeout = next_connect_deadline - now
elif next_connect_deadline + CONNECT_TIMEOUT > now:
timeout = next_connect_deadline + CONNECT_TIMEOUT - now
else:
timeout = None
readable, writeable, _ = select.select(to_read, to_write, [],
timeout)
for fd in writeable:
assert fd == conn.fileno()
next_connect_deadline = 0
if not conn.write():
conn.close()
conn = None
self.stream_id = None
next_connect_deadline = (time.time() +
CONNECT_RETRY_INTERVAL)
for fd in readable:
if fd == _wakeup_fd:
os.read(_wakeup_fd, select.PIPE_BUF)
elif conn and fd == conn.fileno():
if not conn.read():
conn.close()
conn = None
self.stream_id = None
next_connect_deadline = (time.time() +
CONNECT_RETRY_INTERVAL)
if (conn is not None and next_connect_deadline != 0 and
time.time() > next_connect_deadline + CONNECT_TIMEOUT):
logger.error('Connect to %s timeout', conn.addr)
conn.close()
conn = None
self.stream_id = None
next_connect_deadline = (time.time() +
CONNECT_RETRY_INTERVAL)
except Exception:
logger.exception('Thread abort:')
with self._lock:
self._started = False
global _exc_info
_exc_info = sys.exc_info()
thread.interrupt_main()
finally:
self._shutdown()
if conn:
conn.close()
conn = None
with self._lock:
r, w = self._wakeup_fds
os.close(r)
os.close(w)
self._wakeup_fds = None
def _run(self):
try:
with self._lock:
_wakeup_fd = self._wakeup_fds[0]
conn = None
self.stream_id = None
num_conn_retry = 0
connect_deadline = 0
while True:
if not conn and self._master:
conn = Connection(self._master, self)
# reset deadline at first retry of every period
if num_conn_retry < 1:
connect_deadline = self._timeout + time.time()
if time.time() > connect_deadline:
raise Exception("connect reach deadline")
to_write = set()
to_read = set([_wakeup_fd])
if conn:
to_read.add(conn.fileno())
if conn.want_write():
to_write.add(conn.fileno())
readable, writeable, _ = select.select(to_read, to_write, [],
SELECT_TIMEOUT)
if _wakeup_fd in readable:
with self._lock:
if not self._started:
break
if self._new_master != self._master:
if conn is not None:
to_read.discard(conn.fileno())
to_write.discard(conn.fileno())
conn.close()
conn = None
self.stream_id = None
self._master = self._new_master
num_conn_retry = 0
readable.remove(_wakeup_fd)
os.read(_wakeup_fd, PIPE_BUF)
if not conn:
continue
if not conn.connected:
err = conn.handle_connect_event()
if err:
sleep_timeout = self._backoff(num_conn_retry)
num_conn_retry += 1
conn.close()
conn = None
deadline = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(
connect_deadline))
logger.warning(
'connect to %s error: %s,'
' sleep %ds and try again,'
' deadline: %s',
self._master, err, sleep_timeout, deadline)
time.sleep(sleep_timeout)
continue
else:
num_conn_retry = 0
if writeable:
if not conn.write():
conn.close()
conn = None
self.stream_id = None
continue
if readable:
if not conn.read():
conn.close()
conn = None
self.stream_id = None
except Exception:
logger.exception('Thread abort:')
with self._lock:
self._started = False
global _exc_info
_exc_info = sys.exc_info()
thread.interrupt_main()
finally:
self._shutdown()
if conn:
conn.close()
conn = None
with self._lock:
r, w = self._wakeup_fds
os.close(r)
os.close(w)
self._wakeup_fds = None
def _run(self):
try:
with self._lock:
_wakeup_fd = self._wakeup_fds[0]
conn = None
self.stream_id = None
next_connect_deadline = 0
CONNECT_TIMEOUT = 2
CONNECT_RETRY_INTERVAL = 2
while True:
to_write = set()
to_read = set([_wakeup_fd])
with self._lock:
if not self._started:
break
if self._new_master != self._master:
if conn is not None:
conn.close()
conn = None
self.stream_id = None
next_connect_deadline = (
time.time() + CONNECT_RETRY_INTERVAL
)
self._master = self._new_master
if (conn is None and self._master is not None and
time.time() > next_connect_deadline):
conn = Connection(self._master, self)
next_connect_deadline = time.time()
if conn is not None:
if conn.want_write():
to_write.add(conn.fileno())
to_read.add(conn.fileno())
now = time.time()
if next_connect_deadline > now:
timeout = next_connect_deadline - now
elif next_connect_deadline + CONNECT_TIMEOUT > now:
timeout = next_connect_deadline + CONNECT_TIMEOUT - now
else:
timeout = None
readable, writeable, _ = select.select(
to_read, to_write, [], timeout
)
for fd in writeable:
assert fd == conn.fileno()
next_connect_deadline = 0
if not conn.write():
conn.close()
conn = None
self.stream_id = None
next_connect_deadline = (
time.time() + CONNECT_RETRY_INTERVAL
)
for fd in readable:
if fd == _wakeup_fd:
os.read(_wakeup_fd, select.PIPE_BUF)
elif conn and fd == conn.fileno():
if not conn.read():
conn.close()
conn = None
self.stream_id = None
next_connect_deadline = (
time.time() + CONNECT_RETRY_INTERVAL
)
if (conn is not None and next_connect_deadline != 0 and
time.time() > next_connect_deadline + CONNECT_TIMEOUT):
logger.error('Connect to %s timeout', conn.addr)
conn.close()
conn = None
self.stream_id = None
next_connect_deadline = (
time.time() + CONNECT_RETRY_INTERVAL
)
except Exception:
logger.exception('Thread abort:')
with self._lock:
self._started = False
global _exc_info
_exc_info = sys.exc_info()
thread.interrupt_main()
finally:
self._shutdown()
if conn:
conn.close()
conn = None
with self._lock:
r, w = self._wakeup_fds
os.close(r)
os.close(w)
self._wakeup_fds = None
def _run(self):
try:
with self._lock:
_wakeup_fd = self._wakeup_fds[0]
conn = None
self.stream_id = None
num_conn_retry = 0
connect_deadline = 0
while True:
if not conn and self._master:
conn = Connection(self._master, self)
# reset deadline at first retry of every period
if num_conn_retry < 1:
connect_deadline = self._timeout + time.time()
if time.time() > connect_deadline:
raise Exception("connect reach deadline")
to_write = set()
to_read = set([_wakeup_fd])
if conn:
to_read.add(conn.fileno())
if conn.want_write():
to_write.add(conn.fileno())
readable, writeable, _ = select.select(to_read, to_write, [],
SELECT_TIMEOUT)
if _wakeup_fd in readable:
with self._lock:
if not self._started:
break
if self._new_master != self._master:
if conn is not None:
to_read.discard(conn.fileno())
to_write.discard(conn.fileno())
conn.close()
conn = None
self.stream_id = None
self._master = self._new_master
num_conn_retry = 0
readable.remove(_wakeup_fd)
os.read(_wakeup_fd, PIPE_BUF)
if not conn:
continue
if not conn.connected:
err = conn.handle_connect_event()
if err:
sleep_timeout = self._backoff(num_conn_retry)
num_conn_retry += 1
conn.close()
conn = None
deadline = time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(connect_deadline))
logger.warning(
'connect to %s error: %s,'
' sleep %ds and try again,'
' deadline: %s', self._master, err, sleep_timeout,
deadline)
time.sleep(sleep_timeout)
continue
else:
num_conn_retry = 0
if writeable:
if not conn.write():
conn.close()
conn = None
self.stream_id = None
continue
if readable:
if not conn.read():
conn.close()
conn = None
self.stream_id = None
except Exception:
logger.exception('Thread abort:')
with self._lock:
self._started = False
global _exc_info
_exc_info = sys.exc_info()
thread.interrupt_main()
finally:
self._shutdown()
if conn:
conn.close()
conn = None
with self._lock:
r, w = self._wakeup_fds
os.close(r)
os.close(w)
self._wakeup_fds = None
def _stop():
# send SEGINT (instant exit) instead of SIGTERM
_thread.interrupt_main()
