コード例 #1
0
            def on_future_error(e):
                if isinstance(
                        e,
                        KafkaError) and e.retriable and state['delay'] <= 30:
                    _get_logger().debug(
                        "Got Kafka retriable error in future, will retry in %ds: %s"
                        % (state['delay'], repr(e)))
                    time.sleep(state['delay'])
                    state['delay'] *= 2

                    future = f(*args, **kwargs)
                    future.add_callback(callback)
                    future.add_errback(on_future_error)
                else:
                    if errback:
                        _get_logger().error(
                            "Got Kafka fatal error or too many retries for '%s', passing along: %r"
                            % (f.__name__, e))
                        errback(e)
                    else:
                        from six.moves import _thread

                        _get_logger().error(
                            "Got Kafka fatal error or too many retries in future, exiting: %r"
                            % e)
                        _thread.interrupt_main(
                        )  # simulate Ctrl-C in main thread - FIXME: hides real exception
コード例 #2
0
def start_network(config):
    global PREDICTOR_NETWORK
    try:
        PREDICTOR_NETWORK = PredictorNetwork(config)
    except Exception as e:
        # An error occurred loading the model; interrupt the whole server.
        tf.logging.error(e)
        _thread.interrupt_main()
コード例 #3
0
 def check_status(self):
     join_requested = False
     while not join_requested:
         status_response = (
             # 'or False' because this could return None.
             self._interface.send_status_request(check_stop_req=True)
             or False)
         if status_response.run_should_stop:
             thread.interrupt_main()
             return
         join_requested = self._join_event.wait(self._polling_interval)
コード例 #4
0
ファイル: queue.py プロジェクト: planetlabs/datalake
 def _threaded_worker(self, worker_number):
     log.info('upload worker {} starting.'.format(worker_number))
     try:
         while True:
             filename = self._queue.get(block=True)
             msg = 'upload worker {} handling {}'
             msg = msg.format(worker_number, filename)
             log.info(msg)
             self._synchronous_push(filename)
             self._queue.task_done()
     except Exception as e:
         log.exception(e)
         # when a worker fails, we fail the entire process.
         interrupt_main()
コード例 #5
0
 def _threaded_worker(self, worker_number):
     log.info('upload worker {} starting.'.format(worker_number))
     try:
         while True:
             filename = self._queue.get(block=True)
             msg = 'upload worker {} handling {}'
             msg = msg.format(worker_number, filename)
             log.info(msg)
             self._synchronous_push(filename)
             self._queue.task_done()
     except Exception as e:
         log.exception(e)
         # when a worker fails, we fail the entire process.
         interrupt_main()
コード例 #6
0
ファイル: director.py プロジェクト: KevinAnthony/dosage
 def run(self):
     """Process from queue until it is empty."""
     try:
         while not self.stopped:
             scraperobj = jobs.get(False)
             self.name = scraperobj.name
             try:
                 self.getStrips(scraperobj)
             finally:
                 jobs.task_done()
                 self.name = self.origname
     except Empty:
         pass
     except KeyboardInterrupt:
         _thread.interrupt_main()
コード例 #7
0
ファイル: director.py プロジェクト: mostlyuseful/dosage
 def run(self):
     """Process from queue until it is empty."""
     try:
         while not self.stopped:
             scraperobj = jobs.get(False)
             self.name = scraperobj.name
             try:
                 self.getStrips(scraperobj)
             finally:
                 jobs.task_done()
                 self.name = self.origname
     except Empty:
         pass
     except KeyboardInterrupt:
         _thread.interrupt_main()
コード例 #8
0
 def inference(self, model, input_dir, score_threshold):
     config = get_checkpoint_config(network, prompt=False)
     if not config:
         return flask.jsonify({'error': 'network not found'})
     try:
         predictor = PredictorNetwork(config)
         paths = os.listdir(input_dir) if os.path.isdir(input_dir) else [
             input_dir
         ]
         for image_file in paths:
             image = Image.open(image_file).conver('RGB')
             predictor.predict_image(image)
     except Exception as e:
         # An error occurred loading the model; interrupt the whole server.
         _thread.interrupt_main()
コード例 #9
0
ファイル: pyterm.py プロジェクト: shangdawei/pyftdi
 def _reader(self):
     """Loop forever, processing received serial data in terminal mode"""
     try:
         # Try to read as many bytes as possible at once, and use a short
         # timeout to avoid blocking for more data
         self._port.timeout = 0.050
         while self._resume:
             data = self._port.read(4096)
             if data:
                 sys.stdout.write(data.decode('utf8'))
                 sys.stdout.flush()
     except KeyboardInterrupt:
         return
     except Exception as e:
         print_("Exception: %s" % e)
         if self._debug:
             import traceback
             print_(traceback.format_exc(), file=sys.stderr)
         from six.moves import _thread
         _thread.interrupt_main()
コード例 #10
0
ファイル: pyterm.py プロジェクト: kery-chen/pyftdi
 def _reader(self):
     """Loop forever, processing received serial data in terminal mode"""
     try:
         # Try to read as many bytes as possible at once, and use a short
         # timeout to avoid blocking for more data
         self._port.timeout = 0.050
         while self._resume:
             data = self._port.read(4096)
             if data:
                 sys.stdout.write(data.decode('utf8'))
                 sys.stdout.flush()
     except KeyboardInterrupt:
         return
     except Exception as e:
         print_("Exception: %s" % e)
         if self._debug:
             import traceback
             print_(traceback.format_exc(), file=sys.stderr)
         from six.moves import _thread
         _thread.interrupt_main()
コード例 #11
0
    def stop_code(self):
        """
        Attempt to stop the running code by raising a keyboard interrupt in
        the main thread
        """
        self._stop = True  #stops the code if paused
        if self._paused is True:
            self.prompt()

        self._resume_event.set()

        #try a keyboard interrupt - this will not work for the internal engine
        # as the error is raised here instead of the running code, hence put in
        #try clause.
        try:
            thread.interrupt_main()
        except:
            pass

        # send the notification, the debugger may stop after the notification
        dp.send('debugger.ended')
コード例 #12
0
def time_limit(seconds):
    if hasattr(signal, "SIGALRM"):

        def signal_handler(signum, frame):
            raise TimeoutException("Timeout after {} seconds.".format(seconds))

        signal.signal(signal.SIGALRM, signal_handler)
        signal.alarm(seconds)
        try:
            yield
        finally:
            signal.alarm(0)
    else:
        timer = threading.Timer(seconds, lambda: _thread.interrupt_main())
        timer.start()
        try:
            yield
        except KeyboardInterrupt:
            raise TimeoutException("Timeout after {} seconds.".format(seconds))
        finally:
            timer.cancel()
コード例 #13
0
ファイル: timeout.py プロジェクト: buptxz/EQL_Tensorflow
def time_limit(seconds):
    if hasattr(signal, "SIGALRM"):
        # for Linux
        def signal_handler(signum, frame):
            raise TimeoutException("Timeout after {} seconds.".format(seconds))

        signal.signal(signal.SIGALRM, signal_handler)
        signal.alarm(seconds)
        try:
            yield
        finally:
            signal.alarm(0)
    else:
        # for Windows
        timer = threading.Timer(seconds, lambda: _thread.interrupt_main())
        timer.start()
        try:
            yield
        except KeyboardInterrupt:
            raise TimeoutException("Timeout after {} seconds.".format(seconds))
        finally:
            timer.cancel()
コード例 #14
0
    def _run(self):
        try:
            with self._lock:
                _wakeup_fd = self._wakeup_fds[0]

            conn = None
            self.stream_id = None
            next_connect_deadline = 0
            CONNECT_TIMEOUT = 2
            CONNECT_RETRY_INTERVAL = 2
            while True:
                to_write = set()
                to_read = set([_wakeup_fd])
                with self._lock:
                    if not self._started:
                        break

                    if self._new_master != self._master:
                        if conn is not None:
                            conn.close()
                            conn = None
                            self.stream_id = None
                            next_connect_deadline = (time.time() +
                                                     CONNECT_RETRY_INTERVAL)

                        self._master = self._new_master

                    if (conn is None and self._master is not None
                            and time.time() > next_connect_deadline):
                        conn = Connection(self._master, self)
                        next_connect_deadline = time.time()

                if conn is not None:
                    if conn.want_write():
                        to_write.add(conn.fileno())

                    to_read.add(conn.fileno())

                now = time.time()
                if next_connect_deadline > now:
                    timeout = next_connect_deadline - now
                elif next_connect_deadline + CONNECT_TIMEOUT > now:
                    timeout = next_connect_deadline + CONNECT_TIMEOUT - now
                else:
                    timeout = None

                readable, writeable, _ = select.select(to_read, to_write, [],
                                                       timeout)

                for fd in writeable:
                    assert fd == conn.fileno()
                    next_connect_deadline = 0
                    if not conn.write():
                        conn.close()
                        conn = None
                        self.stream_id = None
                        next_connect_deadline = (time.time() +
                                                 CONNECT_RETRY_INTERVAL)

                for fd in readable:
                    if fd == _wakeup_fd:
                        os.read(_wakeup_fd, select.PIPE_BUF)
                    elif conn and fd == conn.fileno():
                        if not conn.read():
                            conn.close()
                            conn = None
                            self.stream_id = None
                            next_connect_deadline = (time.time() +
                                                     CONNECT_RETRY_INTERVAL)

                if (conn is not None and next_connect_deadline != 0 and
                        time.time() > next_connect_deadline + CONNECT_TIMEOUT):
                    logger.error('Connect to %s timeout', conn.addr)
                    conn.close()
                    conn = None
                    self.stream_id = None
                    next_connect_deadline = (time.time() +
                                             CONNECT_RETRY_INTERVAL)

        except Exception:
            logger.exception('Thread abort:')
            with self._lock:
                self._started = False

            global _exc_info
            _exc_info = sys.exc_info()
            thread.interrupt_main()

        finally:
            self._shutdown()

            if conn:
                conn.close()
                conn = None

            with self._lock:
                r, w = self._wakeup_fds
                os.close(r)
                os.close(w)
                self._wakeup_fds = None
コード例 #15
0
ファイル: process.py プロジェクト: douban/pymesos
    def _run(self):
        try:
            with self._lock:
                _wakeup_fd = self._wakeup_fds[0]

            conn = None
            self.stream_id = None
            num_conn_retry = 0
            connect_deadline = 0

            while True:
                if not conn and self._master:
                    conn = Connection(self._master, self)

                    # reset deadline at first retry of every period
                    if num_conn_retry < 1:
                        connect_deadline = self._timeout + time.time()

                    if time.time() > connect_deadline:
                        raise Exception("connect reach deadline")

                to_write = set()
                to_read = set([_wakeup_fd])
                if conn:
                    to_read.add(conn.fileno())
                    if conn.want_write():
                        to_write.add(conn.fileno())

                readable, writeable, _ = select.select(to_read, to_write, [],
                                                       SELECT_TIMEOUT)

                if _wakeup_fd in readable:
                    with self._lock:
                        if not self._started:
                            break

                        if self._new_master != self._master:
                            if conn is not None:
                                to_read.discard(conn.fileno())
                                to_write.discard(conn.fileno())
                                conn.close()
                                conn = None
                                self.stream_id = None
                            self._master = self._new_master
                            num_conn_retry = 0

                    readable.remove(_wakeup_fd)
                    os.read(_wakeup_fd, PIPE_BUF)

                if not conn:
                    continue

                if not conn.connected:
                    err = conn.handle_connect_event()
                    if err:
                        sleep_timeout = self._backoff(num_conn_retry)
                        num_conn_retry += 1
                        conn.close()
                        conn = None

                        deadline = time.strftime('%Y-%m-%d %H:%M:%S',
                                                 time.localtime(
                                                     connect_deadline))
                        logger.warning(
                            'connect to %s error: %s,'
                            ' sleep %ds and try again,'
                            ' deadline: %s',
                            self._master, err, sleep_timeout, deadline)

                        time.sleep(sleep_timeout)
                        continue
                    else:
                        num_conn_retry = 0

                if writeable:
                    if not conn.write():
                        conn.close()
                        conn = None
                        self.stream_id = None
                        continue

                if readable:
                    if not conn.read():
                        conn.close()
                        conn = None
                        self.stream_id = None
        except Exception:
            logger.exception('Thread abort:')
            with self._lock:
                self._started = False

            global _exc_info
            _exc_info = sys.exc_info()
            thread.interrupt_main()

        finally:
            self._shutdown()

            if conn:
                conn.close()
                conn = None

            with self._lock:
                r, w = self._wakeup_fds
                os.close(r)
                os.close(w)
                self._wakeup_fds = None
コード例 #16
0
ファイル: process.py プロジェクト: windreamer/pymesos
    def _run(self):
        try:
            with self._lock:
                _wakeup_fd = self._wakeup_fds[0]

            conn = None
            self.stream_id = None
            next_connect_deadline = 0
            CONNECT_TIMEOUT = 2
            CONNECT_RETRY_INTERVAL = 2
            while True:
                to_write = set()
                to_read = set([_wakeup_fd])
                with self._lock:
                    if not self._started:
                        break

                    if self._new_master != self._master:
                        if conn is not None:
                            conn.close()
                            conn = None
                            self.stream_id = None
                            next_connect_deadline = (
                                time.time() + CONNECT_RETRY_INTERVAL
                            )

                        self._master = self._new_master

                    if (conn is None and self._master is not None and
                            time.time() > next_connect_deadline):
                        conn = Connection(self._master, self)
                        next_connect_deadline = time.time()

                if conn is not None:
                    if conn.want_write():
                        to_write.add(conn.fileno())

                    to_read.add(conn.fileno())

                now = time.time()
                if next_connect_deadline > now:
                    timeout = next_connect_deadline - now
                elif next_connect_deadline + CONNECT_TIMEOUT > now:
                    timeout = next_connect_deadline + CONNECT_TIMEOUT - now
                else:
                    timeout = None

                readable, writeable, _ = select.select(
                    to_read, to_write, [], timeout
                )

                for fd in writeable:
                    assert fd == conn.fileno()
                    next_connect_deadline = 0
                    if not conn.write():
                        conn.close()
                        conn = None
                        self.stream_id = None
                        next_connect_deadline = (
                            time.time() + CONNECT_RETRY_INTERVAL
                        )

                for fd in readable:
                    if fd == _wakeup_fd:
                        os.read(_wakeup_fd, select.PIPE_BUF)
                    elif conn and fd == conn.fileno():
                        if not conn.read():
                            conn.close()
                            conn = None
                            self.stream_id = None
                            next_connect_deadline = (
                                time.time() + CONNECT_RETRY_INTERVAL
                            )

                if (conn is not None and next_connect_deadline != 0 and
                        time.time() > next_connect_deadline + CONNECT_TIMEOUT):
                    logger.error('Connect to %s timeout', conn.addr)
                    conn.close()
                    conn = None
                    self.stream_id = None
                    next_connect_deadline = (
                        time.time() + CONNECT_RETRY_INTERVAL
                    )

        except Exception:
            logger.exception('Thread abort:')
            with self._lock:
                self._started = False

            global _exc_info
            _exc_info = sys.exc_info()
            thread.interrupt_main()

        finally:
            self._shutdown()

            if conn:
                conn.close()
                conn = None

            with self._lock:
                r, w = self._wakeup_fds
                os.close(r)
                os.close(w)
                self._wakeup_fds = None
コード例 #17
0
    def _run(self):
        try:
            with self._lock:
                _wakeup_fd = self._wakeup_fds[0]

            conn = None
            self.stream_id = None
            num_conn_retry = 0
            connect_deadline = 0

            while True:
                if not conn and self._master:
                    conn = Connection(self._master, self)

                    # reset deadline at first retry of every period
                    if num_conn_retry < 1:
                        connect_deadline = self._timeout + time.time()

                    if time.time() > connect_deadline:
                        raise Exception("connect reach deadline")

                to_write = set()
                to_read = set([_wakeup_fd])
                if conn:
                    to_read.add(conn.fileno())
                    if conn.want_write():
                        to_write.add(conn.fileno())

                readable, writeable, _ = select.select(to_read, to_write, [],
                                                       SELECT_TIMEOUT)

                if _wakeup_fd in readable:
                    with self._lock:
                        if not self._started:
                            break

                        if self._new_master != self._master:
                            if conn is not None:
                                to_read.discard(conn.fileno())
                                to_write.discard(conn.fileno())
                                conn.close()
                                conn = None
                                self.stream_id = None
                            self._master = self._new_master
                            num_conn_retry = 0

                    readable.remove(_wakeup_fd)
                    os.read(_wakeup_fd, PIPE_BUF)

                if not conn:
                    continue

                if not conn.connected:
                    err = conn.handle_connect_event()
                    if err:
                        sleep_timeout = self._backoff(num_conn_retry)
                        num_conn_retry += 1
                        conn.close()
                        conn = None

                        deadline = time.strftime(
                            '%Y-%m-%d %H:%M:%S',
                            time.localtime(connect_deadline))
                        logger.warning(
                            'connect to %s error: %s,'
                            ' sleep %ds and try again,'
                            ' deadline: %s', self._master, err, sleep_timeout,
                            deadline)

                        time.sleep(sleep_timeout)
                        continue
                    else:
                        num_conn_retry = 0

                if writeable:
                    if not conn.write():
                        conn.close()
                        conn = None
                        self.stream_id = None
                        continue

                if readable:
                    if not conn.read():
                        conn.close()
                        conn = None
                        self.stream_id = None
        except Exception:
            logger.exception('Thread abort:')
            with self._lock:
                self._started = False

            global _exc_info
            _exc_info = sys.exc_info()
            thread.interrupt_main()

        finally:
            self._shutdown()

            if conn:
                conn.close()
                conn = None

            with self._lock:
                r, w = self._wakeup_fds
                os.close(r)
                os.close(w)
                self._wakeup_fds = None
コード例 #18
0
ファイル: workers.py プロジェクト: ropodaing/vitrage
 def _stop():
     # send SEGINT (instant exit) instead of SIGTERM
     _thread.interrupt_main()