class Runner(BenchmarkThread):

    def __init__(self, *args, **kwargs):
        BenchmarkThread.__init__(self, *args, **kwargs)
        self.num_started = count()
        self.num_finished = count()
        self.event = Event()

    def insert_next(self, previous_result=sentinel):
        if previous_result is not sentinel:
            if isinstance(previous_result, BaseException):
                log.error("Error on insert: %r", previous_result)
            if next(self.num_finished) >= self.num_queries:
                self.event.set()

        if next(self.num_started) <= self.num_queries:
            future = self.session.execute_async(self.query, self.values)
            future.add_callbacks(self.insert_next, self.insert_next)

    def run(self):
        self.start_profile()

        for _ in range(min(120, self.num_queries)):
            self.insert_next()

        self.event.wait()

        self.finish_profile()
Beispiel #2
0
class HeartbeatFuture(object):
    def __init__(self, connection, owner):
        self._exception = None
        self._event = Event()
        self.connection = connection
        self.owner = owner
        log.debug("Sending options message heartbeat on idle connection (%s) %s", id(connection), connection.host)
        with connection.lock:
            if connection.in_flight < connection.max_request_id:
                connection.in_flight += 1
                connection.send_msg(OptionsMessage(), connection.get_request_id(), self._options_callback)
            else:
                self._exception = Exception("Failed to send heartbeat because connection 'in_flight' exceeds threshold")
                self._event.set()

    def wait(self, timeout):
        self._event.wait(timeout)
        if self._event.is_set():
            if self._exception:
                raise self._exception
        else:
            raise OperationTimedOut()

    def _options_callback(self, response):
        if not isinstance(response, SupportedMessage):
            if isinstance(response, ConnectionException):
                self._exception = response
            else:
                self._exception = ConnectionException(
                    "Received unexpected response to OptionsMessage: %s" % (response,)
                )

        log.debug("Received options response on connection (%s) from %s", id(self.connection), self.connection.host)
        self._event.set()
Beispiel #3
0
class Actor:
    def __init__(self):
        self._mailbox = Queue()

    def send(self, msg):
        self._mailbox.put(msg)

    def recv(self):
        msg = self._mailbox.get()
        if msg is ActorExit:
            raise ActorExit()
        return msg

    def start(self):
        self._terminated = Event()
        t = Thread(target=self._bootstrap)
        t.daemon = True
        t.start()

    def _bootstrap(self):
        try:
            self.run()
        except ActorExit:
            pass
        finally:
            self._terminated.set()

    def join(self):
        self._terminated.wait()

    def run(self):
        while True:
            msg = self.recv()
Beispiel #4
0
def test_closes_if_not_hit():
    try:
        da = DummyHandler()
        fa = Event()
        sa = RawServer(fa, 2, 2)
        loop(sa)
        sl(sa, da, beginport + 14)

        sleep(1)
        db = DummyHandler()
        fb = Event()
        sb = RawServer(fb, 100, 100)
        loop(sb)
        sl(sb, db, beginport + 13)
        
        sleep(.5)
        sa.start_connection(('127.0.0.1', beginport + 13))
        sleep(1)
        
        assert da.external_made == []
        assert da.data_in == []
        assert da.lost == []
        assert len(db.external_made) == 1
        del db.external_made[:]
        assert db.data_in == []
        assert db.lost == []

        sleep(3.1)
        
        assert len(da.lost) == 1
        assert len(db.lost) == 1
    finally:
        fa.set()
        fb.set()
Beispiel #5
0
class ResponseWaiter(object):

    def __init__(self, connection, num_responses):
        self.connection = connection
        self.pending = num_responses
        self.error = None
        self.responses = [None] * num_responses
        self.event = Event()

    def got_response(self, response, index):
        with self.connection.lock:
            self.connection.in_flight -= 1
        if isinstance(response, Exception):
            self.error = response
            self.event.set()
        else:
            self.responses[index] = response
            self.pending -= 1
            if not self.pending:
                self.event.set()

    def deliver(self, timeout=None):
        self.event.wait(timeout)
        if self.error:
            raise self.error
        elif not self.event.is_set():
            raise OperationTimedOut()
        else:
            return self.responses
Beispiel #6
0
class Dotter(object):
    def __init__(self,delay=100,symbol='.'):
        self.event=Event()
        self.delay=delay
        self.symbol=symbol
        self.status=False
    def __loop(self):
        while not self.event.is_set():
            stdout.write(self.symbol)
            stdout.flush()
            sleep(self.delay/1000)
    def start(self):
        if not self.status:
            self.event.clear()
            Thread(target=self.__loop).start()
            self.status=True
    def stop(self,newLine=True):
        if self.status:
            self.event.set()
            if newLine:
                stdout.write('\n')
            self.status=False
    def set(self,delay=None,symbol=None):
        if delay!=None:
            self.delay=delay
        if symbol!=None:
            self.symbol=symbol
        if self.status:
            self.stop(False)
            self.start()
Beispiel #7
0
class StoppableQThread(QtCore.QThread):
    """ Base class for QThreads which require the ability
    to be stopped by a thread-safe method call
    """

    def __init__(self, parent=None):
        self._should_stop = Event()
        self._should_stop.clear()
        super(StoppableQThread, self).__init__(parent)

    def join(self, timeout=0):
        """ Joins the current thread and forces it to stop after
        the timeout if necessary

        :param timeout: Timeout duration in seconds
        """
        self._should_stop.wait(timeout)
        if not self.should_stop():
            self.stop()
        super(StoppableQThread, self).wait()

    def stop(self):
        self._should_stop.set()

    def should_stop(self):
        return self._should_stop.is_set()

    def __repr__(self):
        return "<%s(should_stop=%s)>" % (self.__class__.__name__, self.should_stop())
class DeferredResponse( object ):
  """
  A deferred that resolves to a response from TSServer.
  """

  def __init__( self, timeout = RESPONSE_TIMEOUT_SECONDS ):
    self._event = Event()
    self._message = None
    self._timeout = timeout


  def resolve( self, message ):
    self._message = message
    self._event.set()


  def result( self ):
    self._event.wait( timeout = self._timeout )
    if not self._event.isSet():
      raise RuntimeError( 'Response Timeout' )
    message = self._message
    if not message[ 'success' ]:
      raise RuntimeError( message[ 'message' ] )
    if 'body' in message:
      return self._message[ 'body' ]
Beispiel #9
0
 def test_ask_shutdown(self):
     q = queue.Queue()
     done = Event()
     done.set()
     channel = controller.Channel(q, done)
     with tutils.raises(Kill):
         channel.ask("test", Mock(name="test_ask_shutdown"))
Beispiel #10
0
class TestInterruptibleDecorator(TestCase):

    def setUp(self):
        self.quit_condition = False
        self.thread_started = Event()

    @interruptible
    def never_ending(self, cancellation_point):
        self.thread_started.set()
        while True:
            time.sleep(0.01)
            cancellation_point()

    def test_interruptible_decorator(self):
        """ Tests for the @interruptible decorator. """
        thread = Thread(target=self.never_ending, args=(
            lambda: _cancellation_point(lambda: self.quit_condition),))
        thread.start()

        # Wait until thread comes to live
        self.thread_started.wait()

        # Ask to it to quit within 20ms
        self.quit_condition = True
        time.sleep(0.02)

        # Thread is finished
        self.assertFalse(thread.is_alive())
Beispiel #11
0
class ResponseEvent:
    """Event which is fired when the response is returned for a request.

        For each request sent this event is created.
        An application can wait for the event to create a blocking request.
    """
    def __init__(self):
        self.__evt = Event()

    def waiting(self):
        return not self.__evt.isSet()

    def waitForResponse(self, timeOut=None):
        """blocks until the response arrived or timeout is reached."""
        self.__evt.wait(timeOut)
        if self.waiting():
            raise Timeout()
        else:
            if self.response["error"]:
                raise Exception(self.response["error"])
            else:
                return self.response["result"]

    def handleResponse(self, resp):
        self.response = resp
        self.__evt.set()
    def test_delayed_body_read_timeout(self):
        timed_out = Event()

        def socket_handler(listener):
            sock = listener.accept()[0]
            buf = b''
            body = 'Hi'
            while not buf.endswith(b'\r\n\r\n'):
                buf = sock.recv(65536)
            sock.send(('HTTP/1.1 200 OK\r\n'
                       'Content-Type: text/plain\r\n'
                       'Content-Length: %d\r\n'
                       '\r\n' % len(body)).encode('utf-8'))

            timed_out.wait()
            sock.send(body.encode('utf-8'))
            sock.close()

        self._start_server(socket_handler)
        pool = HTTPConnectionPool(self.host, self.port)

        response = yield From(pool.urlopen('GET', '/', retries=0, preload_content=False,
                                timeout=Timeout(connect=1, read=0.1)))
        try:
            self.aioAssertRaises(ReadTimeoutError, response.read)
        finally:
            timed_out.set()
Beispiel #13
0
class Queue(list):

    def __init__(self):
        super(Queue, self).__init__()
        self._lock = Lock()
        self._fill = Event()

    def put(self, obj):
        with self._lock:
            self.append(obj)
            self._fill.set()

    def get(self, block=True):
        with self._lock:
            if len(self) == 0:
                self._fill.clear()
        if not self._fill.isSet():
            if block:
                self._fill.wait()
            else:
                return None
        with self._lock:
            return self.pop(0)

    def delete(self, index):
        if 0 <= index < len(self):
            with self._lock:
                del self[index]

    def remove(self, element):
        if element in self:
            with self._lock:
                del self[self.index(element)]
Beispiel #14
0
    def _score_scenes(self, mov, scene_list, score_obj, analysis_budget):
        '''Get all the scenes scored.

        Results stored in score_obj
        '''
        frame_q = Queue.Queue(maxsize=100)
        halter = Event()

        threads = [
            Thread(target=self._frame_extractor,
                   args=(mov, scene_list, frame_q, halter))]
        for i in range(options.workers):
            threads.append(Thread(target=self._worker,
                                  args=(score_obj, frame_q, halter)))

        for t in threads:
            t.daemon = True
            t.start()

        # Run the frame scoring until we are out of time
        if not halter.wait(analysis_budget):
            _log.info('Out of time sampling frames')
            halter.set()

        for t in threads:
            t.join()

        _log.info('Finished scoring scenes')
Beispiel #15
0
class ThreadedRunner(Runnable):

    def __init__(self, runnable):
        self._runnable = runnable
        self._notifier = Event()
        self._result = None
        self._error = None
        self._traceback = None
        self._thread = None

    def run(self):
        try:
            self._result = self._runnable()
        except:
            self._error, self._traceback = sys.exc_info()[1:]
        self._notifier.set()

    __call__ = run

    def run_in_thread(self, timeout):
        self._thread = Thread(self, name=TIMEOUT_THREAD_NAME)
        self._thread.setDaemon(True)
        self._thread.start()
        self._notifier.wait(timeout)
        return self._notifier.isSet()

    def get_result(self):
        if self._error:
            raise self._error, None, self._traceback
        return self._result

    def stop_thread(self):
        self._thread.stop()
class InterruptableEvent(object):
    """Event class for for Python v2.7 which behaves more like Python v3.2
    threading.Event and allows signals to interrupt waits"""

    def __init__(self):
        self.__event = Event()

    def is_set(self):
        return self.__event.is_set()

    def set(self):
        self.__event.set()

    def clear(self):
        self.__event.clear()

    def wait(self, timeout=None):
        # infinite
        if timeout is None:
            # event with timeout is interruptable
            while not self.__event.wait(60):
                pass
            return True

        # finite
        else:
            # underlying Event will perform timeout argument validation
            return self.__event.wait(timeout)
Beispiel #17
0
class Task(object):
    def __init__(self, name, start_time, calc_next_time, func):
        """
        Initialize a Task.
       
        Arguments:
        name            - Name of task.
        start_time      - First time for task to run
        calc_next_time  - Function to calculate the time of next run,
                          gets one argument, the last run time as a datetime.
                          Returns None when task should no longer be run
        func            - A function to run
        """
        self.name = name
        self.start_time = start_time
        self.scheduled_time = start_time
        self.calc_next_time = calc_next_time
        self.func = func
        self.halt_flag = Event()
       
    def run(self):
        logging.debug("Running %s task, scheduled at: %s" % (self.name, self.scheduled_time,))
        if not self.halt_flag.isSet():
            try:
                try:
                    self.func()
                except:
                    raise
            finally:
                self.scheduled_time = self.calc_next_time(self.scheduled_time)
                logging.debug("Scheduled next run of %s for: %s" % (self.name, self.scheduled_time,))
           
    def halt(self):
        self.halt_flag.set()
Beispiel #18
0
  def FromWatchdogWithSubservers_test( self ):
    all_servers_are_running = Event()

    def KeepServerAliveInAnotherThread():
      while not all_servers_are_running.is_set():
        try:
          self.GetRequest( 'ready' )
        except requests.exceptions.ConnectionError:
          pass
        finally:
          time.sleep( 0.1 )

    self.Start( idle_suicide_seconds = 2, check_interval_seconds = 1 )

    StartThread( KeepServerAliveInAnotherThread )

    try:
      filetypes = [ 'cs',
                    'go',
                    'java',
                    'javascript',
                    'typescript',
                    'rust' ]
      for filetype in filetypes:
        self.StartSubserverForFiletype( filetype )
      self.AssertServersAreRunning()
    finally:
      all_servers_are_running.set()

    self.AssertServersShutDown( timeout = SUBSERVER_SHUTDOWN_TIMEOUT + 10 )
    self.AssertLogfilesAreRemoved()
Beispiel #19
0
    def testMultipleLoad(self):
        """
        In DropBox, the loading of multiple CLIs seems to
        lead to the wrong context being assigned to some
        controls.

        See #4749
        """
        import random
        from threading import Thread, Event

        event = Event()

        class T(Thread):
            def run(self, *args):
                pause = random.random()
                event.wait(pause)
                self.cli = CLI()
                self.cli.loadplugins()
                self.con = self.cli.controls["admin"]
                self.cmp = self.con.ctx

        threads = [T() for x in range(20)]
        for t in threads:
            t.start()
        event.set()
        for t in threads:
            t.join()

        assert len(threads) == len(set([t.cli for t in threads]))
        assert len(threads) == len(set([t.con for t in threads]))
        assert len(threads) == len(set([t.cmp for t in threads]))
class WaitableTimer(Timer):
    def __init__(self, timeout, callback):
        Timer.__init__(self, timeout, callback)
        self.callback = callback
        self.event = Event()

        self.final_exception = None

    def finish(self, time_now):
        try:
            finished = Timer.finish(self, time_now)
            if finished:
                self.event.set()
                return True
            return False

        except Exception as e:
            self.final_exception = e
            self.event.set()
            return True

    def wait(self, timeout=None):
        self.event.wait(timeout)
        if self.final_exception:
            raise self.final_exception
Beispiel #21
0
class RepeatTimer(Thread):

    def __init__(self, interval, function, stop_on_exception=True, args=[], kwargs={}):
        Thread.__init__(self)
        self.interval = interval
        self.function = function
        self.args = args
        self.kwargs = kwargs
        self.stop_on_exception = stop_on_exception
        self.finished = Event()

    def stop(self):
        self.done = True
        self.finished.set()

    def run(self):
        self.done = False
        while True:
            self.finished.wait(self.interval)
            if self.done:
                self.finished.clear()
                break
            if not self.finished.is_set():
                try:
                    self.function(*self.args, **self.kwargs)
                except Exception, e:
                    if self.stop_on_exception:
                        self.finished.clear()
                        raise
                    # XXX Not strictly for auth'ing, think of something better
                    log.exception('Login Fail')
            self.finished.clear()
Beispiel #22
0
class CheckForUpdates(Thread):

    INTERVAL = 24*60*60  # seconds
    daemon = True

    def __init__(self, parent):
        Thread.__init__(self)
        self.shutdown_event = Event()
        self.signal = Signal(parent)

    def run(self):
        while not self.shutdown_event.is_set():
            calibre_update_version = NO_CALIBRE_UPDATE
            plugins_update_found = 0
            try:
                version = get_newest_version()
                if version[:2] > numeric_version[:2]:
                    calibre_update_version = version
            except Exception as e:
                prints('Failed to check for calibre update:', as_unicode(e))
            try:
                update_plugins = get_plugin_updates_available(raise_error=True)
                if update_plugins is not None:
                    plugins_update_found = len(update_plugins)
            except Exception as e:
                prints('Failed to check for plugin update:', as_unicode(e))
            if calibre_update_version != NO_CALIBRE_UPDATE or plugins_update_found > 0:
                self.signal.update_found.emit(calibre_update_version, plugins_update_found)
            self.shutdown_event.wait(self.INTERVAL)

    def shutdown(self):
        self.shutdown_event.set()
Beispiel #23
0
def test_concurrent_rendering():
    '''Best-effort testing that concurrent multi-threaded rendering works.
    The test has no guarantees around being deterministic, but if it fails
    you know something is wrong with concurrent rendering. If it passes,
    things are probably working.'''
    err = None
    def func(sim, event):
        event.wait()
        sim.data.qpos[:] = 0.0
        sim.forward()
        img1 = sim.render(width=40, height=40, camera_name="camera1")
        img2 = sim.render(width=40, height=40, camera_name="camera2")
        try:
            assert np.sum(img1[:]) == 23255
            assert np.sum(img2[:]) == 12007
        except Exception as e:
            nonlocal err
            err = e

    model = load_model_from_xml(BASIC_MODEL_XML)
    sim = MjSim(model)
    sim.render(100, 100)
    event = Event()
    threads = []
    for _ in range(100):
        thread = Thread(target=func, args=(sim, event))
        threads.append(thread)
        thread.start()
    event.set()
    for thread in threads:
        thread.join()
    assert err is None, "Exception: %s" % (str(err))
Beispiel #24
0
class GarbageCollectorThread(Thread):
    """Thread in which garbage collection actually happens."""
    def __init__(self, gc):
        super(GarbageCollectorThread, self).__init__()
        self.gc = gc
        self.daemon = True
        self.pid = getpid()
        self.ready = Event()
    
    def run(self):
        s = self.gc.context.socket(zmq.PULL)
        s.linger = 0
        s.bind(self.gc.url)
        self.ready.set()
        
        while True:
            # detect fork
            if getpid is None or getpid() != self.pid:
                return
            msg = s.recv()
            if msg == b'DIE':
                break
            fmt = 'L' if len(msg) == 4 else 'Q'
            key = struct.unpack(fmt, msg)[0]
            tup = self.gc.refs.pop(key, None)
            if tup and tup.event:
                tup.event.set()
            del tup
        s.close()
def request_token_spotty(spotty, use_creds=True):
    '''request token by using the spotty binary'''
    token_info = None
    if spotty.playback_supported:
        try:
            args = ["-t", "--client-id", CLIENTID, "--scope", ",".join(SCOPE), "-n", "temp-spotty"]
            done = Event()
            spotty = spotty.run_spotty(arguments=args, use_creds=use_creds)
            watcher = Thread(target=kill_on_timeout, args=(done, 5, spotty))
            watcher.daemon = True
            watcher.start()
            stdout, stderr = spotty.communicate()
            done.set()
            result = None
            log_msg("request_token_spotty stdout: %s" % stdout)
            for line in stdout.split():
                line = line.strip()
                if line.startswith("{\"accessToken\""):
                    result = json.loads(line)
            # transform token info to spotipy compatible format
            if result:
                token_info = {}
                token_info["access_token"] = result["accessToken"]
                token_info["expires_in"] = result["expiresIn"]
                token_info["token_type"] = result["tokenType"]
                token_info["scope"] = ' '.join(result["scope"])
                token_info['expires_at'] = int(time.time()) + token_info['expires_in']
                token_info['refresh_token'] = result["accessToken"]
        except Exception as exc:
            log_exception(__name__, exc)
    return token_info
Beispiel #26
0
class StoppableThreadWithResult(Thread):
    """Thread class with a stop() method. The thread itself has to check
    regularly for the stopped() condition."""

    def __init__(self, group=None, target=None, name=None,
                 args=(), kwargs=None, verbose=None):
        super(StoppableThreadWithResult, self).__init__(group=group, target=target,
                        name=name, args=args, kwargs=kwargs, verbose=verbose)
        self._stop = Event()

    def stop(self):
        self._stop.set()
        self._Thread__stop()

    def stopped(self):
        return self._stop.isSet()

    def run(self):
        if self._Thread__target is not None:
            self._return = self._Thread__target(*self._Thread__args,
                                                **self._Thread__kwargs)

    def join(self, timeout=None):
        Thread.join(self, timeout=None)
        return self._return
class TimerWithResume(object):
    def __init__(self, status_subject, refresh_interval):
        self.status_subject = status_subject
        self.abort = Event()
        self.refresh_interval = refresh_interval

    def perform(self):
        while not self.abort.isSet():
            self.status_subject.build_status()
            self.abort.wait(self.refresh_interval)

    def stop(self):
        self.abort.set()

    def start(self):
        self.thread = Thread(target=self.perform)
        self.thread.daemon = True
        self.thread.start()

    def resume(self):
        self.thread.join()
        self.abort.clear()
        self.start()

    def set_refresh_interval(self, new_interval):
        self.refresh_interval = new_interval
class TestInterruptible(unittest.TestCase):
    """ Tests for interrupting cooperative threads """

    def test_interruptible_decorator(self):
        """ Tests for the @interruptible decorator. """
        self.quit_condition = False
        cancellation_point = lambda: _cancellation_point(
            lambda: self.quit_condition)
        self.thread_started = Event()

        @interruptible
        def never_ending(cancellation_point):
            self.thread_started.set()
            while True:
                time.sleep(0.1)
                cancellation_point()
        thread = Thread(target=never_ending, args=(cancellation_point, ))
        thread.start()
        self.thread_started.wait()
        self.quit_condition = True
        countdown = 10
        while thread.is_alive() and countdown > 0:
            time.sleep(0.1)
            countdown -= 1
        self.assertFalse(thread.is_alive())
Beispiel #29
0
class NonSubscribeListener(object):
    def __init__(self):
        self.result = None
        self.status = None
        self.done_event = Event()

    def callback(self, result, status):
        self.result = result
        self.status = status
        self.done_event.set()

    def pn_await(self, timeout=5):
        """ Returns False if a timeout happened, otherwise True"""
        return self.done_event.wait(timeout)

    def await_result(self, timeout=5):
        self.pn_await(timeout)
        return self.result

    def await_result_and_reset(self, timeout=5):
        self.pn_await(timeout)
        cp = copy.copy(self.result)
        self.reset()
        return cp

    def reset(self):
        self.result = None
        self.status = None
        self.done_event.clear()
Beispiel #30
0
  class Worker(Thread):
	  
    def __init__(self):
      Thread.__init__(self)
      self.trigger = Event()
      self.uri = None
      self.sonos = None
      self.daemon = True
      self.lock = Lock()

    def start_url(self, a_group, uri):
      with self.lock:
		self.sonos = a_group
		self.uri = uri
      self.trigger.set()

    
    def run(self):
      while True:
        self.trigger.wait()
        if self.sonos is None:
		  # take this as a sign to shut down gracefully
          break
        with self.lock:
          uri = self.uri
          self.uri = None
          self.trigger.clear()
        result = self.sonos.play_uri( uri)
        print result
class WebLiveBackend(object):
    """ Backend for interacting with the WebLive service """

    client = None
    URL = os.environ.get('SOFTWARE_CENTER_WEBLIVE_HOST',
        'https://weblive.stgraber.org/weblive/json')

    def __init__(self):
        self.weblive = WebLive(self.URL, True)
        self.available_servers = []

        for client in (WebLiveClientX2GO, WebLiveClientQTNX):
            if client.is_supported():
                self.client = client()
                break

        self._ready = Event()

    @property
    def ready(self):
        """ Return true if data from the remote server was loaded
        """

        return self.client and self._ready.is_set()

    def query_available(self):
        """ Get all the available data from WebLive """

        self._ready.clear()
        servers = self.weblive.list_everything()
        self._ready.set()
        return servers

    def query_available_async(self):
        """ Call query_available in a thread and set self.ready """

        def _query_available_helper():
            self.available_servers = self.query_available()

        p = Thread(target=_query_available_helper)
        p.start()

    def is_pkgname_available_on_server(self, pkgname, serverid=None):
        """Check if the package is available (on all servers or
           on 'serverid')
        """

        for server in self.available_servers:
            if not serverid or server.name == serverid:
                for pkg in server.packages:
                    if pkg.pkgname == pkgname:
                        return True
        return False

    def get_servers_for_pkgname(self, pkgname):
        """ Return a list of servers having a given package """

        servers = []
        for server in self.available_servers:
            # No point in returning a server that's full
            if server.current_users >= server.userlimit:
                continue

            for pkg in server.packages:
                if pkg.pkgname == pkgname:
                    servers.append(server)
        return servers

    def create_automatic_user_and_run_session(self, serverid,
                                              session="desktop", wait=False):
        """ Create a user on 'serverid' and start the session """

        # Use the boot_id to get a temporary unique identifier
        # (till next reboot)
        if os.path.exists('/proc/sys/kernel/random/boot_id'):
            uuid = open('/proc/sys/kernel/random/boot_id',
                'r').read().strip().replace('-', '')
            random.seed(uuid)

        # Generate a 20 characters string based on the boot_id
        identifier = ''.join(random.choice(string.ascii_lowercase)
            for x in range(20))

        # Use the current username as the GECOS on the server
        # if it's invalid (by weblive's standard), use "WebLive user" instead
        fullname = str(os.environ.get('USER', 'WebLive user'))
        if not re.match("^[A-Za-z0-9 ]*$", fullname) or len(fullname) == 0:
            fullname = 'WebLive user'

        # Send the user's locale so it's automatically selected when connecting
        locale = os.environ.get("LANG", "None").replace("UTF-8", "utf8")

        # Create the user and retrieve host and port of the target server
        connection = self.weblive.create_user(serverid, identifier, fullname,
            identifier, session, locale)

        # Connect using x2go or fallback to qtnx if not available
        if (self.client):
            self.client.start_session(connection[0], connection[1], session,
                identifier, identifier, wait)
        else:
            raise IOError("No remote desktop client available.")
Beispiel #32
0
        class MyModule(MumoModule):
            def __init__(self, name, manager, configuration=None):
                MumoModule.__init__(self, name, manager, configuration)

                self.estarted = Event()
                self.estopped = Event()
                self.econnected = Event()
                self.edisconnected = Event()

                self.emeta = Event()
                self.econtext = Event()
                self.eserver = Event()

            def onStart(self):
                self.estarted.set()

            def onStop(self):
                self.estopped.set()

            def connected(self):
                man = self.manager()
                man.subscribeMetaCallbacks(self)
                man.subscribeServerCallbacks(self)
                man.subscribeContextCallbacks(self)
                self.econnected.set()

            def disconnected(self):
                self.edisconnected.set()

            def metaCallMe(self, arg1, arg2):
                if arg1 == "arg1" and arg2 == "arg2":
                    self.emeta.set()

            def contextCallMe(self, server, arg1, arg2):
                if arg1 == "arg1" and arg2 == "arg2":
                    self.econtext.set()

            def serverCallMe(self, server, arg1, arg2):
                if arg1 == "arg1" and arg2 == "arg2":
                    self.eserver.set()
Beispiel #33
0
class SteamEngine:
	""" implement routing of mqtt msgs to servers """

	def __init__(self, conf, logger):
		self.conf = conf
		self.logger = logger

		self.my_clientid = conf['my_client_id']
		self.short_circuit = conf.get('allow_short_circuit', allow_short_circuit)
		self.task_timeout = conf.get('network_timeout', network_timeout)
		Topic.encode = conf.get('topic_encode', topic_encode)

		# service_routing is keyed with subscription topic (mqtt flavoured re), values is the queue to send message to
		self.running = True
		self.service_routing = {}
		self.task_id = 0
		self.tasks = {}

		# create thread to handle backflow of answers
		self.answer_task = Thread(target=self.run_answers)
		self.answer_task.name = "answer_th"
		self.answer_task.daemon = True

		# create thread to polling
		self.poll_interval = int(self.conf.get('pollinterval', 3600))
		self.poll_task = Thread(target=self.run_poll)
		self.poll_task.name = "poll_th"
		self.poll_task.daemon = True

		# queue to receive service answers
		self.service_answers = queue.Queue()
		self.service_answer_topic = Topic("+", self.my_clientid, "+", "a")

		# start runners
		self.answer_task.start()
		self.poll_task.start()

		# connect to broker
		self.mqtt_ready = Event()
		self.mqtt_connect()

		# start service
		self.load_and_run_services()


	def shutdown(self):
		self.running = False
		self.mqtt_con.loop_stop()
		self.mqtt_con.disconnect()
		for service in self.conf['services']:
			self.services[service].shutdown()


	def poll_for_status(self):
		for service in self.services:
			if getattr(self.services[service], 'poll', None) \
					and inspect.ismethod(self.services[service].poll):
				self.logger.debug("polling %s", service)
				self.services[service].poll()


	def run_poll(self):
		self.logger.info("Steamlink run_poll start")
		polltime = time.time() - self.poll_interval	+ 3 # force poll 3 sec after boot
		while self.running:
			waitt = max(polltime + self.poll_interval - time.time(), 0)
			if waitt > 0:
				time.sleep(waitt)
			else:
				polltime = time.time()
				self.poll_for_status()


	def check_sv(self, sv, avail_services):
		err = False
		if not sv['type'] in ['server', 'channel']:
			self.logger.error("unknown service type '%s'", sv['type'])
			err = True
		if not sv['class'] in avail_services:
			self.logger.error("unknown service class '%s'", sv['class'])
			err = True
		if err:
			self.logger.error("correct config and re-run")
			sys.exit(1)


	def get_available_services(self):
		# find available channels by searching all classes for a 'type' class variable
		# with a contect of server or channel
		avail_services = {}
		for name, obj in inspect.getmembers(sys.modules[__name__]):
			if inspect.isclass(obj) and getattr(obj, "type", None) in ['server', 'channel']:
				avail_services[name] = obj
		return avail_services

	def load_and_run_services(self):
		self.services = {}
		avail_services = self.get_available_services()

		for service in self.conf['services']:
			sv = self.conf['services'][service]
			self.check_sv(sv, avail_services)
			self.services[service] = avail_services[sv['class']](service, self, self.logger, sv)

		for service in self.conf['services']:
			self.services[service].start()


	def run_answers(self):
		self.logger.info("Steamlink run_answers start")
		while self.running:
			answer = self.service_answers.get()
			topic = Topic(answer['_topic'])
			topic.direction = 'a'
			topic.client_id = answer['_origin']
			self.logger.debug('run_answer publish %s %s', topic, str(answer)[:70]+"...")
			self.mqtt_con.publish(str(topic), dumps(answer))
			self.service_answers.task_done()


	def return_answer(self, pkt):
		tp = Topic(pkt['_topic'])
		if self.short_circuit \
				and tp.client_id == self.my_clientid \
				and tp.task_id in self.tasks:
			self.tasks[task_id].answer = pkt
			self.tasks[task_id].set()
		else:
			self.service_answers.put(pkt)


	def mqtt_connect(self, reconnect=False):
		self.mqtt_ready.clear()
		if reconnect:
			self.mqtt_con.loop_stop()
			self.mqtt_con.disconnect()
			self.mqtt_con.reinitialise(client_id=self.conf['MQTT_CLIENTID'])
		else:
			self.mqtt_con = mqtt.Client(client_id=self.conf['MQTT_CLIENTID'])
		# TODO: figure out cert in conf
		self.mqtt_con.tls_set(self.conf['MQTT_CERT']),
		self.mqtt_con.username_pw_set(self.conf['MQTT_USERNAME'], self.conf['MQTT_PASSWORD'])
		self.mqtt_con.tls_insecure_set(False)
		self.mqtt_con.on_connect = self.mqtt_on_connect
		self.mqtt_con.on_disconnect = self.mqtt_on_disconnect
		self.mqtt_con.on_message = self.mqtt_on_message

		self.logger.info("connecting to MQTT")
		self.mqtt_con.connect(self.conf['MQTT_SERVER'], self.conf['MQTT_PORT'], 60)
		self.mqtt_con.loop_start()
		self.mqtt_con._thread.name = "mqtt_th"	# N.B.


	def mqtt_on_disconnect(self, client, userdata, flags):
		self.mqtt_ready.clear()
		self.logger.warn("disconnected from MQTT boker")


	def mqtt_on_connect(self, client, userdata, flags, rc):
		if self.mqtt_con is None:
			self.mqtt_connect(reconnect=True)
			return
		self.logger.warn("connected to MQTT boker,code "+str(rc))
		# subscribe to all answers for my_client_id
		self.subscribe(self.service_answer_topic)
		self.mqtt_ready.set()
		for service_topic in self.service_routing:
			self.subscribe(service_topic)


	def subscribe(self, service_topic):
		self.mqtt_con.subscribe(str(service_topic))
		self.logger.debug( "subscribe %s" % service_topic)


	def mqtt_on_message(self, client, userdata, msg):
		srv_type = None
		if mqtt.topic_matches_sub(str(self.service_answer_topic), msg.topic):
			srv_type = "answer"
		else:
			for service_sub in self.service_routing:
				if mqtt.topic_matches_sub(service_sub, msg.topic):
					srv_type = self.service_routing[service_sub].typ
					break
		if not srv_type:
			self.logger.error("no registered service for msg topic %s", msg.topic)
			return

		if srv_type == 'channel':
			self.service_routing[service_sub].queue.put(msg)

		elif srv_type in ['server', 'answer']:
			tp = Topic(msg.topic)
			assert tp.client_id == self.my_clientid
			pkt = json.loads(msg.payload.decode('utf-8'))
			pkt['_topic'] = msg.topic
			self.logger.debug("mqtt_on_message %s %s", msg.topic, str(pkt)[:70]+"...")

			if  srv_type is 'answer':
				self.receive_answer(pkt, int(tp.task_id))
			elif srv_type is 'server':
				self.dispatch_service(tp.service, pkt)


	def receive_answer(self, pkt, task_id):
		if  not task_id in self.tasks:
			self.logger.error("received answer for unknown task id %s", task_id)
		else:
			self.tasks[task_id].answer = pkt
			self.tasks[task_id].set()


	def dispatch_service(self, msg_service, pkt):
		msg_service_topic = str(Topic(msg_service, self.my_clientid, "+", "q"))
		if not msg_service_topic in self.service_routing:
			self.logger.error("no service '%s' defined", msg_service)
			return
		self.service_routing[msg_service_topic].queue.put(pkt)
		return


	def publish(self, topic, pkt, as_json=True, retain=False):
		self.mqtt_ready.wait()
		self.logger.debug("publish: %s %s", topic, pkt)
		if as_json:
			opkt = dumps(pkt)
		else:
			opkt = pkt
		self.mqtt_con.publish(str(topic), opkt, retain=retain)


	def register_service(self, service_topic, service_queue, service_type):
		if  service_topic in self.service_routing:
			logger.error("service %s already defined", service_topic)
		else:
			self.logger.debug("register service %s", service_topic)
			self.service_routing[service_topic] = Srv(service_queue, service_type)
			if self.mqtt_ready.is_set():
				self.subscribe(service_topic)


	def ask_question(self, service, client_id,  pkt):
		if client_id == None:
			client_id = self.conf['default_clients'].get(service, self.my_clientid)
		task_id = self.new_task_id()
		self.tasks[task_id] = Event()
		topic = Topic(service, client_id, task_id, "q")
		pkt['_origin'] = self.my_clientid
		if self.short_circuit and client_id == self.my_clientid:
			pkt['_topic'] = str(topic)
			self.dispatch_service(service, pkt)
		else:
			self.publish(topic, pkt)
		if self.tasks[task_id].wait(self.task_timeout):
			answer = self.tasks[task_id].answer
			del self.tasks[task_id]
			del answer['_topic']
		else:
			return Err('TIMEOUT', 'network request  timed out').d()
		return answer['res']


	def new_task_id(self):
		self.task_id +=1
		return self.task_id
class AgentLogicBase:

    def __init__(self, config):
        logging.debug("AgentLogicBase:: __init__() entered")
        self.wait_stop = Event()
        self.heartBitRate = config.getint("general", "heart_beat_rate")
        self.userCheckRate = config.getint("general", "report_user_rate")
        self.appRefreshRate = config.getint("general", "report_application_rate")
        self.disksRefreshRate = config.getint("general", "report_disk_usage")
        self.activeUser = ""
        self.vio = VirtIoChannel(config.get("virtio", "device"))
        self.dr = None
        self.commandHandler = None

    def run(self):
        logging.debug("AgentLogicBase:: run() entered")
        thread.start_new_thread(self.doListen, ())
        thread.start_new_thread(self.doWork, ())

        # Yuck! It's seem that Python block all signals when executing
        # a "real" code. So there is no way just to sit and wait (with
        # no timeout).
        # Try breaking out from this code snippet:
        # $ python -c "import threading; threading.Event().wait()"
        while not self.wait_stop.isSet():
            self.wait_stop.wait(1)

    def stop(self):
        logging.debug("AgentLogicBase:: baseStop() entered")
        self.wait_stop.set()

    def doWork(self):
        logging.debug("AgentLogicBase:: doWork() entered")
        self.sendInfo()
        self.sendUserInfo()
        self.sendAppList()
        counter = 0
        hbsecs = self.heartBitRate
        appsecs = self.appRefreshRate
        disksecs = self.disksRefreshRate
        usersecs = self.userCheckRate

        try:
            while not self.wait_stop.isSet():
                counter +=1
                hbsecs -= 1
                if hbsecs <= 0:
                    self.vio.write('heartbeat',
                                   {'free-ram' : self.dr.getAvailableRAM(),
                                    'memory-stat' : self.dr.getMemoryStats(),
                                    'totalrdpconnections' : self.dr.getTotalRDPConnections()})
                    hbsecs = self.heartBitRate
                usersecs -=1
                if usersecs <=0:
                    self.sendUserInfo()
                    usersecs = self.userCheckRate
                appsecs -= 1
                if appsecs <= 0:
                    self.sendAppList()
                    self.sendInfo()
                    appsecs = self.appRefreshRate
                disksecs -= 1
                if disksecs <= 0:
                    self.sendDisksUsages()
                    disksecs = self.disksRefreshRate
                time.sleep(1)
            logging.debug("AgentLogicBase:: doWork() exiting")
        except:
            logging.exception("AgentLogicBase::doWork")

    def doListen(self):
        logging.debug("AgentLogicBase::doListen() - entered")
        if self.commandHandler == None:
            logging.debug("AgentLogicBase::doListen() - no commandHandler ... exiting doListen thread")
            return
        while not self.wait_stop.isSet():
            try:
                logging.debug("AgentLogicBase::doListen() - in loop before vio.read")
                cmd, args = self.vio.read()
                if cmd:
                    self.parseCommand(cmd, args)
            except:
                logging.exception('Error while reading the virtio-serial channel.')
        logging.debug("AgentLogicBase::doListen() - exiting")

    def parseCommand(self, command, args):
        logging.info("Received an external command: %s..." % (command))
        if command == 'lock-screen':
            self.commandHandler.lock_screen()
        elif command == 'log-off':
            self.commandHandler.logoff()
        elif command == 'shutdown':
            try:
                timeout = int(args['timeout'])
            except:
                timeout = 0
            try:
                msg = args['message']
            except:
                msg = 'System is going down'
            logging.info("Shutting down (timeout = %d, message = '%s')" % (timeout, msg))
            self.commandHandler.shutdown(timeout, msg)
        elif command == 'login':
            username = args['username'].encode('utf8')
            password = args['password'].encode('utf8')
            credentials = struct.pack('>I%ds%ds' % (len(username), len(password) + 1),
                len(username), username, password)
            logging.debug("User log-in (credentials = %s)" % (safe_creds_repr(credentials)))
            self.commandHandler.login(credentials)
        elif command == 'refresh':
            self.sendUserInfo(True)
            self.sendAppList()
            self.sendInfo()
            self.sendDisksUsages()
        elif command == 'echo':
            logging.debug("Echo: %s", args)
            self.vio.write('echo', args)
        elif command == 'hibernate':
            state = args.get('state', 'disk')
            self.commandHandler.hibernate(state)
        elif command == 'reboot':
            self.commandHandler.reboot()
        else:
            logging.error("Unknown external command: %s (%s)" % (command, args))

    def sendUserInfo(self, force=False):
        cur_user = str(self.dr.getActiveUser())
        logging.debug("AgentLogicBase::sendUserInfo - cur_user = '******'"%(cur_user))
        if cur_user != self.activeUser or force:
            self.vio.write('active-user', { 'name' : cur_user })
            self.activeUser = cur_user

    def sendInfo(self):
        self.vio.write('host-name', { 'name' : self.dr.getMachineName() })
        self.vio.write('os-version', { 'version' : self.dr.getOsVersion() })
        self.vio.write('network-interfaces', { 'interfaces' : self.dr.getAllNetworkInterfaces() })

    def sendAppList(self):
        self.vio.write('applications', { 'applications' : self.dr.getApplications() })

    def sendDisksUsages(self):
        self.vio.write('disks-usage', { 'disks' : self.dr.getDisksUsage() })

    def sendMemoryStats(self):
        self.vio.write('memory-stats', { 'memory' : self.dr.getMemoryStats() })

    def sessionLogon(self):
        logging.debug("AgentLogicBase::sessionLogon: user logs on the system.")
        cur_user = self.dr.getActiveUser()
        retries = 0
        while (cur_user == 'None') and (retries < 5):
            time.sleep(1)
            cur_user = self.dr.getActiveUser()
            retries = retries + 1
        self.sendUserInfo()
        self.vio.write('session-logon')

    def sessionLogoff(self):
        logging.debug("AgentLogicBase::sessionLogoff: user logs off from the system.")
        self.activeUser = '******'
        self.vio.write('session-logoff')
        self.vio.write('active-user', { 'name' : self.activeUser })

    def sessionLock(self):
        logging.debug("AgentLogicBase::sessionLock: user locks the workstation.")
        self.vio.write('session-lock')

    def sessionUnlock(self):
        logging.debug("AgentLogicBase::sessionUnlock: user unlocks the workstation.")
        self.vio.write('session-unlock')

    def sessionStartup(self):
        logging.debug("AgentLogicBase::sessionStartup: system starts up.")
        self.vio.write('session-startup')

    def sessionShutdown(self):
        logging.debug("AgentLogicBase::sessionShutdown: system shuts down.")
        self.vio.write('session-shutdown')
Beispiel #35
0
def update_location():
    while not should_stop.is_set():
        with locker:
            for device in devices:
                position = random.choice(['latitude', 'longitude'])
                device[position] += random.random() / 100

        time.sleep(1)


@app.route("/api")
def devices_list():
    with locker:
        return jsonify(devices)


@app.route('/')
def root():
    return render_template('index.html')


if __name__ == "__main__":
    updater = Thread(target=update_location)
    updater.start()

    app.run(debug=True, port=8001)
    should_stop.set()
    updater.join()

Beispiel #36
0
class ThreadManager:
    """
    manages the download threads, assign jobs, reconnect etc.
    """
    def __init__(self, core):
        """
        Constructor.
        """
        self.pyload = core
        self._ = core._

        self.threads = []  #: thread list
        self.local_threads = []  #: addon+decrypter threads

        self.pause = True

        self.reconnecting = Event()
        self.reconnecting.clear()
        self.downloaded = 0  #: number of files downloaded since last cleanup

        self.lock = Lock()

        # some operations require to fetch url info from hoster, so we caching them so it wont be done twice
        # contains a timestamp and will be purged after timeout
        self.info_cache = {}

        # pool of ids for online check
        self.result_ids = 0

        # threads which are fetching hoster results
        self.info_results = {}
        # timeout for cache purge
        self.timestamp = 0

        # pycurl.global_init(pycurl.GLOBAL_DEFAULT)

        for i in range(self.pyload.config.get("download", "max_downloads")):
            self.create_thread()

    def create_thread(self):
        """
        create a download thread.
        """
        thread = DownloadThread(self)
        self.threads.append(thread)

    def create_info_thread(self, data, pid):
        """
        start a thread whichs fetches online status and other infos
        data = [ .. () .. ]
        """
        self.timestamp = time.time() + timedelta(minutes=5).seconds

        InfoThread(self, data, pid)

    @lock
    def create_result_thread(self, data, add=False):
        """
        creates a thread to fetch online status, returns result id.
        """
        self.timestamp = time.time() + timedelta(minutes=5).seconds

        rid = self.result_ids
        self.result_ids += 1

        InfoThread(self, data, rid=rid, add=add)

        return rid

    @lock
    def get_info_result(self, rid):
        """
        returns result and clears it.
        """
        self.timestamp = time.time() + timedelta(minutes=5).seconds

        if rid in self.info_results:
            data = self.info_results[rid]
            self.info_results[rid] = {}
            return data
        else:
            return {}

    @lock
    def set_info_results(self, rid, result):
        self.info_results[rid].update(result)

    def get_active_files(self):
        active = [
            x.active for x in self.threads
            if x.active and isinstance(x.active, PyFile)
        ]

        for t in self.local_threads:
            active.extend(t.get_active_files())

        return active

    def processing_ids(self):
        """
        get a id list of all pyfiles processed.
        """
        return [x.id for x in self.get_active_files()]

    def run(self):
        """
        run all task which have to be done (this is for repetivive call by core)
        """
        try:
            self.try_reconnect()
        except Exception as exc:
            self.pyload.log.error(
                self._("Reconnect Failed: {}").format(exc),
                exc_info=self.pyload.debug > 1,
                stack_info=self.pyload.debug > 2,
            )
            self.reconnecting.clear()
        self.check_thread_count()

        try:
            self.assign_job()
        except Exception as exc:
            self.pyload.log.warning(
                "Assign job error",
                exc,
                exc_info=self.pyload.debug > 1,
                stack_info=self.pyload.debug > 2,
            )

            time.sleep(0.5)
            self.assign_job()
            # it may be failed non critical so we try it again

        if (self.info_cache
                or self.info_results) and self.timestamp < time.time():
            self.info_cache.clear()
            self.info_results.clear()
            self.pyload.log.debug("Cleared Result cache")

    # ----------------------------------------------------------------------
    def try_reconnect(self):
        """
        checks if reconnect needed.
        """
        if not (self.pyload.config.get("reconnect", "enabled")
                and self.pyload.api.is_time_reconnect()):
            return False

        active = [
            x.active.plugin.want_reconnect and x.active.plugin.waiting
            for x in self.threads if x.active
        ]

        if not (0 < active.count(True) == len(active)):
            return False

        reconnect_script = self.pyload.config.get("reconnect", "script")
        if not os.path.isfile(reconnect_script):
            self.pyload.config.set("reconnect", "enabled", False)
            self.pyload.log.warning(self._("Reconnect script not found!"))
            return

        self.reconnecting.set()

        # Do reconnect
        self.pyload.log.info(self._("Starting reconnect"))

        while [x.active.plugin.waiting
               for x in self.threads if x.active].count(True) != 0:
            time.sleep(0.25)

        ip = self.get_ip()

        self.pyload.addon_manager.before_reconnecting(ip)

        self.pyload.log.debug(f"Old IP: {ip}")

        try:
            subprocess.run(reconnect_script)
        except Exception:
            self.pyload.log.warning(
                self._("Failed executing reconnect script!"))
            self.pyload.config.set("reconnect", "enabled", False)
            self.reconnecting.clear()
            return

        time.sleep(1)
        ip = self.get_ip()
        self.pyload.addon_manager.after_reconnecting(ip)

        self.pyload.log.info(self._("Reconnected, new IP: {}").format(ip))

        self.reconnecting.clear()

    def get_ip(self):
        """
        retrieve current ip.
        """
        services = [
            ("http://icanhazip.com/", r"(\S+)"),
            ("http://checkip.dyndns.org/",
             r".*Current IP Address: (\S+)</body>.*"),
            ("http://ifconfig.io/ip", r"(\S+)"),
        ]

        ip = ""
        for i in range(10):
            try:
                sv = choice(services)
                ip = get_url(sv[0])
                ip = re.match(sv[1], ip).group(1)
                break
            except Exception:
                ip = ""
                time.sleep(1)

        return ip

    # ----------------------------------------------------------------------
    def check_thread_count(self):
        """
        checks if there are need for increasing or reducing thread count.
        """
        if len(self.threads) == self.pyload.config.get("download",
                                                       "max_downloads"):
            return True
        elif len(self.threads) < self.pyload.config.get(
                "download", "max_downloads"):
            self.create_thread()
        else:
            free = [x for x in self.threads if not x.active]
            if free:
                free[0].put("quit")

    # def clean_pycurl(self):
    # """
    # make a global curl cleanup (currently ununused)
    # """
    # if self.processing_ids():
    # return False
    # pycurl.global_cleanup()
    # pycurl.global_init(pycurl.GLOBAL_DEFAULT)
    # self.downloaded = 0
    # self.pyload.log.debug("Cleaned up pycurl")
    # return True

    # ----------------------------------------------------------------------
    def assign_job(self):
        """
        assing a job to a thread if possible.
        """
        if self.pause or not self.pyload.api.is_time_download():
            return

        # if self.downloaded > 20:
        #    if not self.clean_py_curl(): return

        free = [x for x in self.threads if not x.active]

        inuse = set([
            (x.active.pluginname, self.get_limit(x)) for x in self.threads
            if x.active and x.active.has_plugin() and x.active.plugin.account
        ])
        inuse = [(
            x[0],
            x[1],
            len([
                y for y in self.threads
                if y.active and y.active.pluginname == x[0]
            ]),
        ) for x in inuse]
        onlimit = [x[0] for x in inuse if x[1] > 0 and x[2] >= x[1]]

        occ = sorted([
            x.active.pluginname for x in self.threads if x.active
            and x.active.has_plugin() and not x.active.plugin.multi_dl
        ] + onlimit)

        occ = tuple(set(occ))
        job = self.pyload.files.get_job(occ)
        if job:
            try:
                job.init_plugin()
                job.set_status("starting")
            except Exception as exc:
                self.pyload.log.critical(exc,
                                         exc_info=True,
                                         stack_info=self.pyload.debug > 2)
                job.set_status("failed")
                job.error = str(exc)
                job.release()
                return

            if job.plugin.__type__ == "downloader":
                space_left = (fs.free_space(
                    self.pyload.config.get("general", "storage_folder")) >> 20)
                if space_left < self.pyload.config.get("general",
                                                       "min_free_space"):
                    self.pyload.log.warning(
                        self._("Not enough space left on device"))
                    self.pause = True

                if free and not self.pause:
                    thread = free[0]
                    # self.downloaded += 1

                    thread.put(job)
                else:
                    # put job back
                    if occ not in self.pyload.files.job_cache:
                        self.pyload.files.job_cache[occ] = []
                    self.pyload.files.job_cache[occ].append(job.id)

                    # check for decrypt jobs
                    job = self.pyload.files.get_decrypt_job()
                    if job:
                        job.init_plugin()
                        thread = DecrypterThread(self, job)

            else:
                thread = DecrypterThread(self, job)

    def get_limit(self, thread):
        limit = thread.active.plugin.account.get_account_data(
            thread.active.plugin.user)["options"].get("limit_dl", ["0"])[0]
        return int(limit)
Beispiel #37
0
class RabbitmqConnection(object):

    Parameters = URLParameters
    Connection = LayerConnection
    Protocol = Protocol

    def __init__(self, url, expiry, group_expiry, get_capacity, crypter):

        self.url = url
        self.expiry = expiry
        self.group_expiry = group_expiry
        self.get_capacity = get_capacity
        self.crypter = crypter

        self.protocols = {}
        self.lock = Lock()
        self.is_open = Event()
        self.parameters = self.Parameters(self.url)
        self.connection = self.Connection(
            parameters=self.parameters,
            on_open_callback=self.start_loop,
            on_close_callback=self.notify_futures,
            on_callback_error_callback=self.protocol_error,
            stop_ioloop_on_close=False,
            lock=self.lock,
        )

    def run(self):

        self.connection.ioloop.start()

    def start_loop(self, connection):

        self.is_open.set()
        self.process(None, (DECLARE_DEAD_LETTERS, (), {}), Future())

    def process(self, ident, method, future):

        if (ident in self.protocols
                and self.protocols[ident].amqp_channel.is_open):
            self.protocols[ident].resolve = future
            self.protocols[ident].apply(*method)
            return
        protocol = self.Protocol(self.expiry, self.group_expiry,
                                 self.get_capacity, ident, self.crypter)
        protocol.resolve = future
        amqp_channel = self.connection.channel(
            partial(protocol.register_channel, method), )
        amqp_channel.on_callback_error_callback = protocol.protocol_error
        self.protocols[ident] = protocol

    def notify_futures(self, connection, code, msg):

        try:
            for protocol in self.protocols.values():
                protocol.resolve.set_exception(ConnectionClosed())
        finally:
            self.connection.ioloop.stop()

    def protocol_error(self, error):

        for protocol in self.protocols.values():
            protocol.resolve.set_exception(error)

    def schedule(self, f, *args, **kwargs):

        if self.connection.is_closing or self.connection.is_closed:
            raise ConnectionClosed
        self.is_open.wait()
        future = Future()
        with self.lock:
            self.process(get_ident(), (f, args, kwargs), future)
        return future

    @property
    def thread_protocol(self):

        return self.protocols[get_ident()]
Beispiel #38
0
class UpdateState:
    """Used to hold the current state of processed updates.
       To retrieve an update, .poll() should be called.
    """
    def __init__(self, polling):
        self._polling = polling
        self.handlers = []
        self._updates_lock = RLock()
        self._updates_available = Event()
        self._updates = deque()

        # https://core.telegram.org/api/updates
        self._state = tl.updates.State(0, 0, datetime.now(), 0, 0)

    def can_poll(self):
        """Returns True if a call to .poll() won't lock"""
        return self._updates_available.is_set()

    def poll(self):
        """Polls an update or blocks until an update object is available"""
        if not self._polling:
            raise ValueError('Updates are not being polled hence not saved.')

        self._updates_available.wait()
        with self._updates_lock:
            update = self._updates.popleft()
            if not self._updates:
                self._updates_available.clear()

        if isinstance(update, Exception):
            raise update  # Some error was set through .set_error()

        return update

    def get_polling(self):
        return self._polling

    def set_polling(self, polling):
        self._polling = polling
        if not polling:
            with self._updates_lock:
                self._updates.clear()

    polling = property(fget=get_polling, fset=set_polling)

    def set_error(self, error):
        """Sets an error, so that the next call to .poll() will raise it.
           Can be (and is) used to pass exceptions between threads.
        """
        with self._updates_lock:
            # Insert at the beginning so the very next poll causes an error
            # TODO Should this reset the pts and such?
            self._updates.insert(0, error)
            self._updates_available.set()

    def check_error(self):
        with self._updates_lock:
            if self._updates and isinstance(self._updates[0], Exception):
                raise self._updates.pop()

    def process(self, update):
        """Processes an update object. This method is normally called by
           the library itself.
        """
        if not self._polling and not self.handlers:
            return

        with self._updates_lock:
            if isinstance(update, tl.updates.State):
                self._state = update
                return  # Nothing else to be done

            pts = getattr(update, 'pts', self._state.pts)
            if pts <= self._state.pts:
                return  # We already handled this update

            self._state.pts = pts
            if self._polling:
                self._updates.append(update)
                self._updates_available.set()

        for handler in self.handlers:
            handler(update)
Beispiel #39
0
class IdentifyWidget(QWidget):  # {{{

    rejected = pyqtSignal()
    results_found = pyqtSignal()
    book_selected = pyqtSignal(object, object)

    def __init__(self, log, parent=None):
        QWidget.__init__(self, parent)
        self.log = log
        self.abort = Event()
        self.caches = {}

        self.l = l = QGridLayout()
        self.setLayout(l)

        names = ['<b>'+p.name+'</b>' for p in metadata_plugins(['identify']) if
                p.is_configured()]
        self.top = QLabel('<p>'+_('calibre is downloading metadata from: ') +
            ', '.join(names))
        self.top.setWordWrap(True)
        l.addWidget(self.top, 0, 0)

        self.results_view = ResultsView(self)
        self.results_view.book_selected.connect(self.emit_book_selected)
        self.get_result = self.results_view.get_result
        l.addWidget(self.results_view, 1, 0)

        self.comments_view = Comments(self)
        l.addWidget(self.comments_view, 1, 1)

        self.results_view.show_details_signal.connect(self.comments_view.show_data)

        self.query = QLabel('download starting...')
        self.query.setWordWrap(True)
        l.addWidget(self.query, 2, 0, 1, 2)

        self.comments_view.show_wait()

    def emit_book_selected(self, book):
        self.book_selected.emit(book, self.caches)

    def start(self, title=None, authors=None, identifiers={}):
        self.log.clear()
        self.log('Starting download')
        parts, simple_desc = [], ''
        if title:
            parts.append('title:'+title)
            simple_desc += _('Title: %s ') % title
        if authors:
            parts.append('authors:'+authors_to_string(authors))
            simple_desc += _('Authors: %s ') % authors_to_string(authors)
        if identifiers:
            x = ', '.join('%s:%s'%(k, v) for k, v in iteritems(identifiers))
            parts.append(x)
            if 'isbn' in identifiers:
                simple_desc += 'ISBN: %s' % identifiers['isbn']
        self.query.setText(simple_desc)
        self.log(unicode_type(self.query.text()))

        self.worker = IdentifyWorker(self.log, self.abort, title,
                authors, identifiers, self.caches)

        self.worker.start()

        QTimer.singleShot(50, self.update)

    def update(self):
        if self.worker.is_alive():
            QTimer.singleShot(50, self.update)
        else:
            self.process_results()

    def process_results(self):
        if self.worker.error is not None:
            error_dialog(self, _('Download failed'),
                    _('Failed to download metadata. Click '
                        'Show Details to see details'),
                    show=True, det_msg=self.worker.error)
            self.rejected.emit()
            return

        if not self.worker.results:
            log = ''.join(self.log.plain_text)
            error_dialog(self, _('No matches found'), '<p>' +
                    _('Failed to find any books that '
                        'match your search. Try making the search <b>less '
                        'specific</b>. For example, use only the author\'s '
                        'last name and a single distinctive word from '
                        'the title.<p>To see the full log, click "Show details".'),
                    show=True, det_msg=log)
            self.rejected.emit()
            return

        self.results_view.show_results(self.worker.results)
        self.results_found.emit()

    def cancel(self):
        self.abort.set()
Beispiel #40
0
class Receive(object):
    """
    A class to perform a Mqlight client creation and then wait to receive
    messages from the server.
    """
    def __init__(self, args):
        self._using_ssl = False
        self._security_options = {}
        ssl_arg_names = [
            'ssl_trust_certificate', 'ssl_client_certificate',
            'ssl_client_key', 'ssl_client_key_passphrase', 'ssl_verify_name'
        ]
        for ssl_arg_name in ssl_arg_names:
            value = args.__dict__[ssl_arg_name]
            if value is not None:
                self._security_options[ssl_arg_name] = value
                self._using_ssl = True
        if 'ssl_verify_name' not in self._security_options:
            self._security_options[ssl_arg_name] = True
        # Select the relevant service
        if args.service is None:
            self._service = AMQPS_SERVICE if self._using_ssl else AMQP_SERVICE
        else:
            if self._using_ssl:
                if not args.service.startswith('amqps'):
                    self._close(
                        None, 'The service URL must start with '
                        '"amqps://" when using any of the ssl '
                        'options.')
            self._service = args.service

        self._topic_pattern = args.topic_pattern
        if args.client_id is not None:
            self._client_id = args.client_id
        else:
            self._client_id = 'recv_' + str(uuid4()).replace('-', '_')[0:7]
        self._delay = args.delay
        self._share = args.share_name
        self._verbose = args.verbose
        self._destination_ttl = args.destination_ttl
        self._file = args.file
        self._client = None
        self._exit = Event()
        self._num_of_messages = AtomicVariable(0)

    def _close(self, client, err=None):
        """
        Handles closing this sample
        :param client the associate client to close
        :param err an error condtion to report.
        """
        if err:
            error = err[1] if isinstance(err, tuple) else err
            print('*** error ***\n{0}'.format(error), file=sys.stderr)
        if client:
            client.stop()
        self._exit.set()

    def _output(self, msg):
        print(msg)
        sys.stdout.flush()

    def _started(self, client):
        """
        Started callback. Indicate a successful connection to client and
        open for business.
        :param client the associate client that has been successfully connected
        """
        print('Connected to {0} using client-id {1}'.format(
            client.get_service(), client.get_id()))
        options = {'qos': mqlight.QOS_AT_LEAST_ONCE, 'auto_confirm': False}
        if self._destination_ttl is not None:
            options['ttl'] = self._destination_ttl * 1000
        if self._delay is not None and self._delay > 0:
            options['credit'] = 1
        client.subscribe(topic_pattern=self._topic_pattern,
                         share=self._share,
                         options=options,
                         on_subscribed=self._subscribed,
                         on_message=self._message)

    def _subscribed(self, client, err, pattern, share):
        """
        Subscribe callback. Indicates the subscription has been completed. If
        not error is report then it is open to receive messages.
        :param client the associate client for this subscription
        :param err issues detected while making the subscription, None means
        successful
        :param pattern the topic or pattern for the subscription
        :param share is the subscription is shared.
        """
        if err is not None:
            self._close(client,
                        'problem with subscribe request {0}'.format(err))
        if pattern:
            if share:
                self._output('Subscribed to share: {0}, pattern: {1}'.format(
                    share, pattern))
            else:
                self._output('Subscribed to pattern: {0}'.format(pattern))

    def _write_to_file(self, filename, data):
        """
        Write the contains of the data to the named file
        :param filename the filepath to write the data to
        :data data the data to be written
        """
        self._output('Writing message data to {0}'.format(self._file))
        try:
            with open(self._file, 'wb') as f:
                if PYTHON2:
                    if isinstance(data, str) or isinstance(data, unicode):
                        f.write(data)
                    else:
                        f.write(''.join(chr(byte) for byte in data))
                else:
                    if isinstance(data, str):
                        f.write(data)
                    else:
                        f.write(bytes(data))
        except IOError as e:
            self._close(
                None, 'Failed to write message to {0} '
                'because {1}'.format(self._file, e))

    def _message(self, message_type, data, delivery):
        """
        Message callback. This indicate there is a message has been
        receive and ready for processing.
        :param client the reference to the connected client
        :param message_type, the type of message received
        (message or malformed)
        :param data, the actual message recevied
        :param delivery, the associated delivery information for the message
        """
        if message_type == mqlight.MALFORMED:
            malformed_message = ('*** received malformed message ***\n'
                                 'Data: {0}\nDelivery: {1}'.format(
                                     data, delivery))
            print(malformed_message, file=sys.stderr)
        else:
            if self._verbose:
                self._output('# received message {0}'.format(
                    self._num_of_messages.add()))
            if self._file:
                self._write_to_file(self._file, data)
                delivery['message']['confirm_delivery']()
            else:
                print('{0}{1}\n'.format(data[:50],
                                        (' ...' if len(data) > 50 else '')),
                      end="")
                sys.stdout.flush()
            if self._verbose:
                self._output(delivery)
            if self._delay > 0:
                sleep(self._delay)
            if self._client.get_state() == 'started':
                delivery['message']['confirm_delivery']()

    def _state_changed(self, client, new_state, error):
        """
        state_change callback. Indicates that there has been a change of
        connection state to the server.
        :param client the client reference whoses state has changed
        :param new_state the new state for the client
        :param any assoicated error with the state change. None means no error
        """
        if error:
            if new_state in ('stopping', 'stopped', 'retrying'):
                self._close(client, error)

    def run(self):
        """
        Start the sample by performing a client connection. The associated
        callback will handle the subsequent stages of receiving messages
        """
        if not self._exit.isSet():
            try:
                self._client = mqlight.Client(
                    service=self._service,
                    client_id=self._client_id,
                    security_options=self._security_options,
                    on_started=self._started,
                    on_state_changed=self._state_changed)
            except Exception as exc:
                self._close(self._client, exc)

            # waiting for finish
            try:
                while not self._exit.wait(1.0):
                    pass
            except KeyboardInterrupt:
                self._output('Closing ...')
                self._close(self._client)
class RawServer:

    def __init__(self, doneflag, timeout_check_interval, timeout, noisy = True, ipv6_enable = True, failfunc = lambda x: None, errorfunc = None, sockethandler = None, excflag = Event(), max_socket_connects = 1000):
        self.timeout_check_interval = timeout_check_interval
        self.timeout = timeout
        self.servers = {}
        self.single_sockets = {}
        self.dead_from_write = []
        self.doneflag = doneflag
        self.noisy = noisy
        self.failfunc = failfunc
        self.errorfunc = errorfunc
        self.exccount = 0
        self.funcs = []
        self.externally_added = []
        self.finished = Event()
        self.tasks_to_kill = []
        self.excflag = excflag
        self.lock = RLock()
        if DEBUG2:
            log('rawserver::__init__: timeout_check_interval', timeout_check_interval, 'timeout', timeout, 'ipv6_enable', ipv6_enable)
        if sockethandler is None:
            if DEBUG2:
                log('rawserver::__init__: create SocketHandler: max_socket_connects', max_socket_connects)
            sockethandler = SocketHandler(timeout, ipv6_enable, READSIZE, max_socket_connects)
        self.sockethandler = sockethandler
        self.thread_ident = None
        self.interrupt_socket = sockethandler.get_interrupt_socket()
        self.add_task(self.scan_for_timeouts, timeout_check_interval)

    def get_exception_flag(self):
        return self.excflag

    def _add_task(self, func, delay, id = None):
        if delay < 0:
            delay = 0
        insort(self.funcs, (clock() + delay, func, id))

    def add_task(self, func, delay = 0, id = None):
        if DEBUG_TASKS:
            log('rawserver::add_task: func', func, 'delay', delay)
        if delay < 0:
            delay = 0
        self.lock.acquire()
        self.externally_added.append((func, delay, id))
        if self.thread_ident != get_ident():
            self.interrupt_socket.interrupt()
        self.lock.release()

    def scan_for_timeouts(self):
        self.add_task(self.scan_for_timeouts, self.timeout_check_interval)
        self.sockethandler.scan_for_timeouts()

    def bind(self, port, bind = '', reuse = False, ipv6_socket_style = 1):
        result = self.sockethandler.bind(port, bind, reuse, ipv6_socket_style)
        return result

    def find_and_bind(self, first_try, minport, maxport, bind = '', reuse = False, ipv6_socket_style = 1, randomizer = False):
        result = self.sockethandler.find_and_bind(first_try, minport, maxport, bind, reuse, ipv6_socket_style, randomizer)
        return result

    def start_connection_raw(self, dns, socktype, handler = None):
        if DEBUG2:
            log('rawserver::start_connection_raw: dns', dns, 'socktype', socktype, 'handler', handler)
        return self.sockethandler.start_connection_raw(dns, socktype, handler)

    def start_connection(self, dns, handler = None, randomize = False):
        if DEBUG2:
            log('rawserver::start_connection: dns', dns, 'randomize', randomize, 'handler', handler)
        return self.sockethandler.start_connection(dns, handler, randomize)

    def get_stats(self):
        return self.sockethandler.get_stats()

    def pop_external(self):
        self.lock.acquire()
        while self.externally_added:
            a, b, c = self.externally_added.pop(0)
            self._add_task(a, b, c)

        self.lock.release()

    def listen_forever(self, handler):
        if DEBUG:
            log('rawserver::listen_forever: handler', handler)
        self.thread_ident = get_ident()
        self.sockethandler.set_handler(handler)
        try:
            while not self.doneflag.isSet():
                try:
                    self.pop_external()
                    self._kill_tasks()
                    if self.funcs:
                        period = self.funcs[0][0] + 0.001 - clock()
                    else:
                        period = 1073741824
                    if period < 0:
                        period = 0
                    events = self.sockethandler.do_poll(period)
                    if self.doneflag.isSet():
                        if DEBUG:
                            log('rawserver::listen_forever: stopping because done flag set')
                        return
                    while self.funcs and self.funcs[0][0] <= clock():
                        garbage1, func, id = self.funcs.pop(0)
                        if id in self.tasks_to_kill:
                            pass
                        try:
                            if DEBUG_TASKS:
                                if func.func_name != '_bgalloc':
                                    log('rawserver::listen_forever: run func:', func.func_name)
                                st = time.time()
                            func()
                            if DEBUG_TASKS:
                                et = time.time()
                                diff = et - st
                                log('rawserver::listen_forever:', func.func_name, 'took %.5f' % diff)
                        except (SystemError, MemoryError) as e:
                            self.failfunc(e)
                            return
                        except KeyboardInterrupt as e:
                            return
                        except error:
                            log('rawserver::listen_forever: func exception')
                            print_exc()
                        except Exception as e:
                            raise

                    self.sockethandler.close_dead()
                    self.sockethandler.handle_events(events)
                    if self.doneflag.isSet():
                        if DEBUG:
                            log('rawserver::listen_forever: stopping because done flag set2')
                        return
                    self.sockethandler.close_dead()
                except (SystemError, MemoryError) as e:
                    if DEBUG:
                        log('rawserver::listen_forever: SYS/MEM exception', e)
                    self.failfunc(e)
                    return
                except error:
                    if DEBUG:
                        log('rawserver::listen_forever: ERROR exception')
                        print_exc()
                    if self.doneflag.isSet():
                        return
                except KeyboardInterrupt as e:
                    self.failfunc(e)
                    return
                except Exception as e:
                    raise

        finally:
            self.finished.set()

    def is_finished(self):
        return self.finished.isSet()

    def wait_until_finished(self):
        self.finished.wait()

    def _kill_tasks(self):
        if self.tasks_to_kill:
            new_funcs = []
            for t, func, id in self.funcs:
                if id not in self.tasks_to_kill:
                    new_funcs.append((t, func, id))

            self.funcs = new_funcs
            self.tasks_to_kill = []

    def kill_tasks(self, id):
        self.tasks_to_kill.append(id)

    def exception(self, e, kbint = False):
        if not kbint:
            self.excflag.set()
        self.exccount += 1
        if self.errorfunc is None:
            print_exc()
        elif not kbint:
            self.errorfunc(e)

    def shutdown(self):
        if DEBUG:
            log('rawserver::shuwdown: ---')
        self.sockethandler.shutdown()

    def create_udpsocket(self, port, host):
        if DEBUG:
            log('rawserver::create_udpsocket: host', host, 'port', port)
        return self.sockethandler.create_udpsocket(port, host)

    def start_listening_udp(self, serversocket, handler):
        if DEBUG:
            log('rawserver::start_listening_udp: serversocket', serversocket, 'handler', handler)
        self.sockethandler.start_listening_udp(serversocket, handler)

    def stop_listening_udp(self, serversocket):
        if DEBUG:
            log('rawserver::stop_listening_udp: serversocket', serversocket)
        self.sockethandler.stop_listening_udp(serversocket)
Beispiel #42
0
class CoversWidget(QWidget):  # {{{

    chosen = pyqtSignal()
    finished = pyqtSignal()

    def __init__(self, log, current_cover, parent=None):
        QWidget.__init__(self, parent)
        self.log = log
        self.abort = Event()

        self.l = l = QGridLayout()
        self.setLayout(l)

        self.msg = QLabel()
        self.msg.setWordWrap(True)
        l.addWidget(self.msg, 0, 0)

        self.covers_view = CoversView(current_cover, self)
        self.covers_view.chosen.connect(self.chosen)
        l.addWidget(self.covers_view, 1, 0)
        self.continue_processing = True

    def reset_covers(self):
        self.covers_view.reset_covers()

    def start(self, book, current_cover, title, authors, caches):
        self.continue_processing = True
        self.abort.clear()
        self.book, self.current_cover = book, current_cover
        self.title, self.authors = title, authors
        self.log('Starting cover download for:', book.title)
        self.log('Query:', title, authors, self.book.identifiers)
        self.msg.setText('<p>'+
            _('Downloading covers for <b>%s</b>, please wait...')%book.title)
        self.covers_view.start()

        self.worker = CoverWorker(self.log, self.abort, self.title,
                self.authors, book.identifiers, caches)
        self.worker.start()
        QTimer.singleShot(50, self.check)
        self.covers_view.setFocus(Qt.OtherFocusReason)

    def check(self):
        if self.worker.is_alive() and not self.abort.is_set():
            QTimer.singleShot(50, self.check)
            try:
                self.process_result(self.worker.rq.get_nowait())
            except Empty:
                pass
        else:
            self.process_results()

    def process_results(self):
        while self.continue_processing:
            try:
                self.process_result(self.worker.rq.get_nowait())
            except Empty:
                break

        if self.continue_processing:
            self.covers_view.clear_failed()

        if self.worker.error and self.worker.error.strip():
            error_dialog(self, _('Download failed'),
                    _('Failed to download any covers, click'
                        ' "Show details" for details.'),
                    det_msg=self.worker.error, show=True)

        num = self.covers_view.model().rowCount()
        if num < 2:
            txt = _('Could not find any covers for <b>%s</b>')%self.book.title
        else:
            txt = _('Found <b>%(num)d</b> possible covers for %(title)s. '
                    'When the download completes, the covers will be sorted by size.')%dict(num=num-1,
                            title=self.title)
        self.msg.setText(txt)
        self.msg.setWordWrap(True)
        self.covers_view.stop()

        self.finished.emit()

    def process_result(self, result):
        if not self.continue_processing:
            return
        plugin_name, width, height, fmt, data = result
        self.covers_view.model().update_result(plugin_name, width, height, data)

    def cleanup(self):
        self.covers_view.delegate.stop_animation()
        self.continue_processing = False

    def cancel(self):
        self.cleanup()
        self.abort.set()

    def cover_pixmap(self):
        idx = None
        for i in self.covers_view.selectionModel().selectedIndexes():
            if i.isValid():
                idx = i
                break
        if idx is None:
            idx = self.covers_view.currentIndex()
        return self.covers_view.model().cover_pixmap(idx)
Beispiel #43
0
class QuantumProgram(object):
    """Quantum Program Class.

     Class internal properties.

     Elements that are not python identifiers or string constants are denoted
     by "--description (type)--". For example, a circuit's name is denoted by
     "--circuit name (string)--" and might have the value "teleport".

     Internal::

        __quantum_registers (list[dic]): An dictionary of quantum registers
            used in the quantum program.
            __quantum_registers =
                {
                --register name (string)--: QuantumRegistor,
                }
        __classical_registers (list[dic]): An ordered list of classical
            registers used in the quantum program.
            __classical_registers =
                {
                --register name (string)--: ClassicalRegistor,
                }
        __quantum_program (dic): An dictionary of quantum circuits
            __quantum_program =
                {
                --circuit name (string)--:  --circuit object --,
                }
        __init_circuit (obj): A quantum circuit object for the initial quantum
            circuit
        __ONLINE_BACKENDS (list[str]): A list of online backends
        __LOCAL_BACKENDS (list[str]): A list of local backends
     """
    # -- FUTURE IMPROVEMENTS --
    # TODO: for status results make ALL_CAPS (check) or some unified method
    # TODO: Jay: coupling_map, basis_gates will move into a config object
    # only exists once you set the api to use the online backends
    __api = {}
    __api_config = {}

    def __init__(self, specs=None):
        self.__quantum_registers = {}
        self.__classical_registers = {}
        self.__quantum_program = {}  # stores all the quantum programs
        self.__init_circuit = None  # stores the intial quantum circuit of the
        # program
        self.__ONLINE_BACKENDS = []  # pylint: disable=invalid-name
        self.__LOCAL_BACKENDS = qiskit.backends.local_backends()  # pylint: disable=invalid-name
        self.mapper = mapper
        if specs:
            self.__init_specs(specs)

        self.callback = None
        self.jobs_results = []
        self.jobs_results_ready_event = Event()
        self.are_multiple_results = False  # are we expecting multiple results?

    def enable_logs(self, level=logging.INFO):
        """Enable the console output of the logging messages.

        Enable the output of logging messages (above level `level`) to the
        console, by configuring the `qiskit` logger accordingly.

        Params:
            level (int): minimum severity of the messages that are displayed.

        Note:
            This is a convenience method over the standard Python logging
            facilities, and modifies the configuration of the 'qiskit.*'
            loggers. If finer control over the logging configuration is needed,
            it is encouraged to bypass this method.
        """
        # Update the handlers and formatters.
        set_qiskit_logger()
        # Set the logger level.
        logging.getLogger('qiskit').setLevel(level)

    def disable_logs(self):
        """Disable the console output of the logging messages.

        Disable the output of logging messages (above level `level`) to the
        console, by removing the handlers from the `qiskit` logger.

        Note:
            This is a convenience method over the standard Python logging
            facilities, and modifies the configuration of the 'qiskit.*'
            loggers. If finer control over the logging configuration is needed,
            it is encouraged to bypass this method.
        """
        unset_qiskit_logger()

    ###############################################################
    # methods to initiate an build a quantum program
    ###############################################################

    def __init_specs(self, specs):
        """Populate the Quantum Program Object with initial Specs.

        Args:
            specs (dict):
                    Q_SPECS = {
                        "circuits": [{
                            "name": "Circuit",
                            "quantum_registers": [{
                                "name": "qr",
                                "size": 4
                            }],
                            "classical_registers": [{
                                "name": "cr",
                                "size": 4
                            }]
                        }],
        """
        quantumr = []
        classicalr = []
        if "circuits" in specs:
            for circuit in specs["circuits"]:
                quantumr = self.create_quantum_registers(
                    circuit["quantum_registers"])
                classicalr = self.create_classical_registers(
                    circuit["classical_registers"])
                self.create_circuit(name=circuit["name"],
                                    qregisters=quantumr,
                                    cregisters=classicalr)
        # TODO: Jay: I think we should return function handles for the
        # registers and circuit. So that we dont need to get them after we
        # create them with get_quantum_register etc

    def create_quantum_register(self, name, size):
        """Create a new Quantum Register.

        Args:
            name (str): the name of the quantum register
            size (int): the size of the quantum register

        Returns:
            QuantumRegister: internal reference to a quantum register in __quantum_registers

        Raises:
            QISKitError: if the register already exists in the program.
        """
        if name in self.__quantum_registers:
            if size != len(self.__quantum_registers[name]):
                raise QISKitError("Can't make this register: Already in"
                                  " program with different size")
            logger.info(">> quantum_register exists: %s %s", name, size)
        else:
            self.__quantum_registers[name] = QuantumRegister(name, size)
            logger.info(">> new quantum_register created: %s %s", name, size)
        return self.__quantum_registers[name]

    def create_quantum_registers(self, register_array):
        """Create a new set of Quantum Registers based on a array of them.

        Args:
            register_array (list[dict]): An array of quantum registers in
                dictionay format::

                    "quantum_registers": [
                        {
                        "name": "qr",
                        "size": 4
                        },
                        ...
                    ]
        Returns:
            list(QuantumRegister): Array of quantum registers objects
        """
        new_registers = []
        for register in register_array:
            register = self.create_quantum_register(register["name"],
                                                    register["size"])
            new_registers.append(register)
        return new_registers

    def create_classical_register(self, name, size):
        """Create a new Classical Register.

        Args:
            name (str): the name of the classical register
            size (int): the size of the classical register
        Returns:
            ClassicalRegister: internal reference to a classical register in __classical_registers

        Raises:
            QISKitError: if the register already exists in the program.
        """
        if name in self.__classical_registers:
            if size != len(self.__classical_registers[name]):
                raise QISKitError("Can't make this register: Already in"
                                  " program with different size")
            logger.info(">> classical register exists: %s %s", name, size)
        else:
            logger.info(">> new classical register created: %s %s", name, size)
            self.__classical_registers[name] = ClassicalRegister(name, size)
        return self.__classical_registers[name]

    def create_classical_registers(self, registers_array):
        """Create a new set of Classical Registers based on a array of them.

        Args:
            registers_array (list[dict]): An array of classical registers in
                dictionay fromat::

                    "classical_registers": [
                        {
                        "name": "qr",
                        "size": 4
                        },
                        ...
                    ]
        Returns:
            list(ClassicalRegister): Array of clasical registers objects
        """
        new_registers = []
        for register in registers_array:
            new_registers.append(
                self.create_classical_register(register["name"],
                                               register["size"]))
        return new_registers

    def create_circuit(self, name, qregisters=None, cregisters=None):
        """Create a empty Quantum Circuit in the Quantum Program.

        Args:
            name (str): the name of the circuit.
            qregisters (list(QuantumRegister)): is an Array of Quantum Registers by object
                reference
            cregisters (list(ClassicalRegister)): is an Array of Classical Registers by
                object reference

        Returns:
            QuantumCircuit: A quantum circuit is created and added to the Quantum Program
        """
        if not qregisters:
            qregisters = []
        if not cregisters:
            cregisters = []
        quantum_circuit = QuantumCircuit()
        if not self.__init_circuit:
            self.__init_circuit = quantum_circuit
        for register in qregisters:
            quantum_circuit.add(register)
        for register in cregisters:
            quantum_circuit.add(register)
        self.add_circuit(name, quantum_circuit)
        return self.__quantum_program[name]

    def add_circuit(self, name, quantum_circuit):
        """Add a new circuit based on an Object representation.

        Args:
            name (str): the name of the circuit to add.
            quantum_circuit (QuantumCircuit): a quantum circuit to add to the program-name
        """
        for qname, qreg in quantum_circuit.get_qregs().items():
            self.create_quantum_register(qname, len(qreg))
        for cname, creg in quantum_circuit.get_cregs().items():
            self.create_classical_register(cname, len(creg))
        self.__quantum_program[name] = quantum_circuit

    def load_qasm_file(self,
                       qasm_file,
                       name=None,
                       basis_gates='u1,u2,u3,cx,id'):
        """ Load qasm file into the quantum program.

        Args:
            qasm_file (str): a string for the filename including its location.
            name (str or None): the name of the quantum circuit after
                loading qasm text into it. If no name is give the name is of
                the text file.
            basis_gates (str): basis gates for the quantum circuit.
        Returns:
            str: Adds a quantum circuit with the gates given in the qasm file to the
            quantum program and returns the name to be used to get this circuit
        Raises:
            QISKitError: if the file cannot be read.
        """
        if not os.path.exists(qasm_file):
            raise QISKitError('qasm file "{0}" not found'.format(qasm_file))
        if not name:
            name = os.path.splitext(os.path.basename(qasm_file))[0]
        node_circuit = qasm.Qasm(filename=qasm_file).parse()  # Node (AST)
        logger.info("circuit name: " + name)
        logger.info("******************************")
        logger.info(node_circuit.qasm())
        # current method to turn it a DAG quantum circuit.
        unrolled_circuit = unroll.Unroller(
            node_circuit, unroll.CircuitBackend(basis_gates.split(",")))
        circuit_unrolled = unrolled_circuit.execute()
        self.add_circuit(name, circuit_unrolled)
        return name

    def load_qasm_text(self,
                       qasm_string,
                       name=None,
                       basis_gates='u1,u2,u3,cx,id'):
        """ Load qasm string in the quantum program.

        Args:
            qasm_string (str): a string for the file name.
            name (str or None): the name of the quantum circuit after loading qasm
                text into it. If no name is give the name is of the text file.
            basis_gates (str): basis gates for the quantum circuit.
        Returns:
            str: Adds a quantum circuit with the gates given in the qasm string to
            the quantum program.
        """
        node_circuit = qasm.Qasm(data=qasm_string).parse()  # Node (AST)
        if not name:
            # Get a random name if none is given
            name = "".join([
                random.choice(string.ascii_letters + string.digits)
                for n in range(10)
            ])
        logger.info("circuit name: " + name)
        logger.info("******************************")
        logger.info(node_circuit.qasm())
        # current method to turn it a DAG quantum circuit.
        unrolled_circuit = unroll.Unroller(
            node_circuit, unroll.CircuitBackend(basis_gates.split(",")))
        circuit_unrolled = unrolled_circuit.execute()
        self.add_circuit(name, circuit_unrolled)
        return name

    ###############################################################
    # methods to get elements from a QuantumProgram
    ###############################################################

    def get_quantum_register(self, name):
        """Return a Quantum Register by name.

        Args:
            name (str): the name of the quantum register
        Returns:
            QuantumRegister: The quantum register with this name
        Raises:
            KeyError: if the quantum register is not on the quantum program.
        """
        try:
            return self.__quantum_registers[name]
        except KeyError:
            raise KeyError('No quantum register "{0}"'.format(name))

    def get_classical_register(self, name):
        """Return a Classical Register by name.

        Args:
            name (str): the name of the classical register
        Returns:
            ClassicalRegister: The classical register with this name
        Raises:
            KeyError: if the classical register is not on the quantum program.
        """
        try:
            return self.__classical_registers[name]
        except KeyError:
            raise KeyError('No classical register "{0}"'.format(name))

    def get_quantum_register_names(self):
        """Return all the names of the quantum Registers."""
        return self.__quantum_registers.keys()

    def get_classical_register_names(self):
        """Return all the names of the classical Registers."""
        return self.__classical_registers.keys()

    def get_circuit(self, name):
        """Return a Circuit Object by name
        Args:
            name (str): the name of the quantum circuit
        Returns:
            QuantumCircuit: The quantum circuit with this name
        Raises:
            KeyError: if the circuit is not on the quantum program.
        """
        try:
            return self.__quantum_program[name]
        except KeyError:
            raise KeyError('No quantum circuit "{0}"'.format(name))

    def get_circuit_names(self):
        """Return all the names of the quantum circuits."""
        return self.__quantum_program.keys()

    def get_qasm(self, name):
        """Get qasm format of circuit by name.

        Args:
            name (str): name of the circuit

        Returns:
            str: The quantum circuit in qasm format
        """
        quantum_circuit = self.get_circuit(name)
        return quantum_circuit.qasm()

    def get_qasms(self, list_circuit_name):
        """Get qasm format of circuit by list of names.

        Args:
            list_circuit_name (list[str]): names of the circuit

        Returns:
            list(QuantumCircuit): List of quantum circuit in qasm format
        """
        qasm_source = []
        for name in list_circuit_name:
            qasm_source.append(self.get_qasm(name))
        return qasm_source

    def get_initial_circuit(self):
        """Return the initialization Circuit."""
        return self.__init_circuit

    ###############################################################
    # methods for working with backends
    ###############################################################

    def set_api(self, token, url, verify=True):
        """ Setup the API.

        Fills the __ONLINE_BACKENDS, __api, and __api_config variables.
        Does not catch exceptions from IBMQuantumExperience.

        Args:
            token (str): The token used to register on the online backend such
                as the quantum experience.
            url (str): The url used for online backend such as the quantum
                experience.
            verify (bool): If False, ignores SSL certificates errors.
        Raises:
            ConnectionError: if the API instantiation failed.
        """
        try:
            self.__api = IBMQuantumExperience(token, {"url": url}, verify)
        except Exception as ex:
            raise ConnectionError(
                "Couldn't connect to IBMQuantumExperience server: {0}".format(
                    ex))
        qiskit.backends.discover_remote_backends(self.__api)
        self.__ONLINE_BACKENDS = self.online_backends()
        self.__api_config["token"] = token
        self.__api_config["url"] = {"url": url}

    def get_api_config(self):
        """Return the program specs."""
        return self.__api_config

    def get_api(self):
        """Returns a function handle to the API."""
        return self.__api

    def save(self, file_name=None, beauty=False):
        """ Save Quantum Program in a Json file.

        Args:
            file_name (str): file name and path.
            beauty (boolean): save the text with indent 4 to make it readable.

        Returns:
            dict: The dictionary with the status and result of the operation

        Raises:
            LookupError: if the file_name is not correct, or writing to the
                file resulted in an error.
        """
        if file_name is None:
            error = {"status": "Error", "result": "Not filename provided"}
            raise LookupError(error['result'])

        if beauty:
            indent = 4
        else:
            indent = 0

        elemements_to_save = self.__quantum_program
        elements_saved = {}

        for circuit in elemements_to_save:
            elements_saved[circuit] = {}
            elements_saved[circuit]["qasm"] = elemements_to_save[circuit].qasm(
            )

        try:
            with open(file_name, 'w') as save_file:
                json.dump(elements_saved, save_file, indent=indent)
            return {'status': 'Done', 'result': elemements_to_save}
        except ValueError:
            error = {
                'status': 'Error',
                'result': 'Some Problem happened to save the file'
            }
            raise LookupError(error['result'])

    def load(self, file_name=None):
        """ Load Quantum Program Json file into the Quantum Program object.

        Args:
            file_name (str): file name and path.

        Returns:
            dict: The dictionary with the status and result of the operation

        Raises:
            LookupError: if the file_name is not correct, or reading from the
                file resulted in an error.
        """
        if file_name is None:
            error = {"status": "Error", "result": "Not filename provided"}
            raise LookupError(error['result'])

        try:
            with open(file_name, 'r') as load_file:
                elemements_loaded = json.load(load_file)

            for circuit in elemements_loaded:
                circuit_qasm = elemements_loaded[circuit]["qasm"]
                elemements_loaded[circuit] = qasm.Qasm(
                    data=circuit_qasm).parse()
            self.__quantum_program = elemements_loaded

            return {"status": 'Done', 'result': self.__quantum_program}

        except ValueError:
            error = {
                'status': 'Error',
                'result': 'Some Problem happened to load the file'
            }
            raise LookupError(error['result'])

    def available_backends(self):
        """All the backends that are seen by QISKIT."""
        return self.__ONLINE_BACKENDS + self.__LOCAL_BACKENDS

    def online_backends(self):
        """Get the online backends.

        Queries network API if it exists and gets the backends that are online.

        Returns:
            list(str): List of online backends names if the online api has been set or an empty
                list if it has not been set.

        Raises:
            ConnectionError: if the API call failed.
        """
        if self.get_api():
            try:
                backends = self.__api.available_backends()
            except Exception as ex:
                raise ConnectionError(
                    "Couldn't get available backend list: {0}".format(ex))
            return [backend['name'] for backend in backends]
        return []

    def online_simulators(self):
        """Gets online simulators via QX API calls.

        Returns:
            list(str): List of online simulator names.

        Raises:
            ConnectionError: if the API call failed.
        """
        online_simulators_list = []
        if self.get_api():
            try:
                backends = self.__api.available_backends()
            except Exception as ex:
                raise ConnectionError(
                    "Couldn't get available backend list: {0}".format(ex))
            for backend in backends:
                if backend['simulator']:
                    online_simulators_list.append(backend['name'])
        return online_simulators_list

    def online_devices(self):
        """Gets online devices via QX API calls.

        Returns:
            list(str): List of online devices names.

        Raises:
            ConnectionError: if the API call failed.
        """
        devices = []
        if self.get_api():
            try:
                backends = self.__api.available_backends()
            except Exception as ex:
                raise ConnectionError(
                    "Couldn't get available backend list: {0}".format(ex))
            for backend in backends:
                if not backend['simulator']:
                    devices.append(backend['name'])
        return devices

    def get_backend_status(self, backend):
        """Return the online backend status.

        It uses QX API call or by local backend is the name of the
        local or online simulator or experiment.

        Args:
            backend (str): The backend to check

        Returns:
            dict: {'available': True}

        Raises:
            ConnectionError: if the API call failed.
            ValueError: if the backend is not available.
        """

        if backend in self.__ONLINE_BACKENDS:
            try:
                return self.__api.backend_status(backend)
            except Exception as ex:
                raise ConnectionError(
                    "Couldn't get backend status: {0}".format(ex))
        elif backend in self.__LOCAL_BACKENDS:
            return {'available': True}
        else:
            raise ValueError(
                'the backend "{0}" is not available'.format(backend))

    def get_backend_configuration(self, backend, list_format=False):
        """Return the configuration of the backend.

        The return is via QX API call.

        Args:
            backend (str):  Name of the backend.
            list_format (bool): Struct used for the configuration coupling
                map: dict (if False) or list (if True).

        Returns:
            dict: The configuration of the named backend.

        Raises:
            ConnectionError: if the API call failed.
            LookupError: if a configuration for the named backend can't be
                found.
        """
        if self.get_api():
            configuration_edit = {}
            try:
                backends = self.__api.available_backends()
            except Exception as ex:
                raise ConnectionError(
                    "Couldn't get available backend list: {0}".format(ex))
            for configuration in backends:
                if configuration['name'] == backend:
                    for key in configuration:
                        new_key = convert(key)
                        # TODO: removed these from the API code
                        if new_key not in [
                                'id', 'serial_number', 'topology_id', 'status',
                                'coupling_map'
                        ]:
                            configuration_edit[new_key] = configuration[key]
                        if new_key == 'coupling_map':
                            if configuration[key] == 'all-to-all':
                                configuration_edit[new_key] = \
                                    configuration[key]
                            else:
                                if not list_format:
                                    cmap = mapper.coupling_list2dict(
                                        configuration[key])
                                else:
                                    cmap = configuration[key]
                                configuration_edit[new_key] = cmap
                    return configuration_edit
        else:
            return qiskit.backends.get_backend_configuration(backend)

    def get_backend_calibration(self, backend):
        """Return the online backend calibrations.

        The return is via QX API call.

        Args:
            backend (str):  Name of the backend.

        Returns:
            dict: The calibration of the named backend.

        Raises:
            ConnectionError: if the API call failed.
            LookupError: If a configuration for the named backend can't be
                found.
        """
        if backend in self.__ONLINE_BACKENDS:
            try:
                calibrations = self.__api.backend_calibration(backend)
            except Exception as ex:
                raise ConnectionError(
                    "Couldn't get backend calibration: {0}".format(ex))
            calibrations_edit = {}
            for key, vals in calibrations.items():
                new_key = convert(key)
                calibrations_edit[new_key] = vals
            return calibrations_edit
        elif backend in self.__LOCAL_BACKENDS:
            return {'backend': backend, 'calibrations': None}
        else:
            raise LookupError(
                'backend calibration for "{0}" not found'.format(backend))

    def get_backend_parameters(self, backend):
        """Return the online backend parameters.

        The return is via QX API call.

        Args:
            backend (str):  Name of the backend.

        Returns:
            dict: The configuration of the named backend.

        Raises:
            ConnectionError: if the API call failed.
            LookupError: If a configuration for the named backend can't be
                found.
        """
        if backend in self.__ONLINE_BACKENDS:
            try:
                parameters = self.__api.backend_parameters(backend)
            except Exception as ex:
                raise ConnectionError(
                    "Couldn't get backend paramters: {0}".format(ex))
            parameters_edit = {}
            for key, vals in parameters.items():
                new_key = convert(key)
                parameters_edit[new_key] = vals
            return parameters_edit
        elif backend in self.__LOCAL_BACKENDS:
            return {'backend': backend, 'parameters': None}
        else:
            raise LookupError(
                'backend parameters for "{0}" not found'.format(backend))

    ###############################################################
    # methods to compile quantum programs into qobj
    ###############################################################

    def compile(self,
                name_of_circuits,
                backend="local_qasm_simulator",
                config=None,
                basis_gates=None,
                coupling_map=None,
                initial_layout=None,
                shots=1024,
                max_credits=3,
                seed=None,
                qobj_id=None):
        """Compile the circuits into the exectution list.

        This builds the internal "to execute" list which is list of quantum
        circuits to run on different backends.

        Args:
            name_of_circuits (list[str]): circuit names to be compiled.
            backend (str): a string representing the backend to compile to
            config (dict): a dictionary of configurations parameters for the
                compiler
            basis_gates (str): a comma seperated string and are the base gates,
                               which by default are: u1,u2,u3,cx,id
            coupling_map (dict): A directed graph of coupling::

                {
                 control(int):
                     [
                         target1(int),
                         target2(int),
                         , ...
                     ],
                     ...
                }

                eg. {0: [2], 1: [2], 3: [2]}

            initial_layout (dict): A mapping of qubit to qubit::

                                  {
                                    ("q", strart(int)): ("q", final(int)),
                                    ...
                                  }
                                  eg.
                                  {
                                    ("q", 0): ("q", 0),
                                    ("q", 1): ("q", 1),
                                    ("q", 2): ("q", 2),
                                    ("q", 3): ("q", 3)
                                  }

            shots (int): the number of shots
            max_credits (int): the max credits to use 3, or 5
            seed (int): the intial seed the simulatros use
            qobj_id (str): identifier of the qobj.

        Returns:
            dict: the job id and populates the qobj::

            qobj =
                {
                    id: --job id (string),
                    config: -- dictionary of config settings (dict)--,
                        {
                        "max_credits" (online only): -- credits (int) --,
                        "shots": -- number of shots (int) --.
                        "backend": -- backend name (str) --
                        }
                    circuits:
                        [
                            {
                            "name": --circuit name (string)--,
                            "compiled_circuit": --compiled quantum circuit (JSON format)--,
                            "compiled_circuit_qasm": --compiled quantum circuit (QASM format)--,
                            "config": --dictionary of additional config settings (dict)--,
                                {
                                "coupling_map": --adjacency list (dict)--,
                                "basis_gates": --comma separated gate names (string)--,
                                "layout": --layout computed by mapper (dict)--,
                                "seed": (simulator only)--initial seed for the simulator (int)--,
                                }
                            },
                            ...
                        ]
                    }

        Raises:
            ValueError: if no names of the circuits have been specified.
            QISKitError: if any of the circuit names cannot be found on the
                Quantum Program.
        """
        # TODO: Jay: currently basis_gates, coupling_map, initial_layout,
        # shots, max_credits and seed are extra inputs but I would like
        # them to go into the config.

        qobj = {}
        if not qobj_id:
            qobj_id = "".join([
                random.choice(string.ascii_letters + string.digits)
                for n in range(30)
            ])
        qobj['id'] = qobj_id
        qobj["config"] = {
            "max_credits": max_credits,
            'backend': backend,
            "shots": shots
        }
        qobj["circuits"] = []

        if not name_of_circuits:
            raise ValueError('"name_of_circuits" must be specified')
        if isinstance(name_of_circuits, str):
            name_of_circuits = [name_of_circuits]
        for name in name_of_circuits:
            if name not in self.__quantum_program:
                raise QISKitError(
                    'circuit "{0}" not found in program'.format(name))
            if not basis_gates:
                basis_gates = "u1,u2,u3,cx,id"  # QE target basis
            # TODO: The circuit object going into this is to have .qasm() method (be careful)
            circuit = self.__quantum_program[name]
            dag_circuit, final_layout = openquantumcompiler.compile(
                circuit.qasm(),
                basis_gates=basis_gates,
                coupling_map=coupling_map,
                initial_layout=initial_layout,
                get_layout=True)
            # making the job to be added to qobj
            job = {}
            job["name"] = name
            # config parameters used by the runner
            if config is None:
                config = {}  # default to empty config dict
            job["config"] = copy.deepcopy(config)
            job["config"]["coupling_map"] = mapper.coupling_dict2list(
                coupling_map)
            # TODO: Jay: make config options optional for different backends
            # Map the layout to a format that can be json encoded
            list_layout = None
            if final_layout:
                list_layout = [[k, v] for k, v in final_layout.items()]
            job["config"]["layout"] = list_layout
            job["config"]["basis_gates"] = basis_gates
            if seed is None:
                job["config"]["seed"] = None
            else:
                job["config"]["seed"] = seed
            # the compiled circuit to be run saved as a dag
            job["compiled_circuit"] = openquantumcompiler.dag2json(
                dag_circuit, basis_gates=basis_gates)
            job["compiled_circuit_qasm"] = dag_circuit.qasm(qeflag=True)
            # add job to the qobj
            qobj["circuits"].append(job)
        return qobj

    def reconfig(self,
                 qobj,
                 backend=None,
                 config=None,
                 shots=None,
                 max_credits=None,
                 seed=None):
        """Change configuration parameters for a compile qobj. Only parameters which
        don't affect the circuit compilation can change, e.g., the coupling_map
        cannot be changed here!

        Notes:
            If the inputs are left as None then the qobj is not updated

        Args:
            qobj (dict): already compile qobj
            backend (str): see .compile
            config (dict): see .compile
            shots (int): see .compile
            max_credits (int): see .compile
            seed (int): see .compile

        Returns:
            qobj: updated qobj
        """
        if backend is not None:
            qobj['config']['backend'] = backend
        if shots is not None:
            qobj['config']['shots'] = shots
        if max_credits is not None:
            qobj['config']['max_credits'] = max_credits

        for circuits in qobj['circuits']:
            if seed is not None:
                circuits['seed'] = seed
            if config is not None:
                circuits['config'].update(config)

        return qobj

    def get_execution_list(self, qobj):
        """Print the compiled circuits that are ready to run.

        Note:
            This method is intended to be used during interactive sessions, and
            prints directly to stdout instead of using the logger.

        Returns:
            list(str): names of the circuits in `qobj`
        """
        if not qobj:
            print("no executions to run")
        execution_list = []

        print("id: %s" % qobj['id'])
        print("backend: %s" % qobj['config']['backend'])
        print("qobj config:")
        for key in qobj['config']:
            if key != 'backend':
                print(' ' + key + ': ' + str(qobj['config'][key]))
        for circuit in qobj['circuits']:
            execution_list.append(circuit["name"])
            print('  circuit name: ' + circuit["name"])
            print('  circuit config:')
            for key in circuit['config']:
                print('   ' + key + ': ' + str(circuit['config'][key]))
        return execution_list

    def get_compiled_configuration(self, qobj, name):
        """Get the compiled layout for the named circuit and backend.

        Args:
            name (str):  the circuit name
            qobj (dict): the qobj

        Returns:
            dict: the config of the circuit.

        Raises:
            QISKitError: if the circuit has no configurations
        """
        try:
            for index in range(len(qobj["circuits"])):
                if qobj["circuits"][index]['name'] == name:
                    return qobj["circuits"][index]["config"]
        except KeyError:
            raise QISKitError(
                'No compiled configurations for circuit "{0}"'.format(name))

    def get_compiled_qasm(self, qobj, name):
        """Return the compiled cricuit in qasm format.

        Args:
            qobj (dict): the qobj
            name (str): name of the quantum circuit

        Returns:
            str: the QASM of the compiled circuit.

        Raises:
            QISKitError: if the circuit has no configurations
        """
        try:
            for index in range(len(qobj["circuits"])):
                if qobj["circuits"][index]['name'] == name:
                    return qobj["circuits"][index]["compiled_circuit_qasm"]
        except KeyError:
            raise QISKitError(
                'No compiled qasm for circuit "{0}"'.format(name))

    ###############################################################
    # methods to run quantum programs
    ###############################################################

    def run(self, qobj, wait=5, timeout=60):
        """Run a program (a pre-compiled quantum program). This function will
        block until the Job is processed.

        The program to run is extracted from the qobj parameter.

        Args:
            qobj (dict): the dictionary of the quantum object to run.
            wait (int): Time interval to wait between requests for results
            timeout (int): Total time to wait until the execution stops

        Returns:
            Result: A Result (class).
        """
        self.callback = None
        self._run_internal([qobj], wait=wait, timeout=timeout)
        self.wait_for_results(timeout)
        return self.jobs_results[0]

    def run_batch(self, qobj_list, wait=5, timeout=120):
        """Run various programs (a list of pre-compiled quantum programs). This
        function will block until all programs are processed.

        The programs to run are extracted from qobj elements of the list.

        Args:
            qobj_list (list(dict)): The list of quantum objects to run.
            wait (int): Time interval to wait between requests for results
            timeout (int): Total time to wait until the execution stops

        Returns:
            list(Result): A list of Result (class). The list will contain one Result object
            per qobj in the input list.
        """
        self._run_internal(qobj_list,
                           wait=wait,
                           timeout=timeout,
                           are_multiple_results=True)
        self.wait_for_results(timeout)
        return self.jobs_results

    def run_async(self, qobj, wait=5, timeout=60, callback=None):
        """Run a program (a pre-compiled quantum program) asynchronously. This
        is a non-blocking function, so it will return inmediately.

        All input for run comes from qobj.

        Args:
            qobj(dict): the dictionary of the quantum object to
                run or list of qobj.
            wait (int): Time interval to wait between requests for results
            timeout (int): Total time to wait until the execution stops
            callback (fn(result)): A function with signature:
                    fn(result):
                    The result param will be a Result object.
        """
        self._run_internal([qobj],
                           wait=wait,
                           timeout=timeout,
                           callback=callback)

    def run_batch_async(self, qobj_list, wait=5, timeout=120, callback=None):
        """Run various programs (a list of pre-compiled quantum program)
        asynchronously. This is a non-blocking function, so it will return
        inmediately.

        All input for run comes from qobj.

        Args:
            qobj_list (list(dict)): The list of quantum objects to run.
            wait (int): Time interval to wait between requests for results
            timeout (int): Total time to wait until the execution stops
            callback (fn(results)): A function with signature:
                    fn(results):
                    The results param will be a list of Result objects, one
                    Result per qobj in the input list.
        """
        self._run_internal(qobj_list,
                           wait=wait,
                           timeout=timeout,
                           callback=callback,
                           are_multiple_results=True)

    def _run_internal(self,
                      qobj_list,
                      wait=5,
                      timeout=60,
                      callback=None,
                      are_multiple_results=False):

        self.callback = callback
        self.are_multiple_results = are_multiple_results

        q_job_list = []
        for qobj in qobj_list:
            q_job = QuantumJob(qobj, preformatted=True)
            q_job_list.append(q_job)

        job_processor = JobProcessor(q_job_list,
                                     max_workers=5,
                                     callback=self._jobs_done_callback)
        job_processor.submit()

    def _jobs_done_callback(self, jobs_results):
        """ This internal callback will be called once all Jobs submitted have
            finished. NOT every time a job has finished.

        Args:
            jobs_results (list): list of Result objects
        """
        if self.callback is None:
            # We are calling from blocking functions (run, run_batch...)
            self.jobs_results = jobs_results
            self.jobs_results_ready_event.set()
            return

        if self.are_multiple_results:
            self.callback(jobs_results)  # for run_batch_async() callback
        else:
            self.callback(jobs_results[0])  # for run_async() callback

    def wait_for_results(self, timeout):
        """Wait for all the results to be ready during an execution."""
        is_ok = self.jobs_results_ready_event.wait(timeout)
        self.jobs_results_ready_event.clear()
        if not is_ok:
            raise QISKitError(
                'Error waiting for Job results: Timeout after {0} '
                'seconds.'.format(timeout))

    def execute(self,
                name_of_circuits,
                backend="local_qasm_simulator",
                config=None,
                wait=5,
                timeout=60,
                basis_gates=None,
                coupling_map=None,
                initial_layout=None,
                shots=1024,
                max_credits=3,
                seed=None):
        """Execute, compile, and run an array of quantum circuits).

        This builds the internal "to execute" list which is list of quantum
        circuits to run on different backends.

        Args:
            name_of_circuits (list[str]): circuit names to be compiled.
            backend (str): a string representing the backend to compile to
            config (dict): a dictionary of configurations parameters for the
                compiler
            wait (int): Time interval to wait between requests for results
            timeout (int): Total time to wait until the execution stops
            basis_gates (str): a comma seperated string and are the base gates,
                               which by default are: u1,u2,u3,cx,id
            coupling_map (dict): A directed graph of coupling::

                                {
                                control(int):
                                    [
                                        target1(int),
                                        target2(int),
                                        , ...
                                    ],
                                    ...
                                }
                                eg. {0: [2], 1: [2], 3: [2]}
            initial_layout (dict): A mapping of qubit to qubit
                                  {
                                  ("q", strart(int)): ("q", final(int)),
                                  ...
                                  }
                                  eg.
                                  {
                                  ("q", 0): ("q", 0),
                                  ("q", 1): ("q", 1),
                                  ("q", 2): ("q", 2),
                                  ("q", 3): ("q", 3)
                                  }
            shots (int): the number of shots
            max_credits (int): the max credits to use 3, or 5
            seed (int): the intial seed the simulatros use

        Returns:
            Result: status done and populates the internal __quantum_program with the
            data
        """
        # TODO: Jay: currently basis_gates, coupling_map, intial_layout, shots,
        # max_credits, and seed are extra inputs but I would like them to go
        # into the config
        qobj = self.compile(name_of_circuits,
                            backend=backend,
                            config=config,
                            basis_gates=basis_gates,
                            coupling_map=coupling_map,
                            initial_layout=initial_layout,
                            shots=shots,
                            max_credits=max_credits,
                            seed=seed)
        result = self.run(qobj, wait=wait, timeout=timeout)
        return result
Beispiel #44
0
class IOLoopThread(Thread):
    """Run a pyzmq ioloop in a thread to send and receive messages
    """
    _exiting = False
    ioloop = None

    def __init__(self):
        super(IOLoopThread, self).__init__()
        self.daemon = True

    @staticmethod
    @atexit.register
    def _notice_exit():
        # Class definitions can be torn down during interpreter shutdown.
        # We only need to set _exiting flag if this hasn't happened.
        if IOLoopThread is not None:
            IOLoopThread._exiting = True

    def start(self):
        """Start the IOLoop thread

        Don't return until self.ioloop is defined,
        which is created in the thread
        """
        self._start_event = Event()
        Thread.start(self)
        self._start_event.wait()

    def run(self):
        """Run my loop, ignoring EINTR events in the poller"""
        if 'asyncio' in sys.modules:
            # tornado may be using asyncio,
            # ensure an eventloop exists for this thread
            import asyncio
            asyncio.set_event_loop(asyncio.new_event_loop())
        self.ioloop = ioloop.IOLoop()
        # signal that self.ioloop is defined
        self._start_event.set()
        while True:
            try:
                self.ioloop.start()
            except ZMQError as e:
                if e.errno == errno.EINTR:
                    continue
                else:
                    raise
            except Exception:
                if self._exiting:
                    break
                else:
                    raise
            else:
                break

    def stop(self):
        """Stop the channel's event loop and join its thread.

        This calls :meth:`~threading.Thread.join` and returns when the thread
        terminates. :class:`RuntimeError` will be raised if
        :meth:`~threading.Thread.start` is called again.
        """
        if self.ioloop is not None:
            self.ioloop.add_callback(self.ioloop.stop)
        self.join()
        self.close()
        self.ioloop = None

    def close(self):
        if self.ioloop is not None:
            try:
                self.ioloop.close(all_fds=True)
            except Exception:
                pass
Beispiel #45
0
class SmuffPlugin(octoprint.plugin.SettingsPlugin,
                  octoprint.plugin.AssetPlugin,
                  octoprint.plugin.TemplatePlugin,
                  octoprint.plugin.StartupPlugin,
                  octoprint.plugin.EventHandlerPlugin,
                  octoprint.plugin.ShutdownPlugin):
    def __init__(self, _logger, _lock):
        self._serial = None  # serial instance to communicate with the SMuFF
        self._serlock = _lock  # lock object for reading/writing
        self._serevent = Event(
        )  # event raised when a valid response has been received
        self._fw_info = "?"  # SMuFFs firmware info
        self._cur_tool = "-1"  # the current tool
        self._pre_tool = "-1"  # the previous tool
        self._pending_tool = "?"  # the tool on a pending tool change
        self._selector = False  # status of the Selector endstop
        self._revolver = False  # status of the Revolver endstop
        self._feeder = False  # status of the Feeder endstop
        self._feeder2 = False  # status of the 2nd Feeder endstop
        self._is_busy = False  # flag set when SMuFF signals "Busy"
        self._is_error = False  # flag set when SMuFF signals "Error"
        self._is_aligned = False  # flag set when Feeder endstop is reached (not used yet)
        self._response = None  # the response string from SMuFF
        self._wait_requested = False  # set when SMuFF requested a "Wait" (in case of jams or similar)
        self._abort_requested = False  # set when SMuFF requested a "Abort"

    ##~~ ShutdownPlugin mixin

    def on_startup(self, host, port):
        self._logger.info("Yeah... starting up...")

    def on_shutdown(self):
        close_SMuFF_serial()
        self._logger.debug("Booo... shutting down...")

    ##~~ StartupPlugin mixin

    def on_after_startup(self):
        port = self._settings.get(["tty"])
        baud = self._settings.get_int(["baudrate"])
        self._logger.debug("Opening serial {0} with {1}".format(port, baud))
        if open_SMuFF_serial(port, baud):
            self._serial = __ser0__
            start_reader_thread()

        # request firmware info from SMuFF
        if self._serial.is_open:
            self._fw_info = self.send_SMuFF_and_wait(M115)
            self._logger.debug("FW-Info: {}".format(self._fw_info))
        pass

    ##~~ EventHandler mixin

    def on_event(self, event, payload):
        #self._logger.debug("Event: [" + event + ", {0}".format(payload) + "]")
        if event == Events.SHUTDOWN:
            self._logger.debug("Shutting down, closing serial")
            close_SMuFF_serial()

    ##~~ SettingsPlugin mixin

    def get_settings_defaults(self):
        self._logger.debug("SMuFF plugin loaded, getting defaults")

        params = dict(firmware_info="No data. Please check connection!",
                      baudrate=SERBAUD,
                      tty=SERDEV,
                      tool=self._cur_tool,
                      selector_end=self._selector,
                      revolver_end=self._revolver,
                      feeder_end=self._feeder,
                      feeder2_end=self._feeder)

        # look up the serial port driver
        #if sys.platform == "win32":
        #	if SERDEV.startswith("tty"):
        #		params['tty'] = "Wrong device on WIN32 ({})".format(SERDEV)
        #	else:
        #		params['tty'] = SERDEV
        #else:
        #	drvr = self.find_file(SERDEV, "/dev")
        #	if len(drvr) > 0:
        #		params['tty'] = "/dev/{}".format(SERDEV)

        return params

    def on_settings_save(self, data):
        global __ser0__
        baud = self._settings.get_int(["baudrate"])
        port = self._settings.get(["tty"])
        octoprint.plugin.SettingsPlugin.on_settings_save(self, data)
        baud_new = self._settings.get_int(["baudrate"])
        port_new = self._settings.get(["tty"])
        self._logger.debug("Settings saved: {0}/{1}  {2}/{3}".format(
            baud, baud_new, port, port_new))
        # did the settings change?
        if not port_new == port or not baud_new == baud:
            # close the previous port, re-open it, start the reader thread and get SMuFF's firmware information
            close_SMuFF_serial()
            open_SMuFF_serial(port_new, baud_new)
            self._serial = __ser0__
            start_reader_thread()
            self._fw_info = self.send_SMuFF_and_wait(M115)

    def get_template_configs(self):
        # self._logger.debug("Settings-Template was requested")
        return [
            dict(type="settings",
                 custom_bindings=True,
                 template='SMuFF_settings.jinja2'),
            dict(type="navbar",
                 custom_bindings=True,
                 template='SMuFF_navbar.jinja2')
        ]

    ##~~ AssetPlugin mixin

    def get_assets(self):
        return dict(js=["js/SMuFF.js"],
                    css=["css/SMuFF.css"],
                    less=["less/SMuFF.less"])

    ##~~ Softwareupdate hook

    def get_update_information(self):
        return dict(SMuFF=dict(
            displayName="SMuFF Plugin",
            displayVersion=self._plugin_version,

            # version check: github repository
            type="github_release",
            user="******",
            repo="OctoPrint-Smuff",
            current=self._plugin_version,

            # update method: pip
            pip=
            "https://github.com/technik-gegg/OctoPrint-Smuff/archive/{target_version}.zip"
        ))

    ##~~ GCode hooks

    def extend_tool_queuing(self, comm_instance, phase, cmd, cmd_type, gcode,
                            subcode, tags, *args, **kwargs):
        # self._logger.debug("Processing queuing: [" + cmd + "," + str(cmd_type)+ "," + str(tags) + "]")

        if gcode and gcode.startswith(TOOL):
            self._logger.debug("OctoPrint current tool: {0}".format(
                comm_instance._currentTool))
            # if the tool that's already loaded is addressed, ignore the filament change
            if cmd == self._cur_tool and self._feeder:
                self._logger.info(cmd + " equals " + self._cur_tool +
                                  " -- no tool change needed")
                return "M117 Tool already selected"
            self._is_aligned = False
            # replace the tool change command
            return [AT_SMUFF + " " + cmd]

        if cmd and cmd.startswith(AT_SMUFF):
            v1 = None
            v2 = None
            spd = 300
            action = None
            tmp = cmd.split()
            if len(tmp):
                action = tmp[1]
                if len(tmp) > 2:
                    v1 = int(tmp[2])
                if len(tmp) > 3:
                    v2 = int(tmp[3])
                if len(tmp) > 4:
                    spd = int(tmp[4])

            # self._logger.debug("1>> " + cmd + "  action: " + str(action) + "  v1,v2: " + str(v1) + ", " + str(v2))

            # @SMuFF MOTORS
            if action and action == MOTORS:
                # send a servo command to SMuFF
                self.send_SMuFF_and_wait(M18)
                return ""

            # @SMuFF FAN
            if action and action == FAN:
                # send a fan command to SMuFF
                if v1 == 1:
                    self.send_SMuFF_and_wait(M106)
                elif v1 == 0:
                    self.send_SMuFF_and_wait(M107)
                return ""

    def extend_tool_sending(self, comm_instance, phase, cmd, cmd_type, gcode,
                            subcode, tags, *args, **kwargs):

        if gcode and gcode.startswith(TOOL):
            return ""

        # is this the replaced tool change command?
        if cmd and cmd.startswith(AT_SMUFF):
            v1 = None
            v2 = None
            spd = 300
            action = None
            tmp = cmd.split()
            if len(tmp):
                action = tmp[1]
                if len(tmp) > 2:
                    v1 = int(tmp[2])
                if len(tmp) > 3:
                    v2 = int(tmp[3])
                if len(tmp) > 4:
                    spd = int(tmp[4])

            self._logger.debug("2>> " + cmd + "  action: " + str(action) +
                               "  v1,v2: " + str(v1) + ", " + str(v2))

            # @SMuFF T0...T99
            if action and action.startswith(TOOL):
                if self._printer.set_job_on_hold(True, False):
                    try:
                        # store the new tool for later
                        self._pending_tool = str(action)
                        # check if there's filament loaded
                        if self._feeder:
                            self._logger.debug(
                                "2>> calling script 'beforeToolChange'")
                            # if so, send the OctoPrints default "Before Tool Change" script to the printer
                            self._printer.script("beforeToolChange")
                        else:
                            self._logger.debug("2>> calling SMuFF LOAD")
                            # not loaded, nothing to retract, so send the tool change to the SMuFF
                            self._printer.commands(AT_SMUFF + " " + LOAD)

                    except UnknownScript:
                        self._logger.error(
                            "Script 'beforeToolChange' not found!")
                    # Notice: set_job_on_hold(False) must not be set yet since the
                    # whole tool change procedure isn't finished yet. Setting it now
                    # will unpause OctoPrint and it'll continue printing without filament!
                    #finally:
                    #self._printer.set_job_on_hold(False)

            # @SMuFF SERVO
            if action and action == SERVO:
                # send a servo command to SMuFF
                self.send_SMuFF_and_wait(M280 + str(v1) + " S" + str(v2))
                return ""

            # @SMuFF SERVOOPEN
            if action and action == SERVOOPEN:
                # send a servo open command to SMuFF
                self.send_SMuFF_and_wait(M280 + str(v1) + " R0")
                return ""

            # @SMuFF SERVOCLOSE
            if action and action == SERVOCLOSE:
                # send a servo close command to SMuFF
                self.send_SMuFF_and_wait(M280 + str(v1) + " R1")
                return ""

            # @SMuFF WIPE
            if action and action == WIPE:
                # send a servo wipe command to SMuFF
                self.send_SMuFF_and_wait(G12)

            # @SMuFF LOAD
            if action and action == LOAD:
                with self._printer.job_on_hold():
                    try:
                        self._logger.debug("1>> LOAD: Feeder: " +
                                           str(self._feeder) + ", Pending: " +
                                           str(self._pending_tool) +
                                           ", Current: " + str(self._cur_tool))
                        retry = 3  # retry up to 3 times
                        while retry > 0:
                            # send a tool change command to SMuFF
                            res = self.send_SMuFF_and_wait(self._pending_tool)
                            # do we have the tool requested now?
                            if str(res) == str(self._pending_tool):
                                self._pre_tool = self._cur_tool
                                self._cur_tool = self._pending_tool
                                comm_instance._currentTool = self.parse_tool_number(
                                    self._cur_tool)
                                self._logger.debug(
                                    "2>> calling script 'afterToolChange'")
                                # send the default OctoPrint "After Tool Change" script to the printer
                                self._printer.script("afterToolChange")
                                retry = 0
                            else:
                                # not the result expected, do it all again
                                self._logger.warning(
                                    "Tool change failed, retrying (<{0}> not <{1}>)"
                                    .format(res, self._pending_tool))
                                if self._abort_requested:
                                    retry = 0
                                if not self._wait_requested:
                                    retry -= 1

                    except UnknownScript:
                        # shouldn't happen at all, since we're using default OctoPrint scripts
                        # but you never know
                        self._logger.error(
                            "Script 'afterToolChange' not found!")

                    finally:
                        # now is the time to release the hold and continue printing
                        self._printer.set_job_on_hold(False)

    def extend_script_variables(self, comm_instance, script_type, script_name,
                                *args, **kwargs):
        # This section was supposed to pass the current endstop states to OctoPrint scripts when requested.
        # Unfortunatelly, OctoPrint caches the variables after the very first call and doesn't update
        # them on consecutive calls while in the same script.
        # This renders my attempt aligning the position of the filament based on the Feeder endstop trigger
        # unusable until OctoPrint updates this feature someday in the future.
        if script_type and script_type == "gcode":
            variables = dict(feeder="on" if self._feeder else "off",
                             feeder2="on" if self._feeder2 else "off",
                             tool=self._cur_tool,
                             aligned="yes" if self._is_aligned else "no")
            #self._logger.debug(" >> Script vars query: [" + str(script_type) + "," + str(script_name) + "] {0}".format(variables))
            return None, None, variables
        return None

    def extend_gcode_received(self, comm_instance, line, *args, **kwargs):
        # Refresh the current tool in OctoPrint on each command coming from the printer - just in case
        # This is needed because OctoPrint manages the current tool itself and it might try to swap
        # tools because of the wrong information.
        comm_instance._currentTool = self.parse_tool_number(self._cur_tool)
        # don't process any of the GCodes received
        return line

    ##~~ helper functions

    # sending data to SMuFF
    def send_SMuFF(self, data):
        if self._serial and self._serial.is_open:
            try:
                b = bytearray(80)  # not expecting commands longer then that
                b = "{}\n".format(data).encode("ascii")
                # lock down the reader thread just in case
                # (shouldn't be needed at all, since simultanous read/write operations should
                # be no big deal)
                self._serlock.acquire()
                n = self._serial.write(b)
                self._serlock.release()
                self._logger.debug("Sending {1} bytes: [{0}]".format(b, n))
                return True
            except (OSError, serial.SerialException):
                self._serlock.release()
                self._logger.error("Can't send command to SMuFF")
                return False
        else:
            self._logger.error("Serial not open")
            return False

    # sending data to SMuFF and waiting for a response
    def send_SMuFF_and_wait(self, data):
        if self.send_SMuFF(data) == False:
            return None

        timeout = 10  # wait max. 10 seconds for a response
        done = False
        resp = None
        self.set_busy(False)  # reset busy and
        self.set_error(False)  # error flags
        while not done:
            self._serevent.clear()
            is_set = self._serevent.wait(timeout)
            if is_set:
                self._logger.info(
                    "To [{0}] SMuFF says [{1}] (is_error = {2})".format(
                        data, self._response, self._is_error))
                resp = self._response
                if self._response == None or self._is_error:
                    done = True
                elif not self._response.startswith('echo:'):
                    done = True

                self._response = None
            else:
                self._logger.info("No event received... aborting")
                if self._is_busy == False:
                    done = True
        return resp

    def find_file(self, pattern, path):
        result = []
        for root, dirs, files in os.walk(path):
            for name in files:
                if fnmatch.fnmatch(name, pattern):
                    result.append(os.path.join(root, name))
        return result

    def set_busy(self, busy):
        self._is_busy = busy

    def set_error(self, error):
        self._is_error = error

    def set_response(self, response):
        self._response = response

    def parse_states(self, states):
        #self._logger.debug("States received: [" + states + "]")
        if len(states) == 0:
            return False
        # Note: SMuFF sends periodically states in this notation:
        # 	"echo: states: T: T4     S: off  R: off  F: off  F2: off"
        m = re.search(
            r'^((\w+:.)(\w+:))\s([T]:\s)(\w+)\s([S]:\s)(\w+)\s([R]:\s)(\w+)\s([F]:\s)(\w+)\s([F,2]+:\s)(\w+)',
            states)
        if m:
            self._cur_tool = m.group(5).strip()
            self._selector = m.group(7).strip() == ESTOP_ON
            self._revolver = m.group(9).strip() == ESTOP_ON
            self._feeder = m.group(11).strip() == ESTOP_ON
            self._feeder2 = m.group(13).strip() == ESTOP_ON
            if hasattr(self, "_plugin_manager"):
                self._plugin_manager.send_plugin_message(
                    self._identifier, {
                        'type': 'status',
                        'tool': self._cur_tool,
                        'feeder': self._feeder,
                        'feeder2': self._feeder2,
                        'fw_info': self._fw_info,
                        'conn': self._serial.is_open
                    })
            return True
        else:
            if hasattr(self, "_logger"):
                self._logger.error("No match in parse_states: [" + states +
                                   "]")
        return False

    def parse_tool_number(self, tool):
        try:
            return int(re.findall(r'[-\d]+', tool)[0])
        except Exception:
            self._logger.error("Can't parse tool number in {0}".format(tool))
        return -1

    def hex_dump(self, s):
        self._logger.debug(":".join("{:02x}".format(ord(c)) for c in s))

    # parse the response we've got from the SMuFF
    def parse_serial_data(self, data):
        if self == None:
            return

        #self._logger.debug("Raw data: [{0}]".format(data.rstrip("\n")))

        global last_response
        self._serevent.clear()

        # after first connect the response from the SMuFF is supposed to be 'start'
        if data.startswith('start\n'):
            self._logger.debug("SMuFF has sent \"start\" response")
            return

        if data.startswith("echo:"):
            # don't process any general debug messages
            if data[6:].startswith("dbg:"):
                self._logger.debug("SMuFF has sent a debug response: [" +
                                   data.rstrip() + "]")
            # but do process the tool/endstop states
            elif data[6:].startswith("states:"):
                last_response = None
                self.parse_states(data.rstrip())
            # and register whether SMuFF is busy
            elif data[6:].startswith("busy"):
                self._logger.debug("SMuFF has sent a busy response: [" +
                                   data.rstrip() + "]")
                self.set_busy(True)
            return

        if data.startswith("error:"):
            self._logger.info("SMuFF has sent a error response: [" +
                              data.rstrip() + "]")
            # maybe the SMuFF has received garbage
            if data[7:].startswith("Unknown command:"):
                self._serial.flushOutput()
            self.set_error(True)
            return

        if data.startswith(ACTION_CMD):
            self._logger.info("SMuFF has sent an action request: [" +
                              data.rstrip() + "]")
            # what action is it? is it a tool change?
            if data[10:].startswith("T"):
                tool = self.parse_tool_number(data[10:])
                # only if the printer isn't printing
                state = self._printer.get_state_id()
                self._logger.debug(
                    "SMuFF requested an action while printer in state '{}'".
                    format(state))
                if state == "OPERATIONAL":
                    # query the nozzle temp
                    temps = self._printer.get_current_temperatures()
                    try:
                        if temps['tool0']['actual'] > 160:
                            self._logger.debug("Nozzle temp. > 160")
                            self._printer.change_tool("tool{}".format(tool))
                            self.send_SMuFF("{0} T: OK".format(ACTION_CMD))
                        else:
                            self._logger.error(
                                "Can't change to tool {}, nozzle too cold".
                                format(tool))
                            self.send_SMuFF(
                                "{0} T: \"Nozzle too cold\"".format(
                                    ACTION_CMD))
                    except:
                        self._logger.debug(
                            "Can't query temperatures. Aborting.")
                        self.send_SMuFF(
                            "{0} T: \"No nozzle temp. avail.\"".format(
                                ACTION_CMD))
                else:
                    self._logger.error(
                        "Can't change to tool {}, printer not ready or printing"
                        .format(tool))
                    self.send_SMuFF(
                        "{0} T: \"Printer not ready\"".format(ACTION_CMD))

            if data[10:].startswith(ACTION_WAIT):
                self._wait_requested = True
                self._logger.debug("waiting for SMuFF to come clear...")

            if data[10:].startswith(ACTION_CONTINUE):
                self._wait_requested = False
                self._abort_requested = False
                self._logger.debug("continuing after SMuFF cleared...")

            if data[10:].startswith(ACTION_ABORT):
                self._wait_requested = False
                self._abort_requested = True
                self._logger.debug("aborting operation...")

            return

        if data.startswith("ok\n"):
            if self._is_error:
                self.set_response(None)
            else:
                self.set_response(last_response)
            self._serevent.set()
            return

        # store the last response before the "ok"
        last_response = data.rstrip("\n")
        self._logger.debug("Received response: [{0}]".format(last_response))
class Tegrastats:
    """
        - Subprocess read:
        https://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python/4896288#4896288
        - Property
        https://www.programiz.com/python-programming/property
    """
    def __init__(self, callback, tegrastats_path):
        self._running = Event()
        # Error message from thread
        self._error = None
        # Start process tegrastats
        self.path = locate_commands("tegrastats", tegrastats_path)
        # Define Tegrastats process
        self._thread = None
        # Initialize callback
        self.callback = callback

    def _decode(self, text):
        # Find and parse all single values
        stats = VALS(text)
        # Parse if exist MTS
        mts = MTS(text)
        if mts:
            stats['MTS'] = mts
        # Parse RAM
        stats['RAM'] = RAM(text)
        # If exists parse SWAP
        swap = SWAP(text)
        if swap:
            stats['SWAP'] = swap
        # If exists parse IRAM
        iram = IRAM(text)
        if iram:
            stats['IRAM'] = iram
        # Parse CPU status
        stats['CPU'] = CPUS(text)
        # Parse temperatures
        stats['TEMP'] = TEMPS(text)
        # Parse Watts
        stats['WATT'] = WATTS(text)
        return stats

    def _read_tegrastats(self, interval, running):
        pts = sp.Popen(
            [self.path, '--interval', str(interval)], stdout=sp.PIPE)
        try:
            # Reading loop
            while running.is_set():
                if pts.poll() is not None:
                    continue
                out = pts.stdout
                if out is not None:
                    # Read line process output
                    line = out.readline()
                    # Decode line in UTF-8
                    tegrastats_data = line.decode("utf-8")
                    # Decode and store
                    stats = self._decode(tegrastats_data)
                    # Launch callback
                    self.callback(stats)
        except AttributeError:
            pass
        except IOError:
            pass
        except Exception:
            # Write error message
            self._error = sys.exc_info()
        finally:
            # Kill process
            try:
                pts.kill()
            except OSError:
                pass

    def open(self, interval=0.5):
        if self._thread is not None:
            return False
        # Set timeout
        interval = int(interval * 1000)
        # Check if thread or process exist
        self._running.set()
        # Start thread Service client
        self._thread = Thread(target=self._read_tegrastats,
                              args=(
                                  interval,
                                  self._running,
                              ))
        self._thread.start()
        return True

    def close(self, timeout=None):
        # Catch exception if exist
        if self._error:
            # Extract exception and raise
            ex_type, ex_value, tb_str = self._error
            ex_value.__traceback__ = tb_str
            raise ex_value
        # Check if thread and process are already empty
        self._running.clear()
        if self._thread is not None:
            self._thread.join(timeout)
            self._thread = None
        return True
Beispiel #47
0
class Service(object):
    """Celery periodic task service."""

    scheduler_cls = PersistentScheduler

    def __init__(self,
                 app,
                 max_interval=None,
                 schedule_filename=None,
                 scheduler_cls=None):
        self.app = app
        self.max_interval = (max_interval or app.conf.beat_max_loop_interval)
        self.scheduler_cls = scheduler_cls or self.scheduler_cls
        self.schedule_filename = (schedule_filename
                                  or app.conf.beat_schedule_filename)

        self._is_shutdown = Event()
        self._is_stopped = Event()

    def __reduce__(self):
        return self.__class__, (self.max_interval, self.schedule_filename,
                                self.scheduler_cls, self.app)

    def start(self, embedded_process=False):
        info('beat: Starting...')
        debug('beat: Ticking with max interval->%s',
              humanize_seconds(self.scheduler.max_interval))

        signals.beat_init.send(sender=self)
        if embedded_process:
            signals.beat_embedded_init.send(sender=self)
            platforms.set_process_title('celery beat')

        try:
            while not self._is_shutdown.is_set():
                interval = self.scheduler.tick()
                if interval and interval > 0.0:
                    debug('beat: Waking up %s.',
                          humanize_seconds(interval, prefix='in '))
                    time.sleep(interval)
                    if self.scheduler.should_sync():
                        self.scheduler._do_sync()
        except (KeyboardInterrupt, SystemExit):
            self._is_shutdown.set()
        finally:
            self.sync()

    def sync(self):
        self.scheduler.close()
        self._is_stopped.set()

    def stop(self, wait=False):
        info('beat: Shutting down...')
        self._is_shutdown.set()
        wait and self._is_stopped.wait()  # block until shutdown done.

    def get_scheduler(self,
                      lazy=False,
                      extension_namespace='celery.beat_schedulers'):
        filename = self.schedule_filename
        aliases = dict(load_extension_class_names(extension_namespace) or {})
        return symbol_by_name(self.scheduler_cls, aliases=aliases)(
            app=self.app,
            schedule_filename=filename,
            max_interval=self.max_interval,
            lazy=lazy,
        )

    @cached_property
    def scheduler(self):
        return self.get_scheduler()
Beispiel #48
0
class Dispatcher(object):
    """This class dispatches all kinds of updates to its registered handlers.

    Attributes:
        bot (:class:`telegram.Bot`): The bot object that should be passed to the handlers.
        update_queue (:obj:`Queue`): The synchronized queue that will contain the updates.
        job_queue (:class:`telegram.ext.JobQueue`): Optional. The :class:`telegram.ext.JobQueue`
            instance to pass onto handler callbacks.
        workers (:obj:`int`): Number of maximum concurrent worker threads for the ``@run_async``
            decorator.
        user_data (:obj:`defaultdict`): A dictionary handlers can use to store data for the user.
        chat_data (:obj:`defaultdict`): A dictionary handlers can use to store data for the chat.
        persistence (:class:`telegram.ext.BasePersistence`): Optional. The persistence class to
            store data that should be persistent over restarts

    Args:
        bot (:class:`telegram.Bot`): The bot object that should be passed to the handlers.
        update_queue (:obj:`Queue`): The synchronized queue that will contain the updates.
        job_queue (:class:`telegram.ext.JobQueue`, optional): The :class:`telegram.ext.JobQueue`
                instance to pass onto handler callbacks.
        workers (:obj:`int`, optional): Number of maximum concurrent worker threads for the
            ``@run_async`` decorator. defaults to 4.
        persistence (:class:`telegram.ext.BasePersistence`, optional): The persistence class to
            store data that should be persistent over restarts
        use_context (:obj:`bool`, optional): If set to ``True`` Use the context based callback API.
            During the deprecation period of the old API the default is ``False``. **New users**:
            set this to ``True``.

    """

    __singleton_lock = Lock()
    __singleton_semaphore = BoundedSemaphore()
    __singleton = None
    logger = logging.getLogger(__name__)

    def __init__(self,
                 bot,
                 update_queue,
                 workers=4,
                 exception_event=None,
                 job_queue=None,
                 persistence=None,
                 use_context=False):
        self.bot = bot
        self.update_queue = update_queue
        self.job_queue = job_queue
        self.workers = workers
        self.use_context = use_context

        if not use_context:
            warnings.warn(
                'Old Handler API is deprecated - see https://git.io/fxJuV for details',
                TelegramDeprecationWarning,
                stacklevel=3)

        self.user_data = defaultdict(dict)
        """:obj:`dict`: A dictionary handlers can use to store data for the user."""
        self.chat_data = defaultdict(dict)
        if persistence:
            if not isinstance(persistence, BasePersistence):
                raise TypeError(
                    "persistence should be based on telegram.ext.BasePersistence"
                )
            self.persistence = persistence
            if self.persistence.store_user_data:
                self.user_data = self.persistence.get_user_data()
                if not isinstance(self.user_data, defaultdict):
                    raise ValueError("user_data must be of type defaultdict")
            if self.persistence.store_chat_data:
                self.chat_data = self.persistence.get_chat_data()
                if not isinstance(self.chat_data, defaultdict):
                    raise ValueError("chat_data must be of type defaultdict")
        else:
            self.persistence = None

        self.handlers = {}
        """Dict[:obj:`int`, List[:class:`telegram.ext.Handler`]]: Holds the handlers per group."""
        self.groups = []
        """List[:obj:`int`]: A list with all groups."""
        self.error_handlers = []
        """List[:obj:`callable`]: A list of errorHandlers."""

        self.running = False
        """:obj:`bool`: Indicates if this dispatcher is running."""
        self.__stop_event = Event()
        self.__exception_event = exception_event or Event()
        self.__async_queue = Queue()
        self.__async_threads = set()

        # For backward compatibility, we allow a "singleton" mode for the dispatcher. When there's
        # only one instance of Dispatcher, it will be possible to use the `run_async` decorator.
        with self.__singleton_lock:
            if self.__singleton_semaphore.acquire(blocking=0):
                self._set_singleton(self)
            else:
                self._set_singleton(None)

    @property
    def exception_event(self):
        return self.__exception_event

    def _init_async_threads(self, base_name, workers):
        base_name = '{}_'.format(base_name) if base_name else ''

        for i in range(workers):
            thread = Thread(target=self._pooled,
                            name='Bot:{}:worker:{}{}'.format(
                                self.bot.id, base_name, i))
            self.__async_threads.add(thread)
            thread.start()

    @classmethod
    def _set_singleton(cls, val):
        cls.logger.debug('Setting singleton dispatcher as %s', val)
        cls.__singleton = weakref.ref(val) if val else None

    @classmethod
    def get_instance(cls):
        """Get the singleton instance of this class.

        Returns:
            :class:`telegram.ext.Dispatcher`

        Raises:
            RuntimeError

        """
        if cls.__singleton is not None:
            return cls.__singleton()  # pylint: disable=not-callable
        else:
            raise RuntimeError(
                '{} not initialized or multiple instances exist'.format(
                    cls.__name__))

    def _pooled(self):
        thr_name = current_thread().getName()
        while 1:
            promise = self.__async_queue.get()

            # If unpacking fails, the thread pool is being closed from Updater._join_async_threads
            if not isinstance(promise, Promise):
                self.logger.debug("Closing run_async thread %s/%d", thr_name,
                                  len(self.__async_threads))
                break

            promise.run()
            if isinstance(promise.exception, DispatcherHandlerStop):
                self.logger.warning(
                    'DispatcherHandlerStop is not supported with async functions; func: %s',
                    promise.pooled_function.__name__)

    def run_async(self, func, *args, **kwargs):
        """Queue a function (with given args/kwargs) to be run asynchronously.

        Warning:
            If you're using @run_async you cannot rely on adding custom attributes to
            :class:`telegram.ext.CallbackContext`. See its docs for more info.

        Args:
            func (:obj:`callable`): The function to run in the thread.
            *args (:obj:`tuple`, optional): Arguments to `func`.
            **kwargs (:obj:`dict`, optional): Keyword arguments to `func`.

        Returns:
            Promise

        """
        # TODO: handle exception in async threads
        #       set a threading.Event to notify caller thread
        promise = Promise(func, args, kwargs)
        self.__async_queue.put(promise)
        return promise

    def start(self, ready=None):
        """Thread target of thread 'dispatcher'.

        Runs in background and processes the update queue.

        Args:
            ready (:obj:`threading.Event`, optional): If specified, the event will be set once the
                dispatcher is ready.

        """
        if self.running:
            self.logger.warning('already running')
            if ready is not None:
                ready.set()
            return

        if self.__exception_event.is_set():
            msg = 'reusing dispatcher after exception event is forbidden'
            self.logger.error(msg)
            raise TelegramError(msg)

        self._init_async_threads(uuid4(), self.workers)
        self.running = True
        self.logger.debug('Dispatcher started')

        if ready is not None:
            ready.set()

        while 1:
            try:
                # Pop update from update queue.
                update = self.update_queue.get(True, 1)
            except Empty:
                if self.__stop_event.is_set():
                    self.logger.debug('orderly stopping')
                    break
                elif self.__exception_event.is_set():
                    self.logger.critical(
                        'stopping due to exception in another thread')
                    break
                continue

            self.logger.debug('Processing Update: %s' % update)
            self.process_update(update)
            self.update_queue.task_done()

        self.running = False
        self.logger.debug('Dispatcher thread stopped')

    def stop(self):
        """Stops the thread."""
        if self.running:
            self.__stop_event.set()
            while self.running:
                sleep(0.1)
            self.__stop_event.clear()

        # async threads must be join()ed only after the dispatcher thread was joined,
        # otherwise we can still have new async threads dispatched
        threads = list(self.__async_threads)
        total = len(threads)

        # Stop all threads in the thread pool by put()ting one non-tuple per thread
        for i in range(total):
            self.__async_queue.put(None)

        for i, thr in enumerate(threads):
            self.logger.debug('Waiting for async thread {0}/{1} to end'.format(
                i + 1, total))
            thr.join()
            self.__async_threads.remove(thr)
            self.logger.debug('async thread {0}/{1} has ended'.format(
                i + 1, total))

    @property
    def has_running_threads(self):
        return self.running or bool(self.__async_threads)

    def process_update(self, update):
        """Processes a single update.

        Args:
            update (:obj:`str` | :class:`telegram.Update` | :class:`telegram.TelegramError`):
                The update to process.

        """
        def persist_update(update):
            """Persist a single update.

            Args:
            update (:class:`telegram.Update`):
                The update to process.

            """
            if self.persistence and isinstance(update, Update):
                if self.persistence.store_chat_data and update.effective_chat:
                    chat_id = update.effective_chat.id
                    try:
                        self.persistence.update_chat_data(
                            chat_id, self.chat_data[chat_id])
                    except Exception as e:
                        try:
                            self.dispatch_error(update, e)
                        except Exception:
                            message = 'Saving chat data raised an error and an ' \
                                      'uncaught error was raised while handling ' \
                                      'the error with an error_handler'
                            self.logger.exception(message)
                if self.persistence.store_user_data and update.effective_user:
                    user_id = update.effective_user.id
                    try:
                        self.persistence.update_user_data(
                            user_id, self.user_data[user_id])
                    except Exception as e:
                        try:
                            self.dispatch_error(update, e)
                        except Exception:
                            message = 'Saving user data raised an error and an ' \
                                      'uncaught error was raised while handling ' \
                                      'the error with an error_handler'
                            self.logger.exception(message)

        # An error happened while polling
        if isinstance(update, TelegramError):
            try:
                self.dispatch_error(None, update)
            except Exception:
                self.logger.exception(
                    'An uncaught error was raised while handling the error')
            return

        context = None

        for group in self.groups:
            try:
                for handler in self.handlers[group]:
                    check = handler.check_update(update)
                    if check is not None and check is not False:
                        if not context and self.use_context:
                            context = CallbackContext.from_update(update, self)
                        handler.handle_update(update, self, check, context)
                        persist_update(update)
                        break

            # Stop processing with any other handler.
            except DispatcherHandlerStop:
                self.logger.debug(
                    'Stopping further handlers due to DispatcherHandlerStop')
                persist_update(update)
                break

            # Dispatch any error.
            except Exception as e:
                try:
                    self.dispatch_error(update, e)
                except DispatcherHandlerStop:
                    self.logger.debug('Error handler stopped further handlers')
                    break
                # Errors should not stop the thread.
                except Exception:
                    self.logger.exception(
                        'An error was raised while processing the update and an '
                        'uncaught error was raised while handling the error '
                        'with an error_handler')

    def add_handler(self, handler, group=DEFAULT_GROUP):
        """Register a handler.

        TL;DR: Order and priority counts. 0 or 1 handlers per group will be used.

        A handler must be an instance of a subclass of :class:`telegram.ext.Handler`. All handlers
        are organized in groups with a numeric value. The default group is 0. All groups will be
        evaluated for handling an update, but only 0 or 1 handler per group will be used. If
        :class:`telegram.ext.DispatcherHandlerStop` is raised from one of the handlers, no further
        handlers (regardless of the group) will be called.

        The priority/order of handlers is determined as follows:

          * Priority of the group (lower group number == higher priority)
          * The first handler in a group which should handle an update (see
            :attr:`telegram.ext.Handler.check_update`) will be used. Other handlers from the
            group will not be used. The order in which handlers were added to the group defines the
            priority.

        Args:
            handler (:class:`telegram.ext.Handler`): A Handler instance.
            group (:obj:`int`, optional): The group identifier. Default is 0.

        """
        # Unfortunately due to circular imports this has to be here
        from .conversationhandler import ConversationHandler

        if not isinstance(handler, Handler):
            raise TypeError('handler is not an instance of {0}'.format(
                Handler.__name__))
        if not isinstance(group, int):
            raise TypeError('group is not int')
        if isinstance(handler, ConversationHandler) and handler.persistent:
            if not self.persistence:
                raise ValueError(
                    "Conversationhandler {} can not be persistent if dispatcher has no "
                    "persistence".format(handler.name))
            handler.conversations = self.persistence.get_conversations(
                handler.name)
            handler.persistence = self.persistence

        if group not in self.handlers:
            self.handlers[group] = list()
            self.groups.append(group)
            self.groups = sorted(self.groups)

        self.handlers[group].append(handler)

    def remove_handler(self, handler, group=DEFAULT_GROUP):
        """Remove a handler from the specified group.

        Args:
            handler (:class:`telegram.ext.Handler`): A Handler instance.
            group (:obj:`object`, optional): The group identifier. Default is 0.

        """
        if handler in self.handlers[group]:
            self.handlers[group].remove(handler)
            if not self.handlers[group]:
                del self.handlers[group]
                self.groups.remove(group)

    def update_persistence(self):
        """Update :attr:`user_data` and :attr:`chat_data` in :attr:`persistence`.
        """
        if self.persistence:
            if self.persistence.store_chat_data:
                for chat_id in self.chat_data:
                    self.persistence.update_chat_data(chat_id,
                                                      self.chat_data[chat_id])
            if self.persistence.store_user_data:
                for user_id in self.user_data:
                    self.persistence.update_user_data(user_id,
                                                      self.user_data[user_id])

    def add_error_handler(self, callback):
        """Registers an error handler in the Dispatcher. This handler will receive every error
        which happens in your bot.

        Warning: The errors handled within these handlers won't show up in the logger, so you
        need to make sure that you reraise the error.

        Args:
            callback (:obj:`callable`): The callback function for this error handler. Will be
                called when an error is raised. Callback signature for context based API:

                ``def callback(update: Update, context: CallbackContext)``

                The error that happened will be present in context.error.

        Note:
            See https://git.io/fxJuV for more info about switching to context based API.
        """
        self.error_handlers.append(callback)

    def remove_error_handler(self, callback):
        """Removes an error handler.

        Args:
            callback (:obj:`callable`): The error handler to remove.

        """
        if callback in self.error_handlers:
            self.error_handlers.remove(callback)

    def dispatch_error(self, update, error):
        """Dispatches an error.

        Args:
            update (:obj:`str` | :class:`telegram.Update` | None): The update that caused the error
            error (:obj:`Exception`): The error that was raised.

        """
        if self.error_handlers:
            for callback in self.error_handlers:
                if self.use_context:
                    callback(update,
                             CallbackContext.from_error(update, error, self))
                else:
                    callback(self.bot, update, error)

        else:
            self.logger.exception(
                'No error handlers are registered, logging exception.',
                exc_info=error)
Beispiel #49
0
class Reporter(InterfaceBase, AbstractContextManager, SetupUploadMixin,
               AsyncManagerMixin):
    """
    A simple metrics reporter class.
    This class caches reports and supports both a explicit flushing and context-based flushing. To ensure reports are
     sent to the backend, please use (assuming an instance of Reporter named 'reporter'):
     - use the context manager feature (which will automatically flush when exiting the context):
        with reporter:
            reporter.report...
            ...
     - explicitly call flush:
        reporter.report...
        ...
        reporter.flush()
    """
    def __init__(self, metrics, flush_threshold=10, async_enable=False):
        """
        Create a reporter
        :param metrics: A Metrics manager instance that handles actual reporting, uploads etc.
        :type metrics: .backend_interface.metrics.Metrics
        :param flush_threshold: Events flush threshold. This determines the threshold over which cached reported events
            are flushed and sent to the backend.
        :type flush_threshold: int
        """
        log = metrics.log.getChild('reporter')
        log.setLevel(log.level)
        super(Reporter, self).__init__(session=metrics.session, log=log)
        self._metrics = metrics
        self._flush_threshold = flush_threshold
        self._events = []
        self._bucket_config = None
        self._storage_uri = None
        self._async_enable = async_enable
        self._flush_frequency = 30.0
        self._exit_flag = False
        self._flush_event = Event()
        self._flush_event.clear()
        self._thread = Thread(target=self._daemon)
        self._thread.daemon = True
        self._thread.start()
        self._max_iteration = 0

    def _set_storage_uri(self, value):
        value = '/'.join(x for x in (value.rstrip('/'),
                                     self._metrics.storage_key_prefix) if x)
        self._storage_uri = value

    storage_uri = property(None, _set_storage_uri)

    @property
    def flush_threshold(self):
        return self._flush_threshold

    @flush_threshold.setter
    def flush_threshold(self, value):
        self._flush_threshold = max(0, value)

    @property
    def async_enable(self):
        return self._async_enable

    @async_enable.setter
    def async_enable(self, value):
        self._async_enable = bool(value)

    @property
    def max_iteration(self):
        return self._max_iteration

    def _daemon(self):
        while not self._exit_flag:
            self._flush_event.wait(self._flush_frequency)
            self._flush_event.clear()
            self._write()
            # wait for all reports
            if self.get_num_results() > 0:
                self.wait_for_results()
        # make sure we flushed everything
        self._async_enable = False
        self._write()
        if self.get_num_results() > 0:
            self.wait_for_results()

    def _report(self, ev):
        ev_iteration = ev.get_iteration()
        if ev_iteration is not None:
            # we have to manually add get_iteration_offset() because event hasn't reached the Metric manager
            self._max_iteration = max(
                self._max_iteration,
                ev_iteration + self._metrics.get_iteration_offset())
        self._events.append(ev)
        if len(self._events) >= self._flush_threshold:
            self.flush()

    def _write(self):
        if not self._events:
            return
        # print('reporting %d events' % len(self._events))
        res = self._metrics.write_events(self._events,
                                         async_enable=self._async_enable,
                                         storage_uri=self._storage_uri)
        if self._async_enable:
            self._add_async_result(res)
        self._events = []

    def flush(self):
        """
        Flush cached reports to backend.
        """
        self._flush_event.set()

    def stop(self):
        self._exit_flag = True
        self._flush_event.set()
        self._thread.join()

    def report_scalar(self, title, series, value, iter):
        """
        Report a scalar value
        :param title: Title (AKA metric)
        :type title: str
        :param series: Series (AKA variant)
        :type series: str
        :param value: Reported value
        :type value: float
        :param iter: Iteration number
        :type value: int
        """
        ev = ScalarEvent(metric=self._normalize_name(title),
                         variant=self._normalize_name(series),
                         value=value,
                         iter=iter)
        self._report(ev)

    def report_vector(self, title, series, values, iter):
        """
        Report a vector of values
        :param title: Title (AKA metric)
        :type title: str
        :param series: Series (AKA variant)
        :type series: str
        :param values: Reported values
        :type value: [float]
        :param iter: Iteration number
        :type value: int
        """
        if not isinstance(values, Iterable):
            raise ValueError('values: expected an iterable')
        ev = VectorEvent(metric=self._normalize_name(title),
                         variant=self._normalize_name(series),
                         values=values,
                         iter=iter)
        self._report(ev)

    def report_plot(self, title, series, plot, iter):
        """
        Report a Plotly chart
        :param title: Title (AKA metric)
        :type title: str
        :param series: Series (AKA variant)
        :type series: str
        :param plot: A JSON describing a plotly chart (see https://help.plot.ly/json-chart-schema/)
        :type plot: str or dict
        :param iter: Iteration number
        :type value: int
        """
        try:

            def default(o):
                if isinstance(o, np.int64):
                    return int(o)
        except Exception:
            default = None

        if isinstance(plot, dict):
            plot = json.dumps(plot, default=default)
        elif not isinstance(plot, six.string_types):
            raise ValueError('Plot should be a string or a dict')
        ev = PlotEvent(metric=self._normalize_name(title),
                       variant=self._normalize_name(series),
                       plot_str=plot,
                       iter=iter)
        self._report(ev)

    def report_image(self, title, series, src, iter):
        """
        Report an image.
        :param title: Title (AKA metric)
        :type title: str
        :param series: Series (AKA variant)
        :type series: str
        :param src: Image source URI. This URI will be used by the webapp and workers when trying to obtain the image
            for presentation of processing. Currently only http(s), file and s3 schemes are supported.
        :type src: str
        :param iter: Iteration number
        :type value: int
        """
        ev = ImageEventNoUpload(metric=self._normalize_name(title),
                                variant=self._normalize_name(series),
                                iter=iter,
                                src=src)
        self._report(ev)

    def report_image_and_upload(self,
                                title,
                                series,
                                iter,
                                path=None,
                                image=None,
                                upload_uri=None,
                                max_image_history=None,
                                delete_after_upload=False):
        """
        Report an image and upload its contents. Image is uploaded to a preconfigured bucket (see setup_upload()) with
         a key (filename) describing the task ID, title, series and iteration.
        :param title: Title (AKA metric)
        :type title: str
        :param series: Series (AKA variant)
        :type series: str
        :param iter: Iteration number
        :type iter: int
        :param path: A path to an image file. Required unless matrix is provided.
        :type path: str
        :param image: Image data. Required unless filename is provided.
        :type image: A PIL.Image.Image object or a 3D numpy.ndarray object
        :param max_image_history: maximum number of image to store per metric/variant combination
        use negative value for unlimited. default is set in global configuration (default=5)
        :param delete_after_upload: if True, one the file was uploaded the local copy will be deleted
        :type delete_after_upload: boolean
        """
        if not self._storage_uri and not upload_uri:
            raise ValueError(
                'Upload configuration is required (use setup_upload())')
        if len([x for x in (path, image) if x is not None]) != 1:
            raise ValueError('Expected only one of [filename, image]')
        kwargs = dict(metric=self._normalize_name(title),
                      variant=self._normalize_name(series),
                      iter=iter,
                      image_file_history_size=max_image_history)
        ev = ImageEvent(image_data=image,
                        upload_uri=upload_uri,
                        local_image_path=path,
                        delete_after_upload=delete_after_upload,
                        **kwargs)
        self._report(ev)

    def report_histogram(self,
                         title,
                         series,
                         histogram,
                         iter,
                         labels=None,
                         xlabels=None,
                         xtitle=None,
                         ytitle=None,
                         comment=None):
        """
        Report an histogram bar plot
        :param title: Title (AKA metric)
        :type title: str
        :param series: Series (AKA variant)
        :type series: str
        :param histogram: The histogram data.
            A row for each dataset(bar in a bar group). A column for each bucket.
        :type histogram: numpy array
        :param iter: Iteration number
        :type value: int
        :param labels: The labels for each bar group.
        :type labels: list of strings.
        :param xlabels: The labels of the x axis.
        :type xlabels: List of strings.
        :param str xtitle: optional x-axis title
        :param str ytitle: optional y-axis title
        :param comment: comment underneath the title
        :type comment: str
        """
        plotly_dict = create_2d_histogram_plot(
            np_row_wise=histogram,
            title=title,
            xtitle=xtitle,
            ytitle=ytitle,
            labels=labels,
            series=series,
            xlabels=xlabels,
            comment=comment,
        )

        return self.report_plot(
            title=self._normalize_name(title),
            series=self._normalize_name(series),
            plot=plotly_dict,
            iter=iter,
        )

    def report_table(self, title, series, table, iteration):
        """
        Report a table plot.

        :param title: Title (AKA metric)
        :type title: str
        :param series: Series (AKA variant)
        :type series: str
        :param table: The table data
        :type table: pandas.DataFrame
        :param iteration: Iteration number
        :type iteration: int
        """
        table_output = create_plotly_table(table, title, series)
        return self.report_plot(
            title=self._normalize_name(title),
            series=self._normalize_name(series),
            plot=table_output,
            iter=iteration,
        )

    def report_line_plot(self,
                         title,
                         series,
                         iter,
                         xtitle,
                         ytitle,
                         mode='lines',
                         reverse_xaxis=False,
                         comment=None):
        """
        Report a (possibly multiple) line plot.

        :param title: Title (AKA metric)
        :type title: str
        :param series: All the series' data, one for each line in the plot.
        :type series: An iterable of LineSeriesInfo.
        :param iter: Iteration number
        :type iter: int
        :param xtitle: x-axis title
        :type xtitle: str
        :param ytitle: y-axis title
        :type ytitle: str
        :param mode: 'lines' / 'markers' / 'lines+markers'
        :type mode: str
        :param reverse_xaxis: If true X axis will be displayed from high to low (reversed)
        :type reverse_xaxis: bool
        :param comment: comment underneath the title
        :type comment: str
        """

        plotly_dict = create_line_plot(
            title=title,
            series=series,
            xtitle=xtitle,
            ytitle=ytitle,
            mode=mode,
            reverse_xaxis=reverse_xaxis,
            comment=comment,
        )

        return self.report_plot(
            title=self._normalize_name(title),
            series='',
            plot=plotly_dict,
            iter=iter,
        )

    def report_2d_scatter(self,
                          title,
                          series,
                          data,
                          iter,
                          mode='lines',
                          xtitle=None,
                          ytitle=None,
                          labels=None,
                          comment=None):
        """
        Report a 2d scatter graph (with lines)

        :param title: Title (AKA metric)
        :type title: str
        :param series: Series (AKA variant)
        :type series: str
        :param data: A scattered data: pairs of x,y as rows in a numpy array
        :type scatter: ndarray
        :param iter: Iteration number
        :type iter: int
        :param mode: (type str) 'lines'/'markers'/'lines+markers'
        :param xtitle: optional x-axis title
        :param ytitle: optional y-axis title
        :param labels: label (text) per point in the scatter (in the same order)
        :param comment: comment underneath the title
        :type comment: str
        """
        plotly_dict = create_2d_scatter_series(
            np_row_wise=data,
            title=title,
            series_name=series,
            mode=mode,
            xtitle=xtitle,
            ytitle=ytitle,
            labels=labels,
            comment=comment,
        )

        return self.report_plot(
            title=self._normalize_name(title),
            series=self._normalize_name(series),
            plot=plotly_dict,
            iter=iter,
        )

    def report_3d_scatter(self,
                          title,
                          series,
                          data,
                          iter,
                          labels=None,
                          mode='lines',
                          color=((217, 217, 217, 0.14), ),
                          marker_size=5,
                          line_width=0.8,
                          xtitle=None,
                          ytitle=None,
                          ztitle=None,
                          fill=None,
                          comment=None):
        """
        Report a 3d scatter graph (with markers)

        :param title: Title (AKA metric)
        :type title: str
        :param series: Series (AKA variant)
        :type series: str
        :param data: A scattered data: pairs of x,y,z as rows in a numpy array. or list of numpy arrays
        :type data: ndarray.
        :param iter: Iteration number
        :type iter: int
        :param labels: label (text) per point in the scatter (in the same order)
        :type labels: str
        :param mode: (type str) 'lines'/'markers'/'lines+markers'
        :param color: list of RGBA colors [(217, 217, 217, 0.14),]
        :param marker_size: marker size in px
        :param line_width: line width in px
        :param xtitle: optional x-axis title
        :param ytitle: optional y-axis title
        :param ztitle: optional z-axis title
        :param comment: comment underneath the title
        """
        data_series = data if isinstance(data, list) else [data]

        def get_labels(i):
            if labels and isinstance(labels, list):
                try:
                    item = labels[i]
                except IndexError:
                    item = labels[-1]
                if isinstance(item, list):
                    return item
            return labels

        plotly_obj = plotly_scatter3d_layout_dict(
            title=title,
            xaxis_title=xtitle,
            yaxis_title=ytitle,
            zaxis_title=ztitle,
            comment=comment,
        )

        for i, values in enumerate(data_series):
            plotly_obj = create_3d_scatter_series(
                np_row_wise=values,
                title=title,
                series_name=series[i] if isinstance(series, list) else None,
                labels=get_labels(i),
                plotly_obj=plotly_obj,
                mode=mode,
                line_width=line_width,
                marker_size=marker_size,
                color=color,
                fill_axis=fill,
            )

        return self.report_plot(
            title=self._normalize_name(title),
            series=self._normalize_name(series)
            if not isinstance(series, list) else None,
            plot=plotly_obj,
            iter=iter,
        )

    def report_value_matrix(self,
                            title,
                            series,
                            data,
                            iter,
                            xtitle=None,
                            ytitle=None,
                            xlabels=None,
                            ylabels=None,
                            comment=None):
        """
        Report a heat-map matrix

        :param title: Title (AKA metric)
        :type title: str
        :param series: Series (AKA variant)
        :type series: str
        :param data: A heat-map matrix (example: confusion matrix)
        :type data: ndarray
        :param iter: Iteration number
        :type iter: int
        :param str xtitle: optional x-axis title
        :param str ytitle: optional y-axis title
        :param xlabels: optional label per column of the matrix
        :param ylabels: optional label per row of the matrix
        :param comment: comment underneath the title
        """

        plotly_dict = create_value_matrix(
            np_value_matrix=data,
            title=title,
            xlabels=xlabels,
            ylabels=ylabels,
            series=series,
            comment=comment,
            xtitle=xtitle,
            ytitle=ytitle,
        )

        return self.report_plot(
            title=self._normalize_name(title),
            series=self._normalize_name(series),
            plot=plotly_dict,
            iter=iter,
        )

    def report_value_surface(self,
                             title,
                             series,
                             data,
                             iter,
                             xlabels=None,
                             ylabels=None,
                             xtitle=None,
                             ytitle=None,
                             ztitle=None,
                             camera=None,
                             comment=None):
        """
        Report a 3d surface (same data as heat-map matrix, only presented differently)

        :param title: Title (AKA metric)
        :type title: str
        :param series: Series (AKA variant)
        :type series: str
        :param data: A heat-map matrix (example: confusion matrix)
        :type data: ndarray
        :param iter: Iteration number
        :type iter: int
        :param xlabels: optional label per column of the matrix
        :param ylabels: optional label per row of the matrix
        :param xtitle: optional x-axis title
        :param ytitle: optional y-axis title
        :param ztitle: optional z-axis title
        :param camera: X,Y,Z camera position. def: (1,1,1)
        :param comment: comment underneath the title
        """

        plotly_dict = create_3d_surface(
            np_value_matrix=data,
            title=title + '/' + series,
            xlabels=xlabels,
            ylabels=ylabels,
            series=series,
            xtitle=xtitle,
            ytitle=ytitle,
            ztitle=ztitle,
            camera=camera,
            comment=comment,
        )

        return self.report_plot(
            title=self._normalize_name(title),
            series=self._normalize_name(series),
            plot=plotly_dict,
            iter=iter,
        )

    def report_image_plot_and_upload(self,
                                     title,
                                     series,
                                     iter,
                                     path=None,
                                     matrix=None,
                                     upload_uri=None,
                                     max_image_history=None,
                                     delete_after_upload=False):
        """
        Report an image as plot and upload its contents.
        Image is uploaded to a preconfigured bucket (see setup_upload()) with a key (filename)
        describing the task ID, title, series and iteration.
        Then a plotly object is created and registered, this plotly objects points to the uploaded image
        :param title: Title (AKA metric)
        :type title: str
        :param series: Series (AKA variant)
        :type series: str
        :param iter: Iteration number
        :type value: int
        :param path: A path to an image file. Required unless matrix is provided.
        :type path: str
        :param matrix: A 3D numpy.ndarray object containing image data (RGB). Required unless filename is provided.
        :type matrix: str
        :param max_image_history: maximum number of image to store per metric/variant combination
        use negative value for unlimited. default is set in global configuration (default=5)
        :param delete_after_upload: if True, one the file was uploaded the local copy will be deleted
        :type delete_after_upload: boolean
        """
        if not upload_uri and not self._storage_uri:
            raise ValueError(
                'Upload configuration is required (use setup_upload())')
        if len([x for x in (path, matrix) if x is not None]) != 1:
            raise ValueError('Expected only one of [filename, matrix]')
        kwargs = dict(metric=self._normalize_name(title),
                      variant=self._normalize_name(series),
                      iter=iter,
                      image_file_history_size=max_image_history)
        ev = UploadEvent(image_data=matrix,
                         upload_uri=upload_uri,
                         local_image_path=path,
                         delete_after_upload=delete_after_upload,
                         **kwargs)
        _, url = ev.get_target_full_upload_uri(
            upload_uri or self._storage_uri, self._metrics.storage_key_prefix)

        # Hack: if the url doesn't start with http/s then the plotly will not be able to show it,
        # then we put the link under images not plots
        if not url.startswith('http'):
            return self.report_image_and_upload(
                title=title,
                series=series,
                iter=iter,
                path=path,
                image=matrix,
                upload_uri=upload_uri,
                max_image_history=max_image_history)

        self._report(ev)
        plotly_dict = create_image_plot(
            image_src=url,
            title=title + '/' + series,
            width=matrix.shape[1] if matrix is not None else 640,
            height=matrix.shape[0] if matrix is not None else 480,
        )

        return self.report_plot(
            title=self._normalize_name(title),
            series=self._normalize_name(series),
            plot=plotly_dict,
            iter=iter,
        )

    @classmethod
    def _normalize_name(cls, name):
        return name

    def __exit__(self, exc_type, exc_val, exc_tb):
        # don't flush in case an exception was raised
        if not exc_type:
            self.flush()
class virtualterminal:
    __stop = False
    __evt_stopping = Event()

    def __init__(self, manager):
        self.__evt_stopping = Event()
        self.__manager = manager

    """ Starts the worker's threads
    """

    def start(self):
        self.__print_welcome()
        self.__spawn_shell_process()
        return self.__evt_stopping

    """ Stops the worker's threads
    """

    def stop(self):
        self.__write('Terminating session...\r\n')
        self.__stop = True

    """ The body of this worker
    """

    def __run(self, process, pin):
        parent = psutil.Process(process.pid)
        sock_reader = self.__manager.get_io().reader
        msg = ''
        errmsg = ''
        while not self.__stop:
            try:

                # read from the socket, terminal stdin, or terminal stdout, whichever produces data first
                rs, ws, es = select(
                    [sock_reader, process.stdout, process.stderr], [], [], 1)
                for r in rs:
                    if r is sock_reader:  # data arrived from socket
                        # read the data
                        c = r.read(1)
                        if len(c) == 0:
                            # a data-length of 0 means that the socket failed to read, must be closed
                            # exit the handler thread
                            self.stop()
                            break
                        elif ord(c) == 4:  # ord(c) of 4 means CTRL^D
                            #     # a data-length of 0 means that the socket failed to read, must be closed
                            #     # exit the handler thread
                            logger.info(
                                'Received CTRL^D, terminating terminal...')
                            self.__write('\r\n')
                            self.stop()
                            break
                        else:
                            children = parent.children(recursive=True)
                            if len(children) == 0:  # echo back to client
                                self.__write(c.replace('\n', '\r\n'))

                            pin.write(c)
                            sys.stdout.flush()

                            children = parent.children(recursive=True)
                            self.__manager.get_io().echo(len(children) == 0)

                            has_data = select([process.stdout, process.stderr],
                                              [], [], 0)[0] != []
                            if c == '\n' and not has_data and len(
                                    children) == 0:
                                self.__write(self.__generate_promptstring())

                    elif r in [process.stdout,
                               process.stderr]:  # data arrived from terminal
                        max = 100

                        def readAllSoFar(stream, max, retVal=''):
                            while (select([stream], [], [], 0)[0] != []
                                   and len(retVal) <= max):
                                c = stream.read(1)
                                if len(c) == 0:
                                    break
                                else:
                                    retVal += c
                            return retVal

                        proc_response = readAllSoFar(r, max)

                        if len(proc_response) == 0:
                            self.stop()
                            break
                        #print 'read bytes!'
                        children = parent.children(recursive=False)
                        #print 'has children ' + `len(children)`
                        proc_response = proc_response.replace('\n', '\r\n')
                        #print proc_response
                        #print 'returning terminal response [%s]' % proc_response

                        self.__write('%s' %
                                     proc_response)  #send only takes string

                        if len(proc_response) < max and len(children) == 0:
                            self.__write(self.__generate_promptstring())

                        time.sleep(
                            0.005
                        )  # Hack currently to fix unknown race condition in CYW library preventing fast output
                        # sys.stderr.flush()
                    # elif r is process.stderr: # error arrived from terminal
                    #     print 'reading from error!'
                    #     errmsg += process.stderr.read(1)
                    #     if len(errmsg) == 0:
                    #         logger.info('Received CTRL^D, terminating terminal...')
                    #         self.stop()
                    #         break
                    #     if errmsg.endswith('>>> '):
                    #         errmsg = errmsg[:-4]
                    #     if errmsg.endswith('\n'):
                    #         self.__write('%s' % errmsg)
                    #         errmsg = ''
            except Exception as e:
                self.__write('\r\nINTERNAL SERVER ERROR: %s\r\n' %
                             e)  #send only takes string
                logger.error(traceback.format_exc())

        logger.info('Virtual Terminal has exited.')
        process.stdout.close()
        process.stderr.close()
        process.terminate()
        self.__evt_stopping.set()

    """ Spawns the Linux `sh` process, captures its pipes, and creates a thread for routing I/O
    """

    def __spawn_shell_process(self):
        cwd = '/bin/bash --norc'
        pw_record = pwd.getpwnam(self.__manager.get_username())
        user_name = pw_record.pw_name
        user_home_dir = pw_record.pw_dir
        user_uid = pw_record.pw_uid
        user_gid = pw_record.pw_gid
        env = os.environ.copy()
        env['HOME'] = user_home_dir
        env['LOGNAME'] = user_name
        env['PWD'] = cwd
        env['USER'] = user_name

        logger.debug('Spawning shell process for user %s' % pw_record.pw_name)
        process = Popen('/bin/sh',
                        preexec_fn=self.__demote(user_uid, user_gid),
                        cwd=user_home_dir,
                        env=env,
                        stdin=PIPE,
                        stdout=PIPE,
                        stderr=PIPE)

        # grab a file descriptor for the virtual terminal, use this to send data to your virtual terminal
        self.pin = process.stdin

        logger.debug('Sending prompt string for user %s' % pw_record.pw_name)
        self.__write(self.__generate_promptstring())

        start_new_thread(self.__run, (process, self.pin))

    """ Writes output to the client transport
    """

    def __write(self, text):
        self.__manager.get_io().write(text)

    # def write(self, message):
    #     self.__write(self.__generate_promptstring())
    #     self.pin.write(msg+'\r\n')
    """ Constructs a Linux-bash style command prompt
    """

    def __generate_promptstring(self):
        workdir = os.getcwd()
        if workdir == os.getenv('HOME'):
            workdir = "~"
        return '%s@%s:%s# ' % (self.__manager.get_username(), platform.node(),
                               workdir)

    """ Writes a welcome message to the client transport
    """

    def __print_welcome(self):
        if os.path.isfile(WELCOME_MESSAGE_PATH):
            logger.info('Sending welcome message from file: %s' %
                        WELCOME_MESSAGE_PATH)
            self.__write('\n')
            with open(WELCOME_MESSAGE_PATH, "r") as welcome_file:
                for line in welcome_file:
                    self.__write(line.strip('\n') + '\r\n')
            self.__write('\r\n\r\n')

    """ Demote child process to target user and group.
        This ensures Linux fs permissions will be honored.
    """

    def __demote(self, user_uid, user_gid):
        def result():
            os.setgid(user_gid)
            os.setuid(user_uid)

        return result
Beispiel #51
0
class _WorkerThread(Thread):
    """WorkerThreads process incoming messages off of the work queue
    on a loop.  By themselves, they don't do any sort of network IO.

    Parameters:
      broker(Broker)
      consumers(dict[str, _ConsumerThread])
      work_queue(Queue)
      worker_timeout(int)
    """
    def __init__(self, *, broker, consumers, work_queue, worker_timeout):
        super().__init__(daemon=True)

        self.logger = get_logger(__name__, "WorkerThread")
        self.running = False
        self.paused = False
        self.paused_event = Event()
        self.broker = broker
        self.consumers = consumers
        self.work_queue = work_queue
        self.timeout = worker_timeout / 1000

    def run(self):
        self.logger.debug("Running worker thread...")
        self.running = True
        while self.running:
            if self.paused:
                self.logger.debug("Worker is paused. Sleeping for %.02f...",
                                  self.timeout)
                self.paused_event.set()
                time.sleep(self.timeout)
                continue

            try:
                _, message = self.work_queue.get(timeout=self.timeout)
                self.process_message(message)
            except Empty:
                continue

        self.broker.emit_before("worker_thread_shutdown", self)
        self.logger.debug("Worker thread stopped.")

    def process_message(self, message):
        """Process a message pulled off of the work queue then push it
        back to its associated consumer for post processing.

        Parameters:
          message(MessageProxy)
        """
        try:
            self.logger.debug("Received message %s with id %r.", message,
                              message.message_id)
            self.broker.emit_before("process_message", message)

            res = None
            if not message.failed:
                res = self.call_actor(message)

            self.broker.emit_after("process_message", message, result=res)

        except SkipMessage:
            self.logger.warning("Message %s was skipped.", message)
            self.broker.emit_after("skip_message", message)

        except MessageCanceled:
            self.logger.warning("Message %s has been canceled", message)
            self.broker.emit_after("message_canceled", message)

        except BaseException as e:
            if isinstance(e, RateLimitExceeded):
                self.logger.warning("Rate limit exceeded in message %s: %s.",
                                    message, e)
            else:
                self.logger.error(
                    "Failed to process message %s with unhandled exception.",
                    message,
                    exc_info=True,
                    extra={
                        "input": {
                            "args": str(message.args),
                            "kwargs": str(message.kwargs)
                        }
                    },
                )

            self.broker.emit_after("process_message", message, exception=e)

        finally:
            # NOTE: There is no race here as any message that was
            # processed must have come off of a consumer.  Therefore,
            # there has to be a consumer for that message's queue so
            # this is safe.  Probably.
            self.consumers[message.queue_name].post_process_message(message)
            self.work_queue.task_done()

    def call_actor(self, message):
        """Call an actor with the arguments stored in the message

        Handle the logging of the start/end of the actor with runtime measure

        Parameters:
          message(MessageProxy)

        Returns:
          Whatever the actor returns.
        """
        actor = self.broker.get_actor(message.actor_name)
        message_id = message.message_id
        try:
            self.logger.info("Started Actor %s",
                             message,
                             extra={"message_id": message_id})
            start = time.perf_counter()
            return actor(*message.args, **message.kwargs)
        finally:
            runtime = (time.perf_counter() - start) * 1000
            extra = {"message_id": message_id, "runtime": runtime}
            self.logger.info("Finished Actor %s after %.02fms.",
                             message,
                             runtime,
                             extra=extra)

    def pause(self):
        """Pause this worker.
        """
        self.paused = True
        self.paused_event.clear()

    def resume(self):
        """Resume this worker.
        """
        self.paused = False
        self.paused_event.clear()

    def stop(self):
        """Initiate the WorkerThread shutdown process.

        Code calling this method should then join on the thread and
        wait for it to finish shutting down.
        """
        self.logger.debug("Stopping worker thread...")
        self.running = False
Beispiel #52
0
NUMBER_OF_FINGERS = 3
NUMBER_OF_JOINTS = 3

initial_angle_ = 0

listeMoteur1_ = [initial_angle_]
listeMoteur2_ = [initial_angle_]
listeMoteur3_ = [initial_angle_]

instructionListe_ = []
command_ = 1
execute = ""

# ====== Concurrency variable and thread =======
routine_event_cv_ = Event()
routine_event_cv_.set()
send_lock_ = RLock()
routine_cv_waiting_for_unlock_ = False
ready_to_send_bloc_ = False

is_play_thread_alive_ = False
stop_flag_lock_ = RLock()
stop_flag_ = False

threads_on_ = True


# Record thread
class Record:

    lock = RLock()
Beispiel #53
0
class Module(MgrModule):
    OPTIONS = [
        {
            'name': 'active'
        },
        {
            'name': 'begin_time'
        },
        {
            'name': 'crush_compat_max_iterations'
        },
        {
            'name': 'crush_compat_metrics'
        },
        {
            'name': 'crush_compat_step'
        },
        {
            'name': 'end_time'
        },
        {
            'name': 'min_score'
        },
        {
            'name': 'mode'
        },
        {
            'name': 'sleep_interval'
        },
        {
            'name': 'upmap_max_iterations'
        },
        {
            'name': 'upmap_max_deviation'
        },
    ]

    COMMANDS = [
        {
            "cmd": "balancer status",
            "desc": "Show balancer status",
            "perm": "r",
        },
        {
            "cmd":
            "balancer mode name=mode,type=CephChoices,strings=none|crush-compat|upmap",
            "desc": "Set balancer mode",
            "perm": "rw",
        },
        {
            "cmd": "balancer on",
            "desc": "Enable automatic balancing",
            "perm": "rw",
        },
        {
            "cmd": "balancer off",
            "desc": "Disable automatic balancing",
            "perm": "rw",
        },
        {
            "cmd": "balancer eval name=option,type=CephString,req=false",
            "desc":
            "Evaluate data distribution for the current cluster or specific pool or specific plan",
            "perm": "r",
        },
        {
            "cmd":
            "balancer eval-verbose name=option,type=CephString,req=false",
            "desc":
            "Evaluate data distribution for the current cluster or specific pool or specific plan (verbosely)",
            "perm": "r",
        },
        {
            "cmd":
            "balancer optimize name=plan,type=CephString name=pools,type=CephString,n=N,req=false",
            "desc": "Run optimizer to create a new plan",
            "perm": "rw",
        },
        {
            "cmd": "balancer show name=plan,type=CephString",
            "desc": "Show details of an optimization plan",
            "perm": "r",
        },
        {
            "cmd": "balancer rm name=plan,type=CephString",
            "desc": "Discard an optimization plan",
            "perm": "rw",
        },
        {
            "cmd": "balancer reset",
            "desc": "Discard all optimization plans",
            "perm": "rw",
        },
        {
            "cmd": "balancer dump name=plan,type=CephString",
            "desc": "Show an optimization plan",
            "perm": "r",
        },
        {
            "cmd": "balancer ls",
            "desc": "List all plans",
            "perm": "r",
        },
        {
            "cmd": "balancer execute name=plan,type=CephString",
            "desc": "Execute an optimization plan",
            "perm": "rw",
        },
    ]
    active = False
    run = True
    plans = {}
    mode = ''

    def __init__(self, *args, **kwargs):
        super(Module, self).__init__(*args, **kwargs)
        self.event = Event()

    def handle_command(self, inbuf, command):
        self.log.warn("Handling command: '%s'" % str(command))
        if command['prefix'] == 'balancer status':
            s = {
                'plans': list(self.plans.keys()),
                'active': self.active,
                'mode': self.get_config('mode', default_mode),
            }
            return (0, json.dumps(s, indent=4), '')
        elif command['prefix'] == 'balancer mode':
            self.set_config('mode', command['mode'])
            return (0, '', '')
        elif command['prefix'] == 'balancer on':
            if not self.active:
                self.set_config('active', '1')
                self.active = True
            self.event.set()
            return (0, '', '')
        elif command['prefix'] == 'balancer off':
            if self.active:
                self.set_config('active', '')
                self.active = False
            self.event.set()
            return (0, '', '')
        elif command['prefix'] == 'balancer eval' or command[
                'prefix'] == 'balancer eval-verbose':
            verbose = command['prefix'] == 'balancer eval-verbose'
            pools = []
            if 'option' in command:
                plan = self.plans.get(command['option'])
                if not plan:
                    # not a plan, does it look like a pool?
                    osdmap = self.get_osdmap()
                    valid_pool_names = [
                        p['pool_name'] for p in osdmap.dump().get('pools', [])
                    ]
                    option = command['option']
                    if option not in valid_pool_names:
                        return (-errno.EINVAL, '',
                                'option "%s" not a plan or a pool' % option)
                    pools.append(option)
                    ms = MappingState(osdmap, self.get("pg_dump"),
                                      'pool "%s"' % option)
                else:
                    pools = plan.pools
                    ms = plan.final_state()
            else:
                ms = MappingState(self.get_osdmap(), self.get("pg_dump"),
                                  'current cluster')
            return (0, self.evaluate(ms, pools, verbose=verbose), '')
        elif command['prefix'] == 'balancer optimize':
            pools = []
            if 'pools' in command:
                pools = command['pools']
            osdmap = self.get_osdmap()
            valid_pool_names = [
                p['pool_name'] for p in osdmap.dump().get('pools', [])
            ]
            invalid_pool_names = []
            for p in pools:
                if p not in valid_pool_names:
                    invalid_pool_names.append(p)
            if len(invalid_pool_names):
                return (-errno.EINVAL, '',
                        'pools %s not found' % invalid_pool_names)
            plan = self.plan_create(command['plan'], osdmap, pools)
            r, detail = self.optimize(plan)
            # remove plan if we are currently unable to find an optimization
            # or distribution is already perfect
            if r:
                self.plan_rm(command['plan'])
            return (r, '', detail)
        elif command['prefix'] == 'balancer rm':
            self.plan_rm(command['plan'])
            return (0, '', '')
        elif command['prefix'] == 'balancer reset':
            self.plans = {}
            return (0, '', '')
        elif command['prefix'] == 'balancer ls':
            return (0, json.dumps([p for p in self.plans], indent=4), '')
        elif command['prefix'] == 'balancer dump':
            plan = self.plans.get(command['plan'])
            if not plan:
                return (-errno.ENOENT, '',
                        'plan %s not found' % command['plan'])
            return (0, plan.dump(), '')
        elif command['prefix'] == 'balancer show':
            plan = self.plans.get(command['plan'])
            if not plan:
                return (-errno.ENOENT, '',
                        'plan %s not found' % command['plan'])
            return (0, plan.show(), '')
        elif command['prefix'] == 'balancer execute':
            plan = self.plans.get(command['plan'])
            if not plan:
                return (-errno.ENOENT, '',
                        'plan %s not found' % command['plan'])
            r, detail = self.execute(plan)
            self.plan_rm(command['plan'])
            return (r, '', detail)
        else:
            return (-errno.EINVAL, '',
                    "Command not found '{0}'".format(command['prefix']))

    def shutdown(self):
        self.log.info('Stopping')
        self.run = False
        self.event.set()

    def time_in_interval(self, tod, begin, end):
        if begin <= end:
            return tod >= begin and tod < end
        else:
            return tod >= begin or tod < end

    def serve(self):
        self.log.info('Starting')
        while self.run:
            self.active = self.get_config('active', '') is not ''
            begin_time = self.get_config('begin_time') or '0000'
            end_time = self.get_config('end_time') or '2400'
            timeofday = time.strftime('%H%M', time.localtime())
            self.log.debug('Waking up [%s, scheduled for %s-%s, now %s]',
                           "active" if self.active else "inactive", begin_time,
                           end_time, timeofday)
            sleep_interval = float(
                self.get_config('sleep_interval', default_sleep_interval))
            if self.active and self.time_in_interval(timeofday, begin_time,
                                                     end_time):
                self.log.debug('Running')
                name = 'auto_%s' % time.strftime(TIME_FORMAT, time.gmtime())
                plan = self.plan_create(name, self.get_osdmap(), [])
                r, detail = self.optimize(plan)
                if r == 0:
                    self.execute(plan)
                self.plan_rm(name)
            self.log.debug('Sleeping for %d', sleep_interval)
            self.event.wait(sleep_interval)
            self.event.clear()

    def plan_create(self, name, osdmap, pools):
        plan = Plan(
            name,
            MappingState(osdmap, self.get("pg_dump"),
                         'plan %s initial' % name), pools)
        self.plans[name] = plan
        return plan

    def plan_rm(self, name):
        if name in self.plans:
            del self.plans[name]

    def calc_eval(self, ms, pools):
        pe = Eval(ms)
        pool_rule = {}
        pool_info = {}
        for p in ms.osdmap_dump.get('pools', []):
            if len(pools) and p['pool_name'] not in pools:
                continue
            # skip dead or not-yet-ready pools too
            if p['pool'] not in ms.poolids:
                continue
            pe.pool_name[p['pool']] = p['pool_name']
            pe.pool_id[p['pool_name']] = p['pool']
            pool_rule[p['pool_name']] = p['crush_rule']
            pe.pool_roots[p['pool_name']] = []
            pool_info[p['pool_name']] = p
        if len(pool_info) == 0:
            return pe
        self.log.debug('pool_name %s' % pe.pool_name)
        self.log.debug('pool_id %s' % pe.pool_id)
        self.log.debug('pools %s' % pools)
        self.log.debug('pool_rule %s' % pool_rule)

        osd_weight = {
            a['osd']: a['weight']
            for a in ms.osdmap_dump.get('osds', []) if a['weight'] > 0
        }

        # get expected distributions by root
        actual_by_root = {}
        rootids = ms.crush.find_takes()
        roots = []
        for rootid in rootids:
            ls = ms.osdmap.get_pools_by_take(rootid)
            want = []
            # find out roots associating with pools we are passed in
            for candidate in ls:
                if candidate in pe.pool_name:
                    want.append(candidate)
            if len(want) == 0:
                continue
            root = ms.crush.get_item_name(rootid)
            pe.root_pools[root] = []
            for poolid in want:
                pe.pool_roots[pe.pool_name[poolid]].append(root)
                pe.root_pools[root].append(pe.pool_name[poolid])
            pe.root_ids[root] = rootid
            roots.append(root)
            weight_map = ms.crush.get_take_weight_osd_map(rootid)
            adjusted_map = {
                osd: cw * osd_weight[osd]
                for osd, cw in six.iteritems(weight_map)
                if osd in osd_weight and cw > 0
            }
            sum_w = sum(adjusted_map.values())
            assert len(adjusted_map) == 0 or sum_w > 0
            pe.target_by_root[root] = {
                osd: w / sum_w
                for osd, w in six.iteritems(adjusted_map)
            }
            actual_by_root[root] = {
                'pgs': {},
                'objects': {},
                'bytes': {},
            }
            for osd in pe.target_by_root[root]:
                actual_by_root[root]['pgs'][osd] = 0
                actual_by_root[root]['objects'][osd] = 0
                actual_by_root[root]['bytes'][osd] = 0
            pe.total_by_root[root] = {
                'pgs': 0,
                'objects': 0,
                'bytes': 0,
            }
        self.log.debug('pool_roots %s' % pe.pool_roots)
        self.log.debug('root_pools %s' % pe.root_pools)
        self.log.debug('target_by_root %s' % pe.target_by_root)

        # pool and root actual
        for pool, pi in six.iteritems(pool_info):
            poolid = pi['pool']
            pm = ms.pg_up_by_poolid[poolid]
            pgs = 0
            objects = 0
            bytes = 0
            pgs_by_osd = {}
            objects_by_osd = {}
            bytes_by_osd = {}
            for root in pe.pool_roots[pool]:
                for osd in pe.target_by_root[root]:
                    pgs_by_osd[osd] = 0
                    objects_by_osd[osd] = 0
                    bytes_by_osd[osd] = 0
            for pgid, up in six.iteritems(pm):
                for osd in [int(osd) for osd in up]:
                    if osd == CRUSHMap.ITEM_NONE:
                        continue
                    pgs_by_osd[osd] += 1
                    objects_by_osd[osd] += ms.pg_stat[pgid]['num_objects']
                    bytes_by_osd[osd] += ms.pg_stat[pgid]['num_bytes']
                    # pick a root to associate this pg instance with.
                    # note that this is imprecise if the roots have
                    # overlapping children.
                    # FIXME: divide bytes by k for EC pools.
                    for root in pe.pool_roots[pool]:
                        if osd in pe.target_by_root[root]:
                            actual_by_root[root]['pgs'][osd] += 1
                            actual_by_root[root]['objects'][osd] += ms.pg_stat[
                                pgid]['num_objects']
                            actual_by_root[root]['bytes'][osd] += ms.pg_stat[
                                pgid]['num_bytes']
                            pgs += 1
                            objects += ms.pg_stat[pgid]['num_objects']
                            bytes += ms.pg_stat[pgid]['num_bytes']
                            pe.total_by_root[root]['pgs'] += 1
                            pe.total_by_root[root]['objects'] += ms.pg_stat[
                                pgid]['num_objects']
                            pe.total_by_root[root]['bytes'] += ms.pg_stat[
                                pgid]['num_bytes']
                            break
            pe.count_by_pool[pool] = {
                'pgs': {k: v
                        for k, v in six.iteritems(pgs_by_osd)},
                'objects': {k: v
                            for k, v in six.iteritems(objects_by_osd)},
                'bytes': {k: v
                          for k, v in six.iteritems(bytes_by_osd)},
            }
            pe.actual_by_pool[pool] = {
                'pgs': {
                    k: float(v) / float(max(pgs, 1))
                    for k, v in six.iteritems(pgs_by_osd)
                },
                'objects': {
                    k: float(v) / float(max(objects, 1))
                    for k, v in six.iteritems(objects_by_osd)
                },
                'bytes': {
                    k: float(v) / float(max(bytes, 1))
                    for k, v in six.iteritems(bytes_by_osd)
                },
            }
            pe.total_by_pool[pool] = {
                'pgs': pgs,
                'objects': objects,
                'bytes': bytes,
            }
        for root in pe.total_by_root:
            pe.count_by_root[root] = {
                'pgs': {
                    k: float(v)
                    for k, v in six.iteritems(actual_by_root[root]['pgs'])
                },
                'objects': {
                    k: float(v)
                    for k, v in six.iteritems(actual_by_root[root]['objects'])
                },
                'bytes': {
                    k: float(v)
                    for k, v in six.iteritems(actual_by_root[root]['bytes'])
                },
            }
            pe.actual_by_root[root] = {
                'pgs': {
                    k: float(v) / float(max(pe.total_by_root[root]['pgs'], 1))
                    for k, v in six.iteritems(actual_by_root[root]['pgs'])
                },
                'objects': {
                    k:
                    float(v) / float(max(pe.total_by_root[root]['objects'], 1))
                    for k, v in six.iteritems(actual_by_root[root]['objects'])
                },
                'bytes': {
                    k:
                    float(v) / float(max(pe.total_by_root[root]['bytes'], 1))
                    for k, v in six.iteritems(actual_by_root[root]['bytes'])
                },
            }
        self.log.debug('actual_by_pool %s' % pe.actual_by_pool)
        self.log.debug('actual_by_root %s' % pe.actual_by_root)

        # average and stddev and score
        pe.stats_by_root = {
            a: pe.calc_stats(b, pe.target_by_root[a], pe.total_by_root[a])
            for a, b in six.iteritems(pe.count_by_root)
        }
        self.log.debug('stats_by_root %s' % pe.stats_by_root)

        # the scores are already normalized
        pe.score_by_root = {
            r: {
                'pgs': pe.stats_by_root[r]['pgs']['score'],
                'objects': pe.stats_by_root[r]['objects']['score'],
                'bytes': pe.stats_by_root[r]['bytes']['score'],
            }
            for r in pe.total_by_root.keys()
        }
        self.log.debug('score_by_root %s' % pe.score_by_root)

        # get the list of score metrics, comma separated
        metrics = self.get_config('crush_compat_metrics',
                                  'pgs,objects,bytes').split(',')

        # total score is just average of normalized stddevs
        pe.score = 0.0
        for r, vs in six.iteritems(pe.score_by_root):
            for k, v in six.iteritems(vs):
                if k in metrics:
                    pe.score += v
        pe.score /= len(metrics) * len(roots)
        return pe

    def evaluate(self, ms, pools, verbose=False):
        pe = self.calc_eval(ms, pools)
        return pe.show(verbose=verbose)

    def optimize(self, plan):
        self.log.info('Optimize plan %s' % plan.name)
        plan.mode = self.get_config('mode', default_mode)
        max_misplaced = float(self.get_option('target_max_misplaced_ratio'))
        self.log.info('Mode %s, max misplaced %f' % (plan.mode, max_misplaced))

        info = self.get('pg_status')
        unknown = info.get('unknown_pgs_ratio', 0.0)
        degraded = info.get('degraded_ratio', 0.0)
        inactive = info.get('inactive_pgs_ratio', 0.0)
        misplaced = info.get('misplaced_ratio', 0.0)
        self.log.debug('unknown %f degraded %f inactive %f misplaced %g',
                       unknown, degraded, inactive, misplaced)
        if unknown > 0.0:
            detail = 'Some PGs (%f) are unknown; try again later' % unknown
            self.log.info(detail)
            return -errno.EAGAIN, detail
        elif degraded > 0.0:
            detail = 'Some objects (%f) are degraded; try again later' % degraded
            self.log.info(detail)
            return -errno.EAGAIN, detail
        elif inactive > 0.0:
            detail = 'Some PGs (%f) are inactive; try again later' % inactive
            self.log.info(detail)
            return -errno.EAGAIN, detail
        elif misplaced >= max_misplaced:
            detail = 'Too many objects (%f > %f) are misplaced; ' \
                     'try again later' % (misplaced, max_misplaced)
            self.log.info(detail)
            return -errno.EAGAIN, detail
        else:
            if plan.mode == 'upmap':
                return self.do_upmap(plan)
            elif plan.mode == 'crush-compat':
                return self.do_crush_compat(plan)
            elif plan.mode == 'none':
                detail = 'Please do "ceph balancer mode" to choose a valid mode first'
                self.log.info('Idle')
                return -errno.ENOEXEC, detail
            else:
                detail = 'Unrecognized mode %s' % plan.mode
                self.log.info(detail)
                return -errno.EINVAL, detail
        ##

    def do_upmap(self, plan):
        self.log.info('do_upmap')
        max_iterations = int(self.get_config('upmap_max_iterations', 10))
        max_deviation = float(self.get_config('upmap_max_deviation', .01))

        ms = plan.initial
        if len(plan.pools):
            pools = plan.pools
        else:  # all
            pools = [
                str(i['pool_name']) for i in ms.osdmap_dump.get('pools', [])
            ]
        if len(pools) == 0:
            detail = 'No pools available'
            self.log.info(detail)
            return -errno.ENOENT, detail
        # shuffle pool list so they all get equal (in)attention
        random.shuffle(pools)
        self.log.info('pools %s' % pools)

        inc = plan.inc
        total_did = 0
        left = max_iterations
        for pool in pools:
            did = ms.osdmap.calc_pg_upmaps(inc, max_deviation, left, [pool])
            total_did += did
            left -= did
            if left <= 0:
                break
        self.log.info('prepared %d/%d changes' % (total_did, max_iterations))
        if total_did == 0:
            return -errno.EALREADY, 'Unable to find further optimization,' \
                                    'or distribution is already perfect'
        return 0, ''

    def do_crush_compat(self, plan):
        self.log.info('do_crush_compat')
        max_iterations = int(self.get_config('crush_compat_max_iterations',
                                             25))
        if max_iterations < 1:
            return -errno.EINVAL, '"crush_compat_max_iterations" must be >= 1'
        step = float(self.get_config('crush_compat_step', .5))
        if step <= 0 or step >= 1.0:
            return -errno.EINVAL, '"crush_compat_step" must be in (0, 1)'
        max_misplaced = float(self.get_option('target_max_misplaced_ratio'))
        min_pg_per_osd = 2

        ms = plan.initial
        osdmap = ms.osdmap
        crush = osdmap.get_crush()
        pe = self.calc_eval(ms, plan.pools)
        min_score_to_optimize = float(self.get_config('min_score', 0))
        if pe.score <= min_score_to_optimize:
            if pe.score == 0:
                detail = 'Distribution is already perfect'
            else:
                detail = 'score %f <= min_score %f, will not optimize' \
                         % (pe.score, min_score_to_optimize)
            self.log.info(detail)
            return -errno.EALREADY, detail

        # get current osd reweights
        orig_osd_weight = {
            a['osd']: a['weight']
            for a in ms.osdmap_dump.get('osds', [])
        }
        reweighted_osds = [
            a for a, b in six.iteritems(orig_osd_weight) if b < 1.0 and b > 0.0
        ]

        # get current compat weight-set weights
        orig_ws = self.get_compat_weight_set_weights(ms)
        if not orig_ws:
            return -errno.EAGAIN, 'compat weight-set not available'
        orig_ws = {a: b for a, b in six.iteritems(orig_ws) if a >= 0}

        # Make sure roots don't overlap their devices.  If so, we
        # can't proceed.
        roots = pe.target_by_root.keys()
        self.log.debug('roots %s', roots)
        visited = {}
        overlap = {}
        root_ids = {}
        for root, wm in six.iteritems(pe.target_by_root):
            for osd in wm:
                if osd in visited:
                    if osd not in overlap:
                        overlap[osd] = [visited[osd]]
                    overlap[osd].append(root)
                visited[osd] = root
        if len(overlap) > 0:
            detail = 'Some osds belong to multiple subtrees: %s' % \
                     overlap
            self.log.error(detail)
            return -errno.EOPNOTSUPP, detail

        # rebalance by pgs, objects, or bytes
        metrics = self.get_config('crush_compat_metrics',
                                  'pgs,objects,bytes').split(',')
        key = metrics[0]  # balancing using the first score metric
        if key not in ['pgs', 'bytes', 'objects']:
            self.log.warn(
                "Invalid crush_compat balancing key %s. Using 'pgs'." % key)
            key = 'pgs'

        # go
        best_ws = copy.deepcopy(orig_ws)
        best_ow = copy.deepcopy(orig_osd_weight)
        best_pe = pe
        left = max_iterations
        bad_steps = 0
        next_ws = copy.deepcopy(best_ws)
        next_ow = copy.deepcopy(best_ow)
        while left > 0:
            # adjust
            self.log.debug('best_ws %s' % best_ws)
            random.shuffle(roots)
            for root in roots:
                pools = best_pe.root_pools[root]
                osds = len(best_pe.target_by_root[root])
                min_pgs = osds * min_pg_per_osd
                if best_pe.total_by_root[root][key] < min_pgs:
                    self.log.info(
                        'Skipping root %s (pools %s), total pgs %d '
                        '< minimum %d (%d per osd)', root, pools,
                        best_pe.total_by_root[root][key], min_pgs,
                        min_pg_per_osd)
                    continue
                self.log.info('Balancing root %s (pools %s) by %s' %
                              (root, pools, key))
                target = best_pe.target_by_root[root]
                actual = best_pe.actual_by_root[root][key]
                queue = sorted(actual.keys(),
                               key=lambda osd: -abs(target[osd] - actual[osd]))
                for osd in queue:
                    if orig_osd_weight[osd] == 0:
                        self.log.debug('skipping out osd.%d', osd)
                    else:
                        deviation = target[osd] - actual[osd]
                        if deviation == 0:
                            break
                        self.log.debug('osd.%d deviation %f', osd, deviation)
                        weight = best_ws[osd]
                        ow = orig_osd_weight[osd]
                        if actual[osd] > 0:
                            calc_weight = target[osd] / actual[
                                osd] * weight * ow
                        else:
                            # not enough to go on here... keep orig weight
                            calc_weight = weight / orig_osd_weight[osd]
                        new_weight = weight * (1.0 - step) + calc_weight * step
                        self.log.debug('Reweight osd.%d %f -> %f', osd, weight,
                                       new_weight)
                        next_ws[osd] = new_weight
                        if ow < 1.0:
                            new_ow = min(
                                1.0, max(step + (1.0 - step) * ow, ow + .005))
                            self.log.debug('Reweight osd.%d reweight %f -> %f',
                                           osd, ow, new_ow)
                            next_ow[osd] = new_ow

                # normalize weights under this root
                root_weight = crush.get_item_weight(pe.root_ids[root])
                root_sum = sum(b for a, b in six.iteritems(next_ws)
                               if a in target.keys())
                if root_sum > 0 and root_weight > 0:
                    factor = root_sum / root_weight
                    self.log.debug(
                        'normalizing root %s %d, weight %f, '
                        'ws sum %f, factor %f', root, pe.root_ids[root],
                        root_weight, root_sum, factor)
                    for osd in actual.keys():
                        next_ws[osd] = next_ws[osd] / factor

            # recalc
            plan.compat_ws = copy.deepcopy(next_ws)
            next_ms = plan.final_state()
            next_pe = self.calc_eval(next_ms, plan.pools)
            next_misplaced = next_ms.calc_misplaced_from(ms)
            self.log.debug('Step result score %f -> %f, misplacing %f',
                           best_pe.score, next_pe.score, next_misplaced)

            if next_misplaced > max_misplaced:
                if best_pe.score < pe.score:
                    self.log.debug('Step misplaced %f > max %f, stopping',
                                   next_misplaced, max_misplaced)
                    break
                step /= 2.0
                next_ws = copy.deepcopy(best_ws)
                next_ow = copy.deepcopy(best_ow)
                self.log.debug(
                    'Step misplaced %f > max %f, reducing step to %f',
                    next_misplaced, max_misplaced, step)
            else:
                if next_pe.score > best_pe.score * 1.0001:
                    bad_steps += 1
                    if bad_steps < 5 and random.randint(0, 100) < 70:
                        self.log.debug('Score got worse, taking another step')
                    else:
                        step /= 2.0
                        next_ws = copy.deepcopy(best_ws)
                        next_ow = copy.deepcopy(best_ow)
                        self.log.debug(
                            'Score got worse, trying smaller step %f', step)
                else:
                    bad_steps = 0
                    best_pe = next_pe
                    best_ws = copy.deepcopy(next_ws)
                    best_ow = copy.deepcopy(next_ow)
                    if best_pe.score == 0:
                        break
            left -= 1

        # allow a small regression if we are phasing out osd weights
        fudge = 0
        if next_ow != orig_osd_weight:
            fudge = .001

        if best_pe.score < pe.score + fudge:
            self.log.info('Success, score %f -> %f', pe.score, best_pe.score)
            plan.compat_ws = best_ws
            for osd, w in six.iteritems(best_ow):
                if w != orig_osd_weight[osd]:
                    self.log.debug('osd.%d reweight %f', osd, w)
                    plan.osd_weights[osd] = w
            return 0, ''
        else:
            self.log.info('Failed to find further optimization, score %f',
                          pe.score)
            plan.compat_ws = {}
            return -errno.EDOM, 'Unable to find further optimization, ' \
                                'change balancer mode and retry might help'

    def get_compat_weight_set_weights(self, ms):
        if not CRUSHMap.have_default_choose_args(ms.crush_dump):
            # enable compat weight-set first
            self.log.debug('ceph osd crush weight-set create-compat')
            result = CommandResult('')
            self.send_command(
                result, 'mon', '',
                json.dumps({
                    'prefix': 'osd crush weight-set create-compat',
                    'format': 'json',
                }), '')
            r, outb, outs = result.wait()
            if r != 0:
                self.log.error('Error creating compat weight-set')
                return

            result = CommandResult('')
            self.send_command(
                result, 'mon', '',
                json.dumps({
                    'prefix': 'osd crush dump',
                    'format': 'json',
                }), '')
            r, outb, outs = result.wait()
            if r != 0:
                self.log.error('Error dumping crush map')
                return
            try:
                crushmap = json.loads(outb)
            except:
                raise RuntimeError('unable to parse crush map')
        else:
            crushmap = ms.crush_dump

        raw = CRUSHMap.get_default_choose_args(crushmap)
        weight_set = {}
        for b in raw:
            bucket = None
            for t in crushmap['buckets']:
                if t['id'] == b['bucket_id']:
                    bucket = t
                    break
            if not bucket:
                raise RuntimeError('could not find bucket %s' % b['bucket_id'])
            self.log.debug('bucket items %s' % bucket['items'])
            self.log.debug('weight set %s' % b['weight_set'][0])
            if len(bucket['items']) != len(b['weight_set'][0]):
                raise RuntimeError(
                    'weight-set size does not match bucket items')
            for pos in range(len(bucket['items'])):
                weight_set[bucket['items'][pos]
                           ['id']] = b['weight_set'][0][pos]

        self.log.debug('weight_set weights %s' % weight_set)
        return weight_set

    def do_crush(self):
        self.log.info('do_crush (not yet implemented)')

    def do_osd_weight(self):
        self.log.info('do_osd_weight (not yet implemented)')

    def execute(self, plan):
        self.log.info('Executing plan %s' % plan.name)

        commands = []

        # compat weight-set
        if len(plan.compat_ws) and \
           not CRUSHMap.have_default_choose_args(plan.initial.crush_dump):
            self.log.debug('ceph osd crush weight-set create-compat')
            result = CommandResult('')
            self.send_command(
                result, 'mon', '',
                json.dumps({
                    'prefix': 'osd crush weight-set create-compat',
                    'format': 'json',
                }), '')
            r, outb, outs = result.wait()
            if r != 0:
                self.log.error('Error creating compat weight-set')
                return r, outs

        for osd, weight in six.iteritems(plan.compat_ws):
            self.log.info(
                'ceph osd crush weight-set reweight-compat osd.%d %f', osd,
                weight)
            result = CommandResult('')
            self.send_command(
                result, 'mon', '',
                json.dumps({
                    'prefix': 'osd crush weight-set reweight-compat',
                    'format': 'json',
                    'item': 'osd.%d' % osd,
                    'weight': [weight],
                }), '')
            commands.append(result)

        # new_weight
        reweightn = {}
        for osd, weight in six.iteritems(plan.osd_weights):
            reweightn[str(osd)] = str(int(weight * float(0x10000)))
        if len(reweightn):
            self.log.info('ceph osd reweightn %s', reweightn)
            result = CommandResult('')
            self.send_command(
                result, 'mon', '',
                json.dumps({
                    'prefix': 'osd reweightn',
                    'format': 'json',
                    'weights': json.dumps(reweightn),
                }), '')
            commands.append(result)

        # upmap
        incdump = plan.inc.dump()
        for pgid in incdump.get('old_pg_upmap_items', []):
            self.log.info('ceph osd rm-pg-upmap-items %s', pgid)
            result = CommandResult('foo')
            self.send_command(
                result, 'mon', '',
                json.dumps({
                    'prefix': 'osd rm-pg-upmap-items',
                    'format': 'json',
                    'pgid': pgid,
                }), 'foo')
            commands.append(result)

        for item in incdump.get('new_pg_upmap_items', []):
            self.log.info('ceph osd pg-upmap-items %s mappings %s',
                          item['pgid'], item['mappings'])
            osdlist = []
            for m in item['mappings']:
                osdlist += [m['from'], m['to']]
            result = CommandResult('foo')
            self.send_command(
                result, 'mon', '',
                json.dumps({
                    'prefix': 'osd pg-upmap-items',
                    'format': 'json',
                    'pgid': item['pgid'],
                    'id': osdlist,
                }), 'foo')
            commands.append(result)

        # wait for commands
        self.log.debug('commands %s' % commands)
        for result in commands:
            r, outb, outs = result.wait()
            if r != 0:
                self.log.error('execute error: r = %d, detail = %s' %
                               (r, outs))
                return r, outs
        self.log.debug('done')
        return 0, ''
Beispiel #54
0
class SSHReaderThread(Thread):  # pylint: disable=too-many-instance-attributes
    """
    Thread that reads data from ssh session socket and forwards it to Queue.
    It is needed because socket buffer gets overflowed if data is sent faster than watchers can process it, so
      we have to have Queue as a buffer with 'endless' memory, and fast reader that reads data from the socket
      and forward it to the Queue.
      As part of this process it splits data into lines, because watchers expect it is organized in this way.
    """
    def __init__(self, session: Session, channel: Channel,
                 timeout: NullableTiming, timeout_read_data: NullableTiming):
        self.stdout = Queue()
        self.stderr = Queue()
        self.timeout_reached = False
        self._session = session
        self._channel = channel
        self._timeout = timeout
        self._timeout_read_data = timeout_read_data
        self.raised = None
        self._can_run = Event()
        self._can_run.set()
        super().__init__(daemon=True)

    def run(self):
        try:
            self._read_output(self._session, self._channel, self._timeout,
                              self._timeout_read_data, self.stdout,
                              self.stderr)
        except Exception as exc:  # pylint: disable=broad-except
            self.raised = exc

    def _read_output(  # pylint: disable=too-many-arguments,too-many-branches
            self, session: Session, channel: Channel, timeout: NullableTiming,
            timeout_read_data: NullableTiming, stdout_stream: Queue,
            stderr_stream: Queue):
        """Reads data from ssh session, split it into lines and forward lines into stderr ad stdout pipes
        It is required for it to be fast, that is why there is code duplications and non-pythonic code
        """
        # pylint: disable=too-many-locals
        stdout_remainder = stderr_remainder = b''
        if timeout is None:
            end_time = float_info.max
        else:
            end_time = perf_counter() + timeout
        eof_result = stdout_size = stderr_size = 1
        while eof_result == LIBSSH2_ERROR_EAGAIN or stdout_size == LIBSSH2_ERROR_EAGAIN or \
                stdout_size > 0 or stderr_size == LIBSSH2_ERROR_EAGAIN or stderr_size > 0:  # pylint: disable=consider-using-in
            if not self._can_run.is_set():
                break
            if perf_counter() > end_time:
                self.timeout_reached = True
                break
            with session.lock:
                if stdout_size == LIBSSH2_ERROR_EAGAIN and stderr_size == LIBSSH2_ERROR_EAGAIN:  # pylint: disable=consider-using-in
                    session.simple_select(timeout=timeout_read_data)
                stdout_size, stdout_chunk = channel.read()
                stderr_size, stderr_chunk = channel.read_stderr()
                eof_result = channel.eof()

            if stdout_chunk and stdout_stream is not None:
                data_splitted = stdout_chunk.split(LINESEP)
                if len(data_splitted) == 1:
                    stdout_remainder = stdout_remainder + data_splitted.pop()
                else:
                    if stdout_remainder:
                        stdout_stream.put(stdout_remainder +
                                          data_splitted.pop(0))
                    stdout_remainder = data_splitted.pop()
                for chunk in data_splitted:
                    stdout_stream.put(chunk)

            if stderr_chunk and stderr_stream is not None:
                data_splitted = stderr_chunk.split(LINESEP)
                if len(data_splitted) == 1:
                    stderr_remainder = stderr_remainder + data_splitted.pop()
                else:
                    if stderr_remainder:
                        stderr_stream.put(stderr_remainder +
                                          data_splitted.pop(0))
                    stderr_remainder = data_splitted.pop()
                for chunk in data_splitted:
                    stderr_stream.put(chunk)
        if stdout_remainder:
            stdout_stream.put(stdout_remainder)
        if stderr_remainder:
            stderr_stream.put(stderr_remainder)

    def stop(self, timeout: float = None):
        self._can_run.clear()
        self.join(timeout)
Beispiel #55
0

print("Searching for device...")
d = MetaWear(sys.argv[1])
d.connect()
print("Connected to " + d.address)
s = State(d)

print("Configuring device")
libmetawear.mbl_mw_settings_set_connection_parameters(s.device.board, 7.5, 7.5,
                                                      0, 6000)
sleep(1.0)

try:
    s.setup_logger()

    print("Logging data for 15s")
    sleep(15.0)

    s.download_data()

except RuntimeError as e:
    print(e)

finally:
    print("Resetting device")
    e = Event()
    s.device.on_disconnect = lambda status: e.set()
    libmetawear.mbl_mw_debug_reset(s.device.board)
    e.wait()
Beispiel #56
0
class _ConsumerThread(Thread):
    def __init__(self, *, broker, queue_name, prefetch, work_queue,
                 worker_timeout):
        super().__init__(daemon=True)

        self.logger = get_logger(__name__, "ConsumerThread(%s)" % queue_name)
        self.running = False
        self.paused = False
        self.paused_event = Event()
        self.consumer = None
        self.broker = broker
        self.prefetch = prefetch
        self.queue_name = queue_name
        self.work_queue = work_queue
        self.worker_timeout = worker_timeout
        self.delay_queue = PriorityQueue()  # type: PriorityQueue

    def run(self):
        self.logger.debug("Running consumer thread...")
        self.running = True
        restart_consumer = CONSUMER_RESTART_DELAY_SECS > 0

        while self.running:
            if self.paused:
                self.logger.debug(
                    "Consumer is paused. Sleeping for %.02fms...",
                    self.worker_timeout)
                self.paused_event.set()
                time.sleep(self.worker_timeout / 1000)
                continue

            try:
                self.consumer = self.broker.consume(
                    queue_name=self.queue_name,
                    prefetch=self.prefetch,
                    timeout=self.worker_timeout)

                for message in self.consumer:
                    if message is not None:
                        self.handle_message(message)

                    elif self.paused:
                        break

                    self.handle_delayed_messages()
                    if not self.running:
                        break

            except ConnectionError as e:
                self.logger.critical(
                    "Consumer encountered a connection error: %s", e)
                self.delay_queue = PriorityQueue()
                if not restart_consumer:
                    self.stop()
            except Exception:
                self.logger.critical(
                    "Consumer encountered an unexpected error.", exc_info=True)
                # Avoid leaving any open file descriptors around when
                # an exception occurs.
                self.close()
                if not restart_consumer:
                    self.stop()

            # While the consumer is running (i.e. hasn't been shut down),
            # try to restart it once every CONSUMER_RESTART_DELAY_SECS second.
            if self.running and restart_consumer:
                self.logger.info("Restarting consumer in %0.2f seconds.",
                                 CONSUMER_RESTART_DELAY_SECS)
                self.close()
                time.sleep(CONSUMER_RESTART_DELAY_SECS)

        # If it's no longer running, then shut it down gracefully.
        self.broker.emit_before("consumer_thread_shutdown", self)
        self.logger.debug("Consumer thread stopped.")

    def handle_delayed_messages(self):
        """Enqueue any delayed messages whose eta has passed.
        """
        for eta, message in iter_queue(self.delay_queue):
            if eta > current_millis():
                self.delay_queue.put((eta, message))
                self.delay_queue.task_done()
                break

            queue_name = q_name(message.queue_name)
            new_message = message.copy(queue_name=queue_name)
            del new_message.options["eta"]

            self.broker.enqueue(new_message)
            self.post_process_message(message)
            self.delay_queue.task_done()

    def handle_message(self, message):
        """Handle a message received off of the underlying consumer.
        If the message has an eta, delay it.  Otherwise, put it on the
        work queue.
        """
        try:
            if "eta" in message.options:
                self.logger.debug("Pushing message %r onto delay queue.",
                                  message.message_id)
                self.broker.emit_before("delay_message", message)
                self.delay_queue.put((message.options.get("eta", 0), message))

            else:
                actor = self.broker.get_actor(message.actor_name)
                self.logger.debug("Pushing message %r onto work queue.",
                                  message.message_id)
                self.work_queue.put((-actor.priority, message))
        except ActorNotFound:
            self.logger.error(
                "Received message for undefined actor %r. Moving it to the DLQ.",
                message.actor_name,
                exc_info=True,
            )
            message.fail()
            self.post_process_message(message)

    def post_process_message(self, message):
        """Called by worker threads whenever they're done processing
        individual messages, signaling that each message is ready to
        be acked or rejected.
        """
        if message.failed:
            self.logger.debug("Rejecting message %r.", message.message_id)
            self.broker.emit_before("nack", message)
            self.consumer.nack(message)
            self.broker.emit_after("nack", message)

        else:
            self.logger.debug("Acknowledging message %r.", message.message_id)
            self.broker.emit_before("ack", message)
            self.consumer.ack(message)
            self.broker.emit_after("ack", message)

    def requeue_messages(self, messages):
        """Called on worker shutdown and whenever there is a
        connection error to move unacked messages back to their
        respective queues asap.
        """
        self.consumer.requeue(messages)

    def pause(self):
        """Pause this consumer.
        """
        self.paused = True
        self.paused_event.clear()

    def resume(self):
        """Resume this consumer.
        """
        self.paused = False
        self.paused_event.clear()

    def stop(self):
        """Initiate the ConsumerThread shutdown sequence.

        Code calling this method should then join on the thread and
        wait for it to finish shutting down.
        """
        self.logger.debug("Stopping consumer thread...")
        self.running = False

    def close(self):
        """Close this consumer thread and its underlying connection.
        """
        try:
            if self.consumer:
                self.requeue_messages(m
                                      for _, m in iter_queue(self.delay_queue))
                self.consumer.close()
        except ConnectionError:
            pass
class JobQueue(object):
    """This class allows you to periodically perform tasks with the bot.

    Attributes:
        _queue (:obj:`PriorityQueue`): The queue that holds the Jobs.
        bot (:class:`telegram.Bot`): The bot instance that should be passed to the jobs.
            DEPRECATED: Use :attr:`set_dispatcher` instead.

    """
    def __init__(self, bot=None):
        self._queue = PriorityQueue()
        if bot:
            warnings.warn(
                "Passing bot to jobqueue is deprecated. Please use set_dispatcher "
                "instead!",
                TelegramDeprecationWarning,
                stacklevel=2)

            class MockDispatcher(object):
                def __init__(self):
                    self.bot = bot
                    self.use_context = False

            self._dispatcher = MockDispatcher()
        else:
            self._dispatcher = None
        self.logger = logging.getLogger(self.__class__.__name__)
        self.__start_lock = Lock()
        self.__next_peek_lock = Lock(
        )  # to protect self._next_peek & self.__tick
        self.__tick = Event()
        self.__thread = None
        self._next_peek = None
        self._running = False

    def set_dispatcher(self, dispatcher):
        """Set the dispatcher to be used by this JobQueue. Use this instead of passing a
        :class:`telegram.Bot` to the JobQueue, which is deprecated.

        Args:
            dispatcher (:class:`telegram.ext.Dispatcher`): The dispatcher.

        """
        self._dispatcher = dispatcher

    def _put(self, job, time_spec=None, previous_t=None):
        """
        Enqueues the job, scheduling its next run at the correct time.

        Args:
            job (telegram.ext.Job): job to enqueue
            time_spec (optional):
                Specification of the time for which the job should be scheduled. The precise
                semantics of this parameter depend on its type (see
                :func:`telegram.ext.JobQueue.run_repeating` for details).
                Defaults to now + ``job.interval``.
            previous_t (optional):
                Time at which the job last ran (``None`` if it hasn't run yet).

        """
        # get time at which to run:
        if time_spec is None:
            time_spec = job.interval
        if time_spec is None:
            raise ValueError(
                "no time specification given for scheduling non-repeating job")
        next_t = to_float_timestamp(time_spec, reference_timestamp=previous_t)

        # enqueue:
        self.logger.debug('Putting job %s with t=%s', job.name, time_spec)
        self._queue.put((next_t, job))

        # Wake up the loop if this job should be executed next
        self._set_next_peek(next_t)

    def run_once(self, callback, when, context=None, name=None):
        """Creates a new ``Job`` that runs once and adds it to the queue.

        Args:
            callback (:obj:`callable`): The callback function that should be executed by the new
                job. Callback signature for context based API:

                    ``def callback(CallbackContext)``

                ``context.job`` is the :class:`telegram.ext.Job` instance. It can be used to access
                its ``job.context`` or change it to a repeating job.
            when (:obj:`int` | :obj:`float` | :obj:`datetime.timedelta` |                         \
                  :obj:`datetime.datetime` | :obj:`datetime.time`):
                Time in or at which the job should run. This parameter will be interpreted
                depending on its type.

                * :obj:`int` or :obj:`float` will be interpreted as "seconds from now" in which the
                  job should run.
                * :obj:`datetime.timedelta` will be interpreted as "time from now" in which the
                  job should run.
                * :obj:`datetime.datetime` will be interpreted as a specific date and time at
                  which the job should run. If the timezone (``datetime.tzinfo``) is ``None``, UTC
                  will be assumed.
                * :obj:`datetime.time` will be interpreted as a specific time of day at which the
                  job should run. This could be either today or, if the time has already passed,
                  tomorrow. If the timezone (``time.tzinfo``) is ``None``, UTC will be assumed.

            context (:obj:`object`, optional): Additional data needed for the callback function.
                Can be accessed through ``job.context`` in the callback. Defaults to ``None``.
            name (:obj:`str`, optional): The name of the new job. Defaults to
                ``callback.__name__``.

        Returns:
            :class:`telegram.ext.Job`: The new ``Job`` instance that has been added to the job
            queue.

        """
        job = Job(callback,
                  repeat=False,
                  context=context,
                  name=name,
                  job_queue=self)
        self._put(job, time_spec=when)
        return job

    def run_repeating(self,
                      callback,
                      interval,
                      first=None,
                      context=None,
                      name=None):
        """Creates a new ``Job`` that runs at specified intervals and adds it to the queue.

        Args:
            callback (:obj:`callable`): The callback function that should be executed by the new
                job. Callback signature for context based API:

                    ``def callback(CallbackContext)``

                ``context.job`` is the :class:`telegram.ext.Job` instance. It can be used to access
                its ``job.context`` or change it to a repeating job.
            interval (:obj:`int` | :obj:`float` | :obj:`datetime.timedelta`): The interval in which
                the job will run. If it is an :obj:`int` or a :obj:`float`, it will be interpreted
                as seconds.
            first (:obj:`int` | :obj:`float` | :obj:`datetime.timedelta` |                        \
                   :obj:`datetime.datetime` | :obj:`datetime.time`, optional):
                Time in or at which the job should run. This parameter will be interpreted
                depending on its type.

                * :obj:`int` or :obj:`float` will be interpreted as "seconds from now" in which the
                  job should run.
                * :obj:`datetime.timedelta` will be interpreted as "time from now" in which the
                  job should run.
                * :obj:`datetime.datetime` will be interpreted as a specific date and time at
                  which the job should run. If the timezone (``datetime.tzinfo``) is ``None``, UTC
                  will be assumed.
                * :obj:`datetime.time` will be interpreted as a specific time of day at which the
                  job should run. This could be either today or, if the time has already passed,
                  tomorrow. If the timezone (``time.tzinfo``) is ``None``, UTC will be assumed.

                Defaults to ``interval``
            context (:obj:`object`, optional): Additional data needed for the callback function.
                Can be accessed through ``job.context`` in the callback. Defaults to ``None``.
            name (:obj:`str`, optional): The name of the new job. Defaults to
                ``callback.__name__``.

        Returns:
            :class:`telegram.ext.Job`: The new ``Job`` instance that has been added to the job
            queue.

        Notes:
             `interval` is always respected "as-is". That means that if DST changes during that
             interval, the job might not run at the time one would expect. It is always recommended
             to pin servers to UTC time, then time related behaviour can always be expected.

        """
        job = Job(callback,
                  interval=interval,
                  repeat=True,
                  context=context,
                  name=name,
                  job_queue=self)
        self._put(job, time_spec=first)
        return job

    def run_daily(self,
                  callback,
                  time,
                  days=Days.EVERY_DAY,
                  context=None,
                  name=None):
        """Creates a new ``Job`` that runs on a daily basis and adds it to the queue.

        Args:
            callback (:obj:`callable`): The callback function that should be executed by the new
                job. Callback signature for context based API:

                    ``def callback(CallbackContext)``

                ``context.job`` is the :class:`telegram.ext.Job` instance. It can be used to access
                its ``job.context`` or change it to a repeating job.
            time (:obj:`datetime.time`): Time of day at which the job should run. If the timezone
                (``time.tzinfo``) is ``None``, UTC will be assumed.
            days (Tuple[:obj:`int`], optional): Defines on which days of the week the job should
                run. Defaults to ``EVERY_DAY``
            context (:obj:`object`, optional): Additional data needed for the callback function.
                Can be accessed through ``job.context`` in the callback. Defaults to ``None``.
            name (:obj:`str`, optional): The name of the new job. Defaults to
                ``callback.__name__``.

        Returns:
            :class:`telegram.ext.Job`: The new ``Job`` instance that has been added to the job
            queue.

        Notes:
             Daily is just an alias for "24 Hours". That means that if DST changes during that
             interval, the job might not run at the time one would expect. It is always recommended
             to pin servers to UTC time, then time related behaviour can always be expected.

        """
        job = Job(callback,
                  interval=datetime.timedelta(days=1),
                  repeat=True,
                  days=days,
                  tzinfo=time.tzinfo,
                  context=context,
                  name=name,
                  job_queue=self)
        self._put(job, time_spec=time)
        return job

    def _set_next_peek(self, t):
        # """
        # Set next peek if not defined or `t` is before next peek.
        # In case the next peek was set, also trigger the `self.__tick` event.
        # """
        with self.__next_peek_lock:
            if not self._next_peek or self._next_peek > t:
                self._next_peek = t
                self.__tick.set()

    def tick(self):
        """Run all jobs that are due and re-enqueue them with their interval."""
        now = time.time()

        self.logger.debug('Ticking jobs with t=%f', now)

        while True:
            try:
                t, job = self._queue.get(False)
            except Empty:
                break

            self.logger.debug('Peeked at %s with t=%f', job.name, t)

            if t > now:
                # We can get here in two conditions:
                # 1. At the second or later pass of the while loop, after we've already
                #    processed the job(s) we were supposed to at this time.
                # 2. At the first iteration of the loop only if `self.put()` had triggered
                #    `self.__tick` because `self._next_peek` wasn't set
                self.logger.debug("Next task isn't due yet. Finished!")
                self._queue.put((t, job))
                self._set_next_peek(t)
                break

            if job.removed:
                self.logger.debug('Removing job %s', job.name)
                continue

            if job.enabled:
                try:
                    current_week_day = datetime.datetime.now(
                        job.tzinfo).date().weekday()
                    if current_week_day in job.days:
                        self.logger.debug('Running job %s', job.name)
                        job.run(self._dispatcher)

                except Exception:
                    self.logger.exception(
                        'An uncaught error was raised while executing job %s',
                        job.name)
            else:
                self.logger.debug('Skipping disabled job %s', job.name)

            if job.repeat and not job.removed:
                self._put(job, previous_t=t)
            else:
                self.logger.debug('Dropping non-repeating or removed job %s',
                                  job.name)

    def start(self):
        """Starts the job_queue thread."""
        self.__start_lock.acquire()

        if not self._running:
            self._running = True
            self.__start_lock.release()
            self.__thread = Thread(target=self._main_loop,
                                   name="Bot:{}:job_queue".format(
                                       self._dispatcher.bot.id))
            self.__thread.start()
            self.logger.debug('%s thread started', self.__class__.__name__)
        else:
            self.__start_lock.release()

    def _main_loop(self):
        """
        Thread target of thread ``job_queue``. Runs in background and performs ticks on the job
        queue.

        """
        while self._running:
            # self._next_peek may be (re)scheduled during self.tick() or self.put()
            with self.__next_peek_lock:
                tmout = self._next_peek - time.time(
                ) if self._next_peek else None
                self._next_peek = None
                self.__tick.clear()

            self.__tick.wait(tmout)

            # If we were woken up by self.stop(), just bail out
            if not self._running:
                break

            self.tick()

        self.logger.debug('%s thread stopped', self.__class__.__name__)

    def stop(self):
        """Stops the thread."""
        with self.__start_lock:
            self._running = False

        self.__tick.set()
        if self.__thread is not None:
            self.__thread.join()

    def jobs(self):
        """Returns a tuple of all jobs that are currently in the ``JobQueue``."""
        with self._queue.mutex:
            return tuple(job[1] for job in self._queue.queue if job)

    def get_jobs_by_name(self, name):
        """Returns a tuple of jobs with the given name that are currently in the ``JobQueue``"""
        with self._queue.mutex:
            return tuple(job[1] for job in self._queue.queue
                         if job and job[1].name == name)
Beispiel #58
0
class TimeoutExecutor(CanCustomizeBind, Executor):
    """An executor which delegates to another executor while applying
    a timeout to each returned future.

    For any futures returned by this executor, if the future hasn't
    completed approximately within `timeout` seconds of its creation,
    an attempt will be made to cancel the future.

    Note that only a single attempt is made to cancel any future, and there
    is no guarantee that this will succeed.

    .. versionadded:: 1.7.0
    """
    def __init__(self, delegate, timeout, logger=None):
        """
        Parameters:

            delegate (~concurrent.futures.Executor):
                an executor to which callables are submitted

            timeout (float):
                timeout (in seconds) after which :meth:`concurrent.futures.Future.cancel()`
                will be invoked on any generated future which has not completed

            logger (~logging.Logger):
                a logger used for messages from this executor
        """
        self._log = logger if logger else LOG
        self._delegate = delegate
        self._timeout = timeout
        self._shutdown = False
        self._jobs = []
        self._jobs_lock = Lock()
        self._jobs_write = Event()

        event = self._jobs_write
        self_ref = weakref.ref(self, lambda _: event.set())

        self._job_thread = Thread(name="TimeoutExecutor",
                                  target=self._job_loop,
                                  args=(self_ref, ))
        self._job_thread.daemon = True
        self._job_thread.start()

    def submit(self, *args, **kwargs):  # pylint: disable=arguments-differ
        return self.submit_timeout(self._timeout, *args, **kwargs)

    def submit_timeout(self, timeout, fn, *args, **kwargs):
        """Like :code:`submit(fn, *args, **kwargs)`, but uses the specified
        timeout rather than this executor's default.

        .. versionadded:: 1.19.0
        """
        delegate_future = self._delegate.submit(fn, *args, **kwargs)
        future = MapFuture(delegate_future)
        future.add_done_callback(self._on_future_done)
        job = Job(future, delegate_future, monotonic() + timeout)
        with self._jobs_lock:
            self._jobs.append(job)
        self._jobs_write.set()
        return future

    def shutdown(self, wait=True):
        self._log.debug("shutdown")
        self._shutdown = True
        self._jobs_write.set()
        self._delegate.shutdown(wait)
        if wait:
            self._job_thread.join(MAX_TIMEOUT)

    def _partition_jobs(self):
        pending = []
        overdue = []
        now = monotonic()
        for job in self._jobs:
            if job.future.done():
                self._log.debug("Discarding job for completed future: %s", job)
            elif job.deadline < now:
                overdue.append(job)
            else:
                pending.append(job)
        return (pending, overdue)

    def _on_future_done(self, future):
        self._log.debug("Waking thread for %s", future)
        self._jobs_write.set()

    def _do_cancel(self, job):
        self._log.debug("Attempting cancel: %s", job)
        cancel_result = job.future.cancel()
        self._log.debug("Cancel of %s resulted in %s", job, cancel_result)

    @classmethod
    @executor_loop
    def _job_loop(cls, executor_ref):
        while True:
            (event, wait_time) = cls._job_loop_iter(executor_ref())
            if not event:
                break
            event.wait(wait_time)
            event.clear()

    @classmethod
    def _job_loop_iter(cls, executor):
        if not executor:
            LOG.debug("Executor was collected")
            return (None, None)

        if executor._shutdown:
            executor._log.debug("Executor was shut down")
            return (None, None)

        executor._log.debug("job loop")

        with executor._jobs_lock:
            (pending, overdue) = executor._partition_jobs()
            executor._jobs = pending

        executor._log.debug("jobs: %s overdue, %s pending", len(overdue),
                            len(pending))

        for job in overdue:
            executor._do_cancel(job)

        wait_time = None
        if pending:
            earliest = min([job.deadline for job in pending])
            wait_time = max(earliest - monotonic(), 0)

        executor._log.debug("Wait until %s", wait_time)
        return (executor._jobs_write, wait_time)
class Job(object):
    """This class encapsulates a Job.

    Attributes:
        callback (:obj:`callable`): The callback function that should be executed by the new job.
        context (:obj:`object`): Optional. Additional data needed for the callback function.
        name (:obj:`str`): Optional. The name of the new job.

    Args:
        callback (:obj:`callable`): The callback function that should be executed by the new job.
            Callback signature for context based API:

                ``def callback(CallbackContext)``

            a ``context.job`` is the :class:`telegram.ext.Job` instance. It can be used to access
            its ``job.context`` or change it to a repeating job.
        interval (:obj:`int` | :obj:`float` | :obj:`datetime.timedelta`, optional): The time
            interval between executions of the job. If it is an :obj:`int` or a :obj:`float`,
            it will be interpreted as seconds. If you don't set this value, you must set
            :attr:`repeat` to ``False`` and specify :attr:`time_spec` when you put the job into
            the job queue.
        repeat (:obj:`bool`, optional): If this job should be periodically execute its callback
            function (``True``) or only once (``False``). Defaults to ``True``.
        context (:obj:`object`, optional): Additional data needed for the callback function. Can be
            accessed through ``job.context`` in the callback. Defaults to ``None``.
        name (:obj:`str`, optional): The name of the new job. Defaults to ``callback.__name__``.
        days (Tuple[:obj:`int`], optional): Defines on which days of the week the job should run.
            Defaults to ``Days.EVERY_DAY``
        job_queue (:class:`telegram.ext.JobQueue`, optional): The ``JobQueue`` this job belongs to.
            Only optional for backward compatibility with ``JobQueue.put()``.
        tzinfo (:obj:`datetime.tzinfo`, optional): timezone associated to this job. Used when
            checking the day of the week to determine whether a job should run (only relevant when
            ``days is not Days.EVERY_DAY``). Defaults to UTC.
    """
    def __init__(self,
                 callback,
                 interval=None,
                 repeat=True,
                 context=None,
                 days=Days.EVERY_DAY,
                 name=None,
                 job_queue=None,
                 tzinfo=None):

        self.callback = callback
        self.context = context
        self.name = name or callback.__name__

        self._repeat = None
        self._interval = None
        self.interval = interval
        self.repeat = repeat

        self._days = None
        self.days = days
        self.tzinfo = tzinfo or _UTC

        self._job_queue = weakref.proxy(
            job_queue) if job_queue is not None else None

        self._remove = Event()
        self._enabled = Event()
        self._enabled.set()

    def run(self, dispatcher):
        """Executes the callback function."""
        if dispatcher.use_context:
            self.callback(CallbackContext.from_job(self, dispatcher))
        else:
            self.callback(dispatcher.bot, self)

    def schedule_removal(self):
        """
        Schedules this job for removal from the ``JobQueue``. It will be removed without executing
        its callback function again.

        """
        self._remove.set()

    @property
    def removed(self):
        """:obj:`bool`: Whether this job is due to be removed."""
        return self._remove.is_set()

    @property
    def enabled(self):
        """:obj:`bool`: Whether this job is enabled."""
        return self._enabled.is_set()

    @enabled.setter
    def enabled(self, status):
        if status:
            self._enabled.set()
        else:
            self._enabled.clear()

    @property
    def interval(self):
        """
        :obj:`int` | :obj:`float` | :obj:`datetime.timedelta`: Optional. The interval in which the
            job will run.

        """
        return self._interval

    @interval.setter
    def interval(self, interval):
        if interval is None and self.repeat:
            raise ValueError(
                "The 'interval' can not be 'None' when 'repeat' is set to 'True'"
            )

        if not (interval is None or isinstance(interval,
                                               (Number, datetime.timedelta))):
            raise ValueError(
                "The 'interval' must be of type 'datetime.timedelta',"
                " 'int' or 'float'")

        self._interval = interval

    @property
    def interval_seconds(self):
        """:obj:`int`: The interval for this job in seconds."""
        interval = self.interval
        if isinstance(interval, datetime.timedelta):
            return interval.total_seconds()
        else:
            return interval

    @property
    def repeat(self):
        """:obj:`bool`: Optional. If this job should periodically execute its callback function."""
        return self._repeat

    @repeat.setter
    def repeat(self, repeat):
        if self.interval is None and repeat:
            raise ValueError(
                "'repeat' can not be set to 'True' when no 'interval' is set")
        self._repeat = repeat

    @property
    def days(self):
        """Tuple[:obj:`int`]: Optional. Defines on which days of the week the job should run."""
        return self._days

    @days.setter
    def days(self, days):
        if not isinstance(days, tuple):
            raise ValueError("The 'days' argument should be of type 'tuple'")

        if not all(isinstance(day, int) for day in days):
            raise ValueError(
                "The elements of the 'days' argument should be of type 'int'")

        if not all(0 <= day <= 6 for day in days):
            raise ValueError(
                "The elements of the 'days' argument should be from 0 up to and "
                "including 6")

        self._days = days

    @property
    def job_queue(self):
        """:class:`telegram.ext.JobQueue`: Optional. The ``JobQueue`` this job belongs to."""
        return self._job_queue

    @job_queue.setter
    def job_queue(self, job_queue):
        # Property setter for backward compatibility with JobQueue.put()
        if not self._job_queue:
            self._job_queue = weakref.proxy(job_queue)
        else:
            raise RuntimeError(
                "The 'job_queue' attribute can only be set once.")

    def __lt__(self, other):
        return False
Beispiel #60
0
class BookingApi:
    ENDPOINTS = {"countries": "list", "hotels": "list"}

    def __init__(self, login, password, version):
        major_minor = version.split(".")
        assert len(major_minor) == 2
        assert int(major_minor[0]) >= 2
        assert 0 <= int(major_minor[1]) <= 4

        self._event = Event()
        self._event.set()
        self._timeout = 5 * 60  # in seconds
        self._login = login
        self._password = password
        self._base_url = f"https://distribution-xml.booking.com/{version}/json"
        self._set_endpoints()

    @sleep_and_retry
    @limits(calls=LIMIT_REQUESTS_PER_MINUTE, period=60)
    def call_endpoint(self, endpoint, **params):
        self._event.wait()
        try:
            attempts = ATTEMPTS_COUNT
            while attempts:
                attempts -= 1
                response = None
                try:
                    response = requests.get(f"{self._base_url}/{endpoint}",
                                            auth=(self._login, self._password),
                                            params=params,
                                            timeout=self._timeout)
                except requests.exceptions.ReadTimeout:
                    logging.exception("Timeout error.")
                    continue
                if response.status_code == 200:
                    data = response.json()
                    return data["result"]
                else:
                    self._handle_errors(response)
            raise AttemptsSpentError(f"{ATTEMPTS_COUNT} attempts were spent.")
        except Exception as e:
            if not self._event.is_set():
                self._event.set()
            raise e

    def _handle_errors(self, response):
        error_message = ""
        data = response.json()
        try:
            error_message = ",".join(x["message"] for x in data["errors"])
        except KeyError:
            error_message = data

        if response.status_code == 429:
            self._event.clear()
            wait_seconds = randint(*MINMAX_LIMIT_WAIT_AFTER_429_ERROR_SECONDS)
            logging.warning(
                f"Http error {response.status_code}: {error_message}. "
                f"It waits {wait_seconds} seconds and tries again.")
            sleep(wait_seconds)
            self._event.set()
        else:
            raise HTTPError(
                f"Http error with code {response.status_code}: {error_message}."
            )

    def _set_endpoints(self):
        for endpoint in BookingApi.ENDPOINTS:
            setattr(self, endpoint, partial(self.call_endpoint, endpoint))