class Runner(BenchmarkThread): def __init__(self, *args, **kwargs): BenchmarkThread.__init__(self, *args, **kwargs) self.num_started = count() self.num_finished = count() self.event = Event() def insert_next(self, previous_result=sentinel): if previous_result is not sentinel: if isinstance(previous_result, BaseException): log.error("Error on insert: %r", previous_result) if next(self.num_finished) >= self.num_queries: self.event.set() if next(self.num_started) <= self.num_queries: future = self.session.execute_async(self.query, self.values) future.add_callbacks(self.insert_next, self.insert_next) def run(self): self.start_profile() for _ in range(min(120, self.num_queries)): self.insert_next() self.event.wait() self.finish_profile()
class HeartbeatFuture(object): def __init__(self, connection, owner): self._exception = None self._event = Event() self.connection = connection self.owner = owner log.debug("Sending options message heartbeat on idle connection (%s) %s", id(connection), connection.host) with connection.lock: if connection.in_flight < connection.max_request_id: connection.in_flight += 1 connection.send_msg(OptionsMessage(), connection.get_request_id(), self._options_callback) else: self._exception = Exception("Failed to send heartbeat because connection 'in_flight' exceeds threshold") self._event.set() def wait(self, timeout): self._event.wait(timeout) if self._event.is_set(): if self._exception: raise self._exception else: raise OperationTimedOut() def _options_callback(self, response): if not isinstance(response, SupportedMessage): if isinstance(response, ConnectionException): self._exception = response else: self._exception = ConnectionException( "Received unexpected response to OptionsMessage: %s" % (response,) ) log.debug("Received options response on connection (%s) from %s", id(self.connection), self.connection.host) self._event.set()
class Actor: def __init__(self): self._mailbox = Queue() def send(self, msg): self._mailbox.put(msg) def recv(self): msg = self._mailbox.get() if msg is ActorExit: raise ActorExit() return msg def start(self): self._terminated = Event() t = Thread(target=self._bootstrap) t.daemon = True t.start() def _bootstrap(self): try: self.run() except ActorExit: pass finally: self._terminated.set() def join(self): self._terminated.wait() def run(self): while True: msg = self.recv()
def test_closes_if_not_hit(): try: da = DummyHandler() fa = Event() sa = RawServer(fa, 2, 2) loop(sa) sl(sa, da, beginport + 14) sleep(1) db = DummyHandler() fb = Event() sb = RawServer(fb, 100, 100) loop(sb) sl(sb, db, beginport + 13) sleep(.5) sa.start_connection(('127.0.0.1', beginport + 13)) sleep(1) assert da.external_made == [] assert da.data_in == [] assert da.lost == [] assert len(db.external_made) == 1 del db.external_made[:] assert db.data_in == [] assert db.lost == [] sleep(3.1) assert len(da.lost) == 1 assert len(db.lost) == 1 finally: fa.set() fb.set()
class ResponseWaiter(object): def __init__(self, connection, num_responses): self.connection = connection self.pending = num_responses self.error = None self.responses = [None] * num_responses self.event = Event() def got_response(self, response, index): with self.connection.lock: self.connection.in_flight -= 1 if isinstance(response, Exception): self.error = response self.event.set() else: self.responses[index] = response self.pending -= 1 if not self.pending: self.event.set() def deliver(self, timeout=None): self.event.wait(timeout) if self.error: raise self.error elif not self.event.is_set(): raise OperationTimedOut() else: return self.responses
class Dotter(object): def __init__(self,delay=100,symbol='.'): self.event=Event() self.delay=delay self.symbol=symbol self.status=False def __loop(self): while not self.event.is_set(): stdout.write(self.symbol) stdout.flush() sleep(self.delay/1000) def start(self): if not self.status: self.event.clear() Thread(target=self.__loop).start() self.status=True def stop(self,newLine=True): if self.status: self.event.set() if newLine: stdout.write('\n') self.status=False def set(self,delay=None,symbol=None): if delay!=None: self.delay=delay if symbol!=None: self.symbol=symbol if self.status: self.stop(False) self.start()
class StoppableQThread(QtCore.QThread): """ Base class for QThreads which require the ability to be stopped by a thread-safe method call """ def __init__(self, parent=None): self._should_stop = Event() self._should_stop.clear() super(StoppableQThread, self).__init__(parent) def join(self, timeout=0): """ Joins the current thread and forces it to stop after the timeout if necessary :param timeout: Timeout duration in seconds """ self._should_stop.wait(timeout) if not self.should_stop(): self.stop() super(StoppableQThread, self).wait() def stop(self): self._should_stop.set() def should_stop(self): return self._should_stop.is_set() def __repr__(self): return "<%s(should_stop=%s)>" % (self.__class__.__name__, self.should_stop())
class DeferredResponse( object ): """ A deferred that resolves to a response from TSServer. """ def __init__( self, timeout = RESPONSE_TIMEOUT_SECONDS ): self._event = Event() self._message = None self._timeout = timeout def resolve( self, message ): self._message = message self._event.set() def result( self ): self._event.wait( timeout = self._timeout ) if not self._event.isSet(): raise RuntimeError( 'Response Timeout' ) message = self._message if not message[ 'success' ]: raise RuntimeError( message[ 'message' ] ) if 'body' in message: return self._message[ 'body' ]
def test_ask_shutdown(self): q = queue.Queue() done = Event() done.set() channel = controller.Channel(q, done) with tutils.raises(Kill): channel.ask("test", Mock(name="test_ask_shutdown"))
class TestInterruptibleDecorator(TestCase): def setUp(self): self.quit_condition = False self.thread_started = Event() @interruptible def never_ending(self, cancellation_point): self.thread_started.set() while True: time.sleep(0.01) cancellation_point() def test_interruptible_decorator(self): """ Tests for the @interruptible decorator. """ thread = Thread(target=self.never_ending, args=( lambda: _cancellation_point(lambda: self.quit_condition),)) thread.start() # Wait until thread comes to live self.thread_started.wait() # Ask to it to quit within 20ms self.quit_condition = True time.sleep(0.02) # Thread is finished self.assertFalse(thread.is_alive())
class ResponseEvent: """Event which is fired when the response is returned for a request. For each request sent this event is created. An application can wait for the event to create a blocking request. """ def __init__(self): self.__evt = Event() def waiting(self): return not self.__evt.isSet() def waitForResponse(self, timeOut=None): """blocks until the response arrived or timeout is reached.""" self.__evt.wait(timeOut) if self.waiting(): raise Timeout() else: if self.response["error"]: raise Exception(self.response["error"]) else: return self.response["result"] def handleResponse(self, resp): self.response = resp self.__evt.set()
def test_delayed_body_read_timeout(self): timed_out = Event() def socket_handler(listener): sock = listener.accept()[0] buf = b'' body = 'Hi' while not buf.endswith(b'\r\n\r\n'): buf = sock.recv(65536) sock.send(('HTTP/1.1 200 OK\r\n' 'Content-Type: text/plain\r\n' 'Content-Length: %d\r\n' '\r\n' % len(body)).encode('utf-8')) timed_out.wait() sock.send(body.encode('utf-8')) sock.close() self._start_server(socket_handler) pool = HTTPConnectionPool(self.host, self.port) response = yield From(pool.urlopen('GET', '/', retries=0, preload_content=False, timeout=Timeout(connect=1, read=0.1))) try: self.aioAssertRaises(ReadTimeoutError, response.read) finally: timed_out.set()
class Queue(list): def __init__(self): super(Queue, self).__init__() self._lock = Lock() self._fill = Event() def put(self, obj): with self._lock: self.append(obj) self._fill.set() def get(self, block=True): with self._lock: if len(self) == 0: self._fill.clear() if not self._fill.isSet(): if block: self._fill.wait() else: return None with self._lock: return self.pop(0) def delete(self, index): if 0 <= index < len(self): with self._lock: del self[index] def remove(self, element): if element in self: with self._lock: del self[self.index(element)]
def _score_scenes(self, mov, scene_list, score_obj, analysis_budget): '''Get all the scenes scored. Results stored in score_obj ''' frame_q = Queue.Queue(maxsize=100) halter = Event() threads = [ Thread(target=self._frame_extractor, args=(mov, scene_list, frame_q, halter))] for i in range(options.workers): threads.append(Thread(target=self._worker, args=(score_obj, frame_q, halter))) for t in threads: t.daemon = True t.start() # Run the frame scoring until we are out of time if not halter.wait(analysis_budget): _log.info('Out of time sampling frames') halter.set() for t in threads: t.join() _log.info('Finished scoring scenes')
class ThreadedRunner(Runnable): def __init__(self, runnable): self._runnable = runnable self._notifier = Event() self._result = None self._error = None self._traceback = None self._thread = None def run(self): try: self._result = self._runnable() except: self._error, self._traceback = sys.exc_info()[1:] self._notifier.set() __call__ = run def run_in_thread(self, timeout): self._thread = Thread(self, name=TIMEOUT_THREAD_NAME) self._thread.setDaemon(True) self._thread.start() self._notifier.wait(timeout) return self._notifier.isSet() def get_result(self): if self._error: raise self._error, None, self._traceback return self._result def stop_thread(self): self._thread.stop()
class InterruptableEvent(object): """Event class for for Python v2.7 which behaves more like Python v3.2 threading.Event and allows signals to interrupt waits""" def __init__(self): self.__event = Event() def is_set(self): return self.__event.is_set() def set(self): self.__event.set() def clear(self): self.__event.clear() def wait(self, timeout=None): # infinite if timeout is None: # event with timeout is interruptable while not self.__event.wait(60): pass return True # finite else: # underlying Event will perform timeout argument validation return self.__event.wait(timeout)
class Task(object): def __init__(self, name, start_time, calc_next_time, func): """ Initialize a Task. Arguments: name - Name of task. start_time - First time for task to run calc_next_time - Function to calculate the time of next run, gets one argument, the last run time as a datetime. Returns None when task should no longer be run func - A function to run """ self.name = name self.start_time = start_time self.scheduled_time = start_time self.calc_next_time = calc_next_time self.func = func self.halt_flag = Event() def run(self): logging.debug("Running %s task, scheduled at: %s" % (self.name, self.scheduled_time,)) if not self.halt_flag.isSet(): try: try: self.func() except: raise finally: self.scheduled_time = self.calc_next_time(self.scheduled_time) logging.debug("Scheduled next run of %s for: %s" % (self.name, self.scheduled_time,)) def halt(self): self.halt_flag.set()
def FromWatchdogWithSubservers_test( self ): all_servers_are_running = Event() def KeepServerAliveInAnotherThread(): while not all_servers_are_running.is_set(): try: self.GetRequest( 'ready' ) except requests.exceptions.ConnectionError: pass finally: time.sleep( 0.1 ) self.Start( idle_suicide_seconds = 2, check_interval_seconds = 1 ) StartThread( KeepServerAliveInAnotherThread ) try: filetypes = [ 'cs', 'go', 'java', 'javascript', 'typescript', 'rust' ] for filetype in filetypes: self.StartSubserverForFiletype( filetype ) self.AssertServersAreRunning() finally: all_servers_are_running.set() self.AssertServersShutDown( timeout = SUBSERVER_SHUTDOWN_TIMEOUT + 10 ) self.AssertLogfilesAreRemoved()
def testMultipleLoad(self): """ In DropBox, the loading of multiple CLIs seems to lead to the wrong context being assigned to some controls. See #4749 """ import random from threading import Thread, Event event = Event() class T(Thread): def run(self, *args): pause = random.random() event.wait(pause) self.cli = CLI() self.cli.loadplugins() self.con = self.cli.controls["admin"] self.cmp = self.con.ctx threads = [T() for x in range(20)] for t in threads: t.start() event.set() for t in threads: t.join() assert len(threads) == len(set([t.cli for t in threads])) assert len(threads) == len(set([t.con for t in threads])) assert len(threads) == len(set([t.cmp for t in threads]))
class WaitableTimer(Timer): def __init__(self, timeout, callback): Timer.__init__(self, timeout, callback) self.callback = callback self.event = Event() self.final_exception = None def finish(self, time_now): try: finished = Timer.finish(self, time_now) if finished: self.event.set() return True return False except Exception as e: self.final_exception = e self.event.set() return True def wait(self, timeout=None): self.event.wait(timeout) if self.final_exception: raise self.final_exception
class RepeatTimer(Thread): def __init__(self, interval, function, stop_on_exception=True, args=[], kwargs={}): Thread.__init__(self) self.interval = interval self.function = function self.args = args self.kwargs = kwargs self.stop_on_exception = stop_on_exception self.finished = Event() def stop(self): self.done = True self.finished.set() def run(self): self.done = False while True: self.finished.wait(self.interval) if self.done: self.finished.clear() break if not self.finished.is_set(): try: self.function(*self.args, **self.kwargs) except Exception, e: if self.stop_on_exception: self.finished.clear() raise # XXX Not strictly for auth'ing, think of something better log.exception('Login Fail') self.finished.clear()
class CheckForUpdates(Thread): INTERVAL = 24*60*60 # seconds daemon = True def __init__(self, parent): Thread.__init__(self) self.shutdown_event = Event() self.signal = Signal(parent) def run(self): while not self.shutdown_event.is_set(): calibre_update_version = NO_CALIBRE_UPDATE plugins_update_found = 0 try: version = get_newest_version() if version[:2] > numeric_version[:2]: calibre_update_version = version except Exception as e: prints('Failed to check for calibre update:', as_unicode(e)) try: update_plugins = get_plugin_updates_available(raise_error=True) if update_plugins is not None: plugins_update_found = len(update_plugins) except Exception as e: prints('Failed to check for plugin update:', as_unicode(e)) if calibre_update_version != NO_CALIBRE_UPDATE or plugins_update_found > 0: self.signal.update_found.emit(calibre_update_version, plugins_update_found) self.shutdown_event.wait(self.INTERVAL) def shutdown(self): self.shutdown_event.set()
def test_concurrent_rendering(): '''Best-effort testing that concurrent multi-threaded rendering works. The test has no guarantees around being deterministic, but if it fails you know something is wrong with concurrent rendering. If it passes, things are probably working.''' err = None def func(sim, event): event.wait() sim.data.qpos[:] = 0.0 sim.forward() img1 = sim.render(width=40, height=40, camera_name="camera1") img2 = sim.render(width=40, height=40, camera_name="camera2") try: assert np.sum(img1[:]) == 23255 assert np.sum(img2[:]) == 12007 except Exception as e: nonlocal err err = e model = load_model_from_xml(BASIC_MODEL_XML) sim = MjSim(model) sim.render(100, 100) event = Event() threads = [] for _ in range(100): thread = Thread(target=func, args=(sim, event)) threads.append(thread) thread.start() event.set() for thread in threads: thread.join() assert err is None, "Exception: %s" % (str(err))
class GarbageCollectorThread(Thread): """Thread in which garbage collection actually happens.""" def __init__(self, gc): super(GarbageCollectorThread, self).__init__() self.gc = gc self.daemon = True self.pid = getpid() self.ready = Event() def run(self): s = self.gc.context.socket(zmq.PULL) s.linger = 0 s.bind(self.gc.url) self.ready.set() while True: # detect fork if getpid is None or getpid() != self.pid: return msg = s.recv() if msg == b'DIE': break fmt = 'L' if len(msg) == 4 else 'Q' key = struct.unpack(fmt, msg)[0] tup = self.gc.refs.pop(key, None) if tup and tup.event: tup.event.set() del tup s.close()
def request_token_spotty(spotty, use_creds=True): '''request token by using the spotty binary''' token_info = None if spotty.playback_supported: try: args = ["-t", "--client-id", CLIENTID, "--scope", ",".join(SCOPE), "-n", "temp-spotty"] done = Event() spotty = spotty.run_spotty(arguments=args, use_creds=use_creds) watcher = Thread(target=kill_on_timeout, args=(done, 5, spotty)) watcher.daemon = True watcher.start() stdout, stderr = spotty.communicate() done.set() result = None log_msg("request_token_spotty stdout: %s" % stdout) for line in stdout.split(): line = line.strip() if line.startswith("{\"accessToken\""): result = json.loads(line) # transform token info to spotipy compatible format if result: token_info = {} token_info["access_token"] = result["accessToken"] token_info["expires_in"] = result["expiresIn"] token_info["token_type"] = result["tokenType"] token_info["scope"] = ' '.join(result["scope"]) token_info['expires_at'] = int(time.time()) + token_info['expires_in'] token_info['refresh_token'] = result["accessToken"] except Exception as exc: log_exception(__name__, exc) return token_info
class StoppableThreadWithResult(Thread): """Thread class with a stop() method. The thread itself has to check regularly for the stopped() condition.""" def __init__(self, group=None, target=None, name=None, args=(), kwargs=None, verbose=None): super(StoppableThreadWithResult, self).__init__(group=group, target=target, name=name, args=args, kwargs=kwargs, verbose=verbose) self._stop = Event() def stop(self): self._stop.set() self._Thread__stop() def stopped(self): return self._stop.isSet() def run(self): if self._Thread__target is not None: self._return = self._Thread__target(*self._Thread__args, **self._Thread__kwargs) def join(self, timeout=None): Thread.join(self, timeout=None) return self._return
class TimerWithResume(object): def __init__(self, status_subject, refresh_interval): self.status_subject = status_subject self.abort = Event() self.refresh_interval = refresh_interval def perform(self): while not self.abort.isSet(): self.status_subject.build_status() self.abort.wait(self.refresh_interval) def stop(self): self.abort.set() def start(self): self.thread = Thread(target=self.perform) self.thread.daemon = True self.thread.start() def resume(self): self.thread.join() self.abort.clear() self.start() def set_refresh_interval(self, new_interval): self.refresh_interval = new_interval
class TestInterruptible(unittest.TestCase): """ Tests for interrupting cooperative threads """ def test_interruptible_decorator(self): """ Tests for the @interruptible decorator. """ self.quit_condition = False cancellation_point = lambda: _cancellation_point( lambda: self.quit_condition) self.thread_started = Event() @interruptible def never_ending(cancellation_point): self.thread_started.set() while True: time.sleep(0.1) cancellation_point() thread = Thread(target=never_ending, args=(cancellation_point, )) thread.start() self.thread_started.wait() self.quit_condition = True countdown = 10 while thread.is_alive() and countdown > 0: time.sleep(0.1) countdown -= 1 self.assertFalse(thread.is_alive())
class NonSubscribeListener(object): def __init__(self): self.result = None self.status = None self.done_event = Event() def callback(self, result, status): self.result = result self.status = status self.done_event.set() def pn_await(self, timeout=5): """ Returns False if a timeout happened, otherwise True""" return self.done_event.wait(timeout) def await_result(self, timeout=5): self.pn_await(timeout) return self.result def await_result_and_reset(self, timeout=5): self.pn_await(timeout) cp = copy.copy(self.result) self.reset() return cp def reset(self): self.result = None self.status = None self.done_event.clear()
class Worker(Thread): def __init__(self): Thread.__init__(self) self.trigger = Event() self.uri = None self.sonos = None self.daemon = True self.lock = Lock() def start_url(self, a_group, uri): with self.lock: self.sonos = a_group self.uri = uri self.trigger.set() def run(self): while True: self.trigger.wait() if self.sonos is None: # take this as a sign to shut down gracefully break with self.lock: uri = self.uri self.uri = None self.trigger.clear() result = self.sonos.play_uri( uri) print result
class WebLiveBackend(object): """ Backend for interacting with the WebLive service """ client = None URL = os.environ.get('SOFTWARE_CENTER_WEBLIVE_HOST', 'https://weblive.stgraber.org/weblive/json') def __init__(self): self.weblive = WebLive(self.URL, True) self.available_servers = [] for client in (WebLiveClientX2GO, WebLiveClientQTNX): if client.is_supported(): self.client = client() break self._ready = Event() @property def ready(self): """ Return true if data from the remote server was loaded """ return self.client and self._ready.is_set() def query_available(self): """ Get all the available data from WebLive """ self._ready.clear() servers = self.weblive.list_everything() self._ready.set() return servers def query_available_async(self): """ Call query_available in a thread and set self.ready """ def _query_available_helper(): self.available_servers = self.query_available() p = Thread(target=_query_available_helper) p.start() def is_pkgname_available_on_server(self, pkgname, serverid=None): """Check if the package is available (on all servers or on 'serverid') """ for server in self.available_servers: if not serverid or server.name == serverid: for pkg in server.packages: if pkg.pkgname == pkgname: return True return False def get_servers_for_pkgname(self, pkgname): """ Return a list of servers having a given package """ servers = [] for server in self.available_servers: # No point in returning a server that's full if server.current_users >= server.userlimit: continue for pkg in server.packages: if pkg.pkgname == pkgname: servers.append(server) return servers def create_automatic_user_and_run_session(self, serverid, session="desktop", wait=False): """ Create a user on 'serverid' and start the session """ # Use the boot_id to get a temporary unique identifier # (till next reboot) if os.path.exists('/proc/sys/kernel/random/boot_id'): uuid = open('/proc/sys/kernel/random/boot_id', 'r').read().strip().replace('-', '') random.seed(uuid) # Generate a 20 characters string based on the boot_id identifier = ''.join(random.choice(string.ascii_lowercase) for x in range(20)) # Use the current username as the GECOS on the server # if it's invalid (by weblive's standard), use "WebLive user" instead fullname = str(os.environ.get('USER', 'WebLive user')) if not re.match("^[A-Za-z0-9 ]*$", fullname) or len(fullname) == 0: fullname = 'WebLive user' # Send the user's locale so it's automatically selected when connecting locale = os.environ.get("LANG", "None").replace("UTF-8", "utf8") # Create the user and retrieve host and port of the target server connection = self.weblive.create_user(serverid, identifier, fullname, identifier, session, locale) # Connect using x2go or fallback to qtnx if not available if (self.client): self.client.start_session(connection[0], connection[1], session, identifier, identifier, wait) else: raise IOError("No remote desktop client available.")
class MyModule(MumoModule): def __init__(self, name, manager, configuration=None): MumoModule.__init__(self, name, manager, configuration) self.estarted = Event() self.estopped = Event() self.econnected = Event() self.edisconnected = Event() self.emeta = Event() self.econtext = Event() self.eserver = Event() def onStart(self): self.estarted.set() def onStop(self): self.estopped.set() def connected(self): man = self.manager() man.subscribeMetaCallbacks(self) man.subscribeServerCallbacks(self) man.subscribeContextCallbacks(self) self.econnected.set() def disconnected(self): self.edisconnected.set() def metaCallMe(self, arg1, arg2): if arg1 == "arg1" and arg2 == "arg2": self.emeta.set() def contextCallMe(self, server, arg1, arg2): if arg1 == "arg1" and arg2 == "arg2": self.econtext.set() def serverCallMe(self, server, arg1, arg2): if arg1 == "arg1" and arg2 == "arg2": self.eserver.set()
class SteamEngine: """ implement routing of mqtt msgs to servers """ def __init__(self, conf, logger): self.conf = conf self.logger = logger self.my_clientid = conf['my_client_id'] self.short_circuit = conf.get('allow_short_circuit', allow_short_circuit) self.task_timeout = conf.get('network_timeout', network_timeout) Topic.encode = conf.get('topic_encode', topic_encode) # service_routing is keyed with subscription topic (mqtt flavoured re), values is the queue to send message to self.running = True self.service_routing = {} self.task_id = 0 self.tasks = {} # create thread to handle backflow of answers self.answer_task = Thread(target=self.run_answers) self.answer_task.name = "answer_th" self.answer_task.daemon = True # create thread to polling self.poll_interval = int(self.conf.get('pollinterval', 3600)) self.poll_task = Thread(target=self.run_poll) self.poll_task.name = "poll_th" self.poll_task.daemon = True # queue to receive service answers self.service_answers = queue.Queue() self.service_answer_topic = Topic("+", self.my_clientid, "+", "a") # start runners self.answer_task.start() self.poll_task.start() # connect to broker self.mqtt_ready = Event() self.mqtt_connect() # start service self.load_and_run_services() def shutdown(self): self.running = False self.mqtt_con.loop_stop() self.mqtt_con.disconnect() for service in self.conf['services']: self.services[service].shutdown() def poll_for_status(self): for service in self.services: if getattr(self.services[service], 'poll', None) \ and inspect.ismethod(self.services[service].poll): self.logger.debug("polling %s", service) self.services[service].poll() def run_poll(self): self.logger.info("Steamlink run_poll start") polltime = time.time() - self.poll_interval + 3 # force poll 3 sec after boot while self.running: waitt = max(polltime + self.poll_interval - time.time(), 0) if waitt > 0: time.sleep(waitt) else: polltime = time.time() self.poll_for_status() def check_sv(self, sv, avail_services): err = False if not sv['type'] in ['server', 'channel']: self.logger.error("unknown service type '%s'", sv['type']) err = True if not sv['class'] in avail_services: self.logger.error("unknown service class '%s'", sv['class']) err = True if err: self.logger.error("correct config and re-run") sys.exit(1) def get_available_services(self): # find available channels by searching all classes for a 'type' class variable # with a contect of server or channel avail_services = {} for name, obj in inspect.getmembers(sys.modules[__name__]): if inspect.isclass(obj) and getattr(obj, "type", None) in ['server', 'channel']: avail_services[name] = obj return avail_services def load_and_run_services(self): self.services = {} avail_services = self.get_available_services() for service in self.conf['services']: sv = self.conf['services'][service] self.check_sv(sv, avail_services) self.services[service] = avail_services[sv['class']](service, self, self.logger, sv) for service in self.conf['services']: self.services[service].start() def run_answers(self): self.logger.info("Steamlink run_answers start") while self.running: answer = self.service_answers.get() topic = Topic(answer['_topic']) topic.direction = 'a' topic.client_id = answer['_origin'] self.logger.debug('run_answer publish %s %s', topic, str(answer)[:70]+"...") self.mqtt_con.publish(str(topic), dumps(answer)) self.service_answers.task_done() def return_answer(self, pkt): tp = Topic(pkt['_topic']) if self.short_circuit \ and tp.client_id == self.my_clientid \ and tp.task_id in self.tasks: self.tasks[task_id].answer = pkt self.tasks[task_id].set() else: self.service_answers.put(pkt) def mqtt_connect(self, reconnect=False): self.mqtt_ready.clear() if reconnect: self.mqtt_con.loop_stop() self.mqtt_con.disconnect() self.mqtt_con.reinitialise(client_id=self.conf['MQTT_CLIENTID']) else: self.mqtt_con = mqtt.Client(client_id=self.conf['MQTT_CLIENTID']) # TODO: figure out cert in conf self.mqtt_con.tls_set(self.conf['MQTT_CERT']), self.mqtt_con.username_pw_set(self.conf['MQTT_USERNAME'], self.conf['MQTT_PASSWORD']) self.mqtt_con.tls_insecure_set(False) self.mqtt_con.on_connect = self.mqtt_on_connect self.mqtt_con.on_disconnect = self.mqtt_on_disconnect self.mqtt_con.on_message = self.mqtt_on_message self.logger.info("connecting to MQTT") self.mqtt_con.connect(self.conf['MQTT_SERVER'], self.conf['MQTT_PORT'], 60) self.mqtt_con.loop_start() self.mqtt_con._thread.name = "mqtt_th" # N.B. def mqtt_on_disconnect(self, client, userdata, flags): self.mqtt_ready.clear() self.logger.warn("disconnected from MQTT boker") def mqtt_on_connect(self, client, userdata, flags, rc): if self.mqtt_con is None: self.mqtt_connect(reconnect=True) return self.logger.warn("connected to MQTT boker,code "+str(rc)) # subscribe to all answers for my_client_id self.subscribe(self.service_answer_topic) self.mqtt_ready.set() for service_topic in self.service_routing: self.subscribe(service_topic) def subscribe(self, service_topic): self.mqtt_con.subscribe(str(service_topic)) self.logger.debug( "subscribe %s" % service_topic) def mqtt_on_message(self, client, userdata, msg): srv_type = None if mqtt.topic_matches_sub(str(self.service_answer_topic), msg.topic): srv_type = "answer" else: for service_sub in self.service_routing: if mqtt.topic_matches_sub(service_sub, msg.topic): srv_type = self.service_routing[service_sub].typ break if not srv_type: self.logger.error("no registered service for msg topic %s", msg.topic) return if srv_type == 'channel': self.service_routing[service_sub].queue.put(msg) elif srv_type in ['server', 'answer']: tp = Topic(msg.topic) assert tp.client_id == self.my_clientid pkt = json.loads(msg.payload.decode('utf-8')) pkt['_topic'] = msg.topic self.logger.debug("mqtt_on_message %s %s", msg.topic, str(pkt)[:70]+"...") if srv_type is 'answer': self.receive_answer(pkt, int(tp.task_id)) elif srv_type is 'server': self.dispatch_service(tp.service, pkt) def receive_answer(self, pkt, task_id): if not task_id in self.tasks: self.logger.error("received answer for unknown task id %s", task_id) else: self.tasks[task_id].answer = pkt self.tasks[task_id].set() def dispatch_service(self, msg_service, pkt): msg_service_topic = str(Topic(msg_service, self.my_clientid, "+", "q")) if not msg_service_topic in self.service_routing: self.logger.error("no service '%s' defined", msg_service) return self.service_routing[msg_service_topic].queue.put(pkt) return def publish(self, topic, pkt, as_json=True, retain=False): self.mqtt_ready.wait() self.logger.debug("publish: %s %s", topic, pkt) if as_json: opkt = dumps(pkt) else: opkt = pkt self.mqtt_con.publish(str(topic), opkt, retain=retain) def register_service(self, service_topic, service_queue, service_type): if service_topic in self.service_routing: logger.error("service %s already defined", service_topic) else: self.logger.debug("register service %s", service_topic) self.service_routing[service_topic] = Srv(service_queue, service_type) if self.mqtt_ready.is_set(): self.subscribe(service_topic) def ask_question(self, service, client_id, pkt): if client_id == None: client_id = self.conf['default_clients'].get(service, self.my_clientid) task_id = self.new_task_id() self.tasks[task_id] = Event() topic = Topic(service, client_id, task_id, "q") pkt['_origin'] = self.my_clientid if self.short_circuit and client_id == self.my_clientid: pkt['_topic'] = str(topic) self.dispatch_service(service, pkt) else: self.publish(topic, pkt) if self.tasks[task_id].wait(self.task_timeout): answer = self.tasks[task_id].answer del self.tasks[task_id] del answer['_topic'] else: return Err('TIMEOUT', 'network request timed out').d() return answer['res'] def new_task_id(self): self.task_id +=1 return self.task_id
class AgentLogicBase: def __init__(self, config): logging.debug("AgentLogicBase:: __init__() entered") self.wait_stop = Event() self.heartBitRate = config.getint("general", "heart_beat_rate") self.userCheckRate = config.getint("general", "report_user_rate") self.appRefreshRate = config.getint("general", "report_application_rate") self.disksRefreshRate = config.getint("general", "report_disk_usage") self.activeUser = "" self.vio = VirtIoChannel(config.get("virtio", "device")) self.dr = None self.commandHandler = None def run(self): logging.debug("AgentLogicBase:: run() entered") thread.start_new_thread(self.doListen, ()) thread.start_new_thread(self.doWork, ()) # Yuck! It's seem that Python block all signals when executing # a "real" code. So there is no way just to sit and wait (with # no timeout). # Try breaking out from this code snippet: # $ python -c "import threading; threading.Event().wait()" while not self.wait_stop.isSet(): self.wait_stop.wait(1) def stop(self): logging.debug("AgentLogicBase:: baseStop() entered") self.wait_stop.set() def doWork(self): logging.debug("AgentLogicBase:: doWork() entered") self.sendInfo() self.sendUserInfo() self.sendAppList() counter = 0 hbsecs = self.heartBitRate appsecs = self.appRefreshRate disksecs = self.disksRefreshRate usersecs = self.userCheckRate try: while not self.wait_stop.isSet(): counter +=1 hbsecs -= 1 if hbsecs <= 0: self.vio.write('heartbeat', {'free-ram' : self.dr.getAvailableRAM(), 'memory-stat' : self.dr.getMemoryStats(), 'totalrdpconnections' : self.dr.getTotalRDPConnections()}) hbsecs = self.heartBitRate usersecs -=1 if usersecs <=0: self.sendUserInfo() usersecs = self.userCheckRate appsecs -= 1 if appsecs <= 0: self.sendAppList() self.sendInfo() appsecs = self.appRefreshRate disksecs -= 1 if disksecs <= 0: self.sendDisksUsages() disksecs = self.disksRefreshRate time.sleep(1) logging.debug("AgentLogicBase:: doWork() exiting") except: logging.exception("AgentLogicBase::doWork") def doListen(self): logging.debug("AgentLogicBase::doListen() - entered") if self.commandHandler == None: logging.debug("AgentLogicBase::doListen() - no commandHandler ... exiting doListen thread") return while not self.wait_stop.isSet(): try: logging.debug("AgentLogicBase::doListen() - in loop before vio.read") cmd, args = self.vio.read() if cmd: self.parseCommand(cmd, args) except: logging.exception('Error while reading the virtio-serial channel.') logging.debug("AgentLogicBase::doListen() - exiting") def parseCommand(self, command, args): logging.info("Received an external command: %s..." % (command)) if command == 'lock-screen': self.commandHandler.lock_screen() elif command == 'log-off': self.commandHandler.logoff() elif command == 'shutdown': try: timeout = int(args['timeout']) except: timeout = 0 try: msg = args['message'] except: msg = 'System is going down' logging.info("Shutting down (timeout = %d, message = '%s')" % (timeout, msg)) self.commandHandler.shutdown(timeout, msg) elif command == 'login': username = args['username'].encode('utf8') password = args['password'].encode('utf8') credentials = struct.pack('>I%ds%ds' % (len(username), len(password) + 1), len(username), username, password) logging.debug("User log-in (credentials = %s)" % (safe_creds_repr(credentials))) self.commandHandler.login(credentials) elif command == 'refresh': self.sendUserInfo(True) self.sendAppList() self.sendInfo() self.sendDisksUsages() elif command == 'echo': logging.debug("Echo: %s", args) self.vio.write('echo', args) elif command == 'hibernate': state = args.get('state', 'disk') self.commandHandler.hibernate(state) elif command == 'reboot': self.commandHandler.reboot() else: logging.error("Unknown external command: %s (%s)" % (command, args)) def sendUserInfo(self, force=False): cur_user = str(self.dr.getActiveUser()) logging.debug("AgentLogicBase::sendUserInfo - cur_user = '******'"%(cur_user)) if cur_user != self.activeUser or force: self.vio.write('active-user', { 'name' : cur_user }) self.activeUser = cur_user def sendInfo(self): self.vio.write('host-name', { 'name' : self.dr.getMachineName() }) self.vio.write('os-version', { 'version' : self.dr.getOsVersion() }) self.vio.write('network-interfaces', { 'interfaces' : self.dr.getAllNetworkInterfaces() }) def sendAppList(self): self.vio.write('applications', { 'applications' : self.dr.getApplications() }) def sendDisksUsages(self): self.vio.write('disks-usage', { 'disks' : self.dr.getDisksUsage() }) def sendMemoryStats(self): self.vio.write('memory-stats', { 'memory' : self.dr.getMemoryStats() }) def sessionLogon(self): logging.debug("AgentLogicBase::sessionLogon: user logs on the system.") cur_user = self.dr.getActiveUser() retries = 0 while (cur_user == 'None') and (retries < 5): time.sleep(1) cur_user = self.dr.getActiveUser() retries = retries + 1 self.sendUserInfo() self.vio.write('session-logon') def sessionLogoff(self): logging.debug("AgentLogicBase::sessionLogoff: user logs off from the system.") self.activeUser = '******' self.vio.write('session-logoff') self.vio.write('active-user', { 'name' : self.activeUser }) def sessionLock(self): logging.debug("AgentLogicBase::sessionLock: user locks the workstation.") self.vio.write('session-lock') def sessionUnlock(self): logging.debug("AgentLogicBase::sessionUnlock: user unlocks the workstation.") self.vio.write('session-unlock') def sessionStartup(self): logging.debug("AgentLogicBase::sessionStartup: system starts up.") self.vio.write('session-startup') def sessionShutdown(self): logging.debug("AgentLogicBase::sessionShutdown: system shuts down.") self.vio.write('session-shutdown')
def update_location(): while not should_stop.is_set(): with locker: for device in devices: position = random.choice(['latitude', 'longitude']) device[position] += random.random() / 100 time.sleep(1) @app.route("/api") def devices_list(): with locker: return jsonify(devices) @app.route('/') def root(): return render_template('index.html') if __name__ == "__main__": updater = Thread(target=update_location) updater.start() app.run(debug=True, port=8001) should_stop.set() updater.join()
class ThreadManager: """ manages the download threads, assign jobs, reconnect etc. """ def __init__(self, core): """ Constructor. """ self.pyload = core self._ = core._ self.threads = [] #: thread list self.local_threads = [] #: addon+decrypter threads self.pause = True self.reconnecting = Event() self.reconnecting.clear() self.downloaded = 0 #: number of files downloaded since last cleanup self.lock = Lock() # some operations require to fetch url info from hoster, so we caching them so it wont be done twice # contains a timestamp and will be purged after timeout self.info_cache = {} # pool of ids for online check self.result_ids = 0 # threads which are fetching hoster results self.info_results = {} # timeout for cache purge self.timestamp = 0 # pycurl.global_init(pycurl.GLOBAL_DEFAULT) for i in range(self.pyload.config.get("download", "max_downloads")): self.create_thread() def create_thread(self): """ create a download thread. """ thread = DownloadThread(self) self.threads.append(thread) def create_info_thread(self, data, pid): """ start a thread whichs fetches online status and other infos data = [ .. () .. ] """ self.timestamp = time.time() + timedelta(minutes=5).seconds InfoThread(self, data, pid) @lock def create_result_thread(self, data, add=False): """ creates a thread to fetch online status, returns result id. """ self.timestamp = time.time() + timedelta(minutes=5).seconds rid = self.result_ids self.result_ids += 1 InfoThread(self, data, rid=rid, add=add) return rid @lock def get_info_result(self, rid): """ returns result and clears it. """ self.timestamp = time.time() + timedelta(minutes=5).seconds if rid in self.info_results: data = self.info_results[rid] self.info_results[rid] = {} return data else: return {} @lock def set_info_results(self, rid, result): self.info_results[rid].update(result) def get_active_files(self): active = [ x.active for x in self.threads if x.active and isinstance(x.active, PyFile) ] for t in self.local_threads: active.extend(t.get_active_files()) return active def processing_ids(self): """ get a id list of all pyfiles processed. """ return [x.id for x in self.get_active_files()] def run(self): """ run all task which have to be done (this is for repetivive call by core) """ try: self.try_reconnect() except Exception as exc: self.pyload.log.error( self._("Reconnect Failed: {}").format(exc), exc_info=self.pyload.debug > 1, stack_info=self.pyload.debug > 2, ) self.reconnecting.clear() self.check_thread_count() try: self.assign_job() except Exception as exc: self.pyload.log.warning( "Assign job error", exc, exc_info=self.pyload.debug > 1, stack_info=self.pyload.debug > 2, ) time.sleep(0.5) self.assign_job() # it may be failed non critical so we try it again if (self.info_cache or self.info_results) and self.timestamp < time.time(): self.info_cache.clear() self.info_results.clear() self.pyload.log.debug("Cleared Result cache") # ---------------------------------------------------------------------- def try_reconnect(self): """ checks if reconnect needed. """ if not (self.pyload.config.get("reconnect", "enabled") and self.pyload.api.is_time_reconnect()): return False active = [ x.active.plugin.want_reconnect and x.active.plugin.waiting for x in self.threads if x.active ] if not (0 < active.count(True) == len(active)): return False reconnect_script = self.pyload.config.get("reconnect", "script") if not os.path.isfile(reconnect_script): self.pyload.config.set("reconnect", "enabled", False) self.pyload.log.warning(self._("Reconnect script not found!")) return self.reconnecting.set() # Do reconnect self.pyload.log.info(self._("Starting reconnect")) while [x.active.plugin.waiting for x in self.threads if x.active].count(True) != 0: time.sleep(0.25) ip = self.get_ip() self.pyload.addon_manager.before_reconnecting(ip) self.pyload.log.debug(f"Old IP: {ip}") try: subprocess.run(reconnect_script) except Exception: self.pyload.log.warning( self._("Failed executing reconnect script!")) self.pyload.config.set("reconnect", "enabled", False) self.reconnecting.clear() return time.sleep(1) ip = self.get_ip() self.pyload.addon_manager.after_reconnecting(ip) self.pyload.log.info(self._("Reconnected, new IP: {}").format(ip)) self.reconnecting.clear() def get_ip(self): """ retrieve current ip. """ services = [ ("http://icanhazip.com/", r"(\S+)"), ("http://checkip.dyndns.org/", r".*Current IP Address: (\S+)</body>.*"), ("http://ifconfig.io/ip", r"(\S+)"), ] ip = "" for i in range(10): try: sv = choice(services) ip = get_url(sv[0]) ip = re.match(sv[1], ip).group(1) break except Exception: ip = "" time.sleep(1) return ip # ---------------------------------------------------------------------- def check_thread_count(self): """ checks if there are need for increasing or reducing thread count. """ if len(self.threads) == self.pyload.config.get("download", "max_downloads"): return True elif len(self.threads) < self.pyload.config.get( "download", "max_downloads"): self.create_thread() else: free = [x for x in self.threads if not x.active] if free: free[0].put("quit") # def clean_pycurl(self): # """ # make a global curl cleanup (currently ununused) # """ # if self.processing_ids(): # return False # pycurl.global_cleanup() # pycurl.global_init(pycurl.GLOBAL_DEFAULT) # self.downloaded = 0 # self.pyload.log.debug("Cleaned up pycurl") # return True # ---------------------------------------------------------------------- def assign_job(self): """ assing a job to a thread if possible. """ if self.pause or not self.pyload.api.is_time_download(): return # if self.downloaded > 20: # if not self.clean_py_curl(): return free = [x for x in self.threads if not x.active] inuse = set([ (x.active.pluginname, self.get_limit(x)) for x in self.threads if x.active and x.active.has_plugin() and x.active.plugin.account ]) inuse = [( x[0], x[1], len([ y for y in self.threads if y.active and y.active.pluginname == x[0] ]), ) for x in inuse] onlimit = [x[0] for x in inuse if x[1] > 0 and x[2] >= x[1]] occ = sorted([ x.active.pluginname for x in self.threads if x.active and x.active.has_plugin() and not x.active.plugin.multi_dl ] + onlimit) occ = tuple(set(occ)) job = self.pyload.files.get_job(occ) if job: try: job.init_plugin() job.set_status("starting") except Exception as exc: self.pyload.log.critical(exc, exc_info=True, stack_info=self.pyload.debug > 2) job.set_status("failed") job.error = str(exc) job.release() return if job.plugin.__type__ == "downloader": space_left = (fs.free_space( self.pyload.config.get("general", "storage_folder")) >> 20) if space_left < self.pyload.config.get("general", "min_free_space"): self.pyload.log.warning( self._("Not enough space left on device")) self.pause = True if free and not self.pause: thread = free[0] # self.downloaded += 1 thread.put(job) else: # put job back if occ not in self.pyload.files.job_cache: self.pyload.files.job_cache[occ] = [] self.pyload.files.job_cache[occ].append(job.id) # check for decrypt jobs job = self.pyload.files.get_decrypt_job() if job: job.init_plugin() thread = DecrypterThread(self, job) else: thread = DecrypterThread(self, job) def get_limit(self, thread): limit = thread.active.plugin.account.get_account_data( thread.active.plugin.user)["options"].get("limit_dl", ["0"])[0] return int(limit)
class RabbitmqConnection(object): Parameters = URLParameters Connection = LayerConnection Protocol = Protocol def __init__(self, url, expiry, group_expiry, get_capacity, crypter): self.url = url self.expiry = expiry self.group_expiry = group_expiry self.get_capacity = get_capacity self.crypter = crypter self.protocols = {} self.lock = Lock() self.is_open = Event() self.parameters = self.Parameters(self.url) self.connection = self.Connection( parameters=self.parameters, on_open_callback=self.start_loop, on_close_callback=self.notify_futures, on_callback_error_callback=self.protocol_error, stop_ioloop_on_close=False, lock=self.lock, ) def run(self): self.connection.ioloop.start() def start_loop(self, connection): self.is_open.set() self.process(None, (DECLARE_DEAD_LETTERS, (), {}), Future()) def process(self, ident, method, future): if (ident in self.protocols and self.protocols[ident].amqp_channel.is_open): self.protocols[ident].resolve = future self.protocols[ident].apply(*method) return protocol = self.Protocol(self.expiry, self.group_expiry, self.get_capacity, ident, self.crypter) protocol.resolve = future amqp_channel = self.connection.channel( partial(protocol.register_channel, method), ) amqp_channel.on_callback_error_callback = protocol.protocol_error self.protocols[ident] = protocol def notify_futures(self, connection, code, msg): try: for protocol in self.protocols.values(): protocol.resolve.set_exception(ConnectionClosed()) finally: self.connection.ioloop.stop() def protocol_error(self, error): for protocol in self.protocols.values(): protocol.resolve.set_exception(error) def schedule(self, f, *args, **kwargs): if self.connection.is_closing or self.connection.is_closed: raise ConnectionClosed self.is_open.wait() future = Future() with self.lock: self.process(get_ident(), (f, args, kwargs), future) return future @property def thread_protocol(self): return self.protocols[get_ident()]
class UpdateState: """Used to hold the current state of processed updates. To retrieve an update, .poll() should be called. """ def __init__(self, polling): self._polling = polling self.handlers = [] self._updates_lock = RLock() self._updates_available = Event() self._updates = deque() # https://core.telegram.org/api/updates self._state = tl.updates.State(0, 0, datetime.now(), 0, 0) def can_poll(self): """Returns True if a call to .poll() won't lock""" return self._updates_available.is_set() def poll(self): """Polls an update or blocks until an update object is available""" if not self._polling: raise ValueError('Updates are not being polled hence not saved.') self._updates_available.wait() with self._updates_lock: update = self._updates.popleft() if not self._updates: self._updates_available.clear() if isinstance(update, Exception): raise update # Some error was set through .set_error() return update def get_polling(self): return self._polling def set_polling(self, polling): self._polling = polling if not polling: with self._updates_lock: self._updates.clear() polling = property(fget=get_polling, fset=set_polling) def set_error(self, error): """Sets an error, so that the next call to .poll() will raise it. Can be (and is) used to pass exceptions between threads. """ with self._updates_lock: # Insert at the beginning so the very next poll causes an error # TODO Should this reset the pts and such? self._updates.insert(0, error) self._updates_available.set() def check_error(self): with self._updates_lock: if self._updates and isinstance(self._updates[0], Exception): raise self._updates.pop() def process(self, update): """Processes an update object. This method is normally called by the library itself. """ if not self._polling and not self.handlers: return with self._updates_lock: if isinstance(update, tl.updates.State): self._state = update return # Nothing else to be done pts = getattr(update, 'pts', self._state.pts) if pts <= self._state.pts: return # We already handled this update self._state.pts = pts if self._polling: self._updates.append(update) self._updates_available.set() for handler in self.handlers: handler(update)
class IdentifyWidget(QWidget): # {{{ rejected = pyqtSignal() results_found = pyqtSignal() book_selected = pyqtSignal(object, object) def __init__(self, log, parent=None): QWidget.__init__(self, parent) self.log = log self.abort = Event() self.caches = {} self.l = l = QGridLayout() self.setLayout(l) names = ['<b>'+p.name+'</b>' for p in metadata_plugins(['identify']) if p.is_configured()] self.top = QLabel('<p>'+_('calibre is downloading metadata from: ') + ', '.join(names)) self.top.setWordWrap(True) l.addWidget(self.top, 0, 0) self.results_view = ResultsView(self) self.results_view.book_selected.connect(self.emit_book_selected) self.get_result = self.results_view.get_result l.addWidget(self.results_view, 1, 0) self.comments_view = Comments(self) l.addWidget(self.comments_view, 1, 1) self.results_view.show_details_signal.connect(self.comments_view.show_data) self.query = QLabel('download starting...') self.query.setWordWrap(True) l.addWidget(self.query, 2, 0, 1, 2) self.comments_view.show_wait() def emit_book_selected(self, book): self.book_selected.emit(book, self.caches) def start(self, title=None, authors=None, identifiers={}): self.log.clear() self.log('Starting download') parts, simple_desc = [], '' if title: parts.append('title:'+title) simple_desc += _('Title: %s ') % title if authors: parts.append('authors:'+authors_to_string(authors)) simple_desc += _('Authors: %s ') % authors_to_string(authors) if identifiers: x = ', '.join('%s:%s'%(k, v) for k, v in iteritems(identifiers)) parts.append(x) if 'isbn' in identifiers: simple_desc += 'ISBN: %s' % identifiers['isbn'] self.query.setText(simple_desc) self.log(unicode_type(self.query.text())) self.worker = IdentifyWorker(self.log, self.abort, title, authors, identifiers, self.caches) self.worker.start() QTimer.singleShot(50, self.update) def update(self): if self.worker.is_alive(): QTimer.singleShot(50, self.update) else: self.process_results() def process_results(self): if self.worker.error is not None: error_dialog(self, _('Download failed'), _('Failed to download metadata. Click ' 'Show Details to see details'), show=True, det_msg=self.worker.error) self.rejected.emit() return if not self.worker.results: log = ''.join(self.log.plain_text) error_dialog(self, _('No matches found'), '<p>' + _('Failed to find any books that ' 'match your search. Try making the search <b>less ' 'specific</b>. For example, use only the author\'s ' 'last name and a single distinctive word from ' 'the title.<p>To see the full log, click "Show details".'), show=True, det_msg=log) self.rejected.emit() return self.results_view.show_results(self.worker.results) self.results_found.emit() def cancel(self): self.abort.set()
class Receive(object): """ A class to perform a Mqlight client creation and then wait to receive messages from the server. """ def __init__(self, args): self._using_ssl = False self._security_options = {} ssl_arg_names = [ 'ssl_trust_certificate', 'ssl_client_certificate', 'ssl_client_key', 'ssl_client_key_passphrase', 'ssl_verify_name' ] for ssl_arg_name in ssl_arg_names: value = args.__dict__[ssl_arg_name] if value is not None: self._security_options[ssl_arg_name] = value self._using_ssl = True if 'ssl_verify_name' not in self._security_options: self._security_options[ssl_arg_name] = True # Select the relevant service if args.service is None: self._service = AMQPS_SERVICE if self._using_ssl else AMQP_SERVICE else: if self._using_ssl: if not args.service.startswith('amqps'): self._close( None, 'The service URL must start with ' '"amqps://" when using any of the ssl ' 'options.') self._service = args.service self._topic_pattern = args.topic_pattern if args.client_id is not None: self._client_id = args.client_id else: self._client_id = 'recv_' + str(uuid4()).replace('-', '_')[0:7] self._delay = args.delay self._share = args.share_name self._verbose = args.verbose self._destination_ttl = args.destination_ttl self._file = args.file self._client = None self._exit = Event() self._num_of_messages = AtomicVariable(0) def _close(self, client, err=None): """ Handles closing this sample :param client the associate client to close :param err an error condtion to report. """ if err: error = err[1] if isinstance(err, tuple) else err print('*** error ***\n{0}'.format(error), file=sys.stderr) if client: client.stop() self._exit.set() def _output(self, msg): print(msg) sys.stdout.flush() def _started(self, client): """ Started callback. Indicate a successful connection to client and open for business. :param client the associate client that has been successfully connected """ print('Connected to {0} using client-id {1}'.format( client.get_service(), client.get_id())) options = {'qos': mqlight.QOS_AT_LEAST_ONCE, 'auto_confirm': False} if self._destination_ttl is not None: options['ttl'] = self._destination_ttl * 1000 if self._delay is not None and self._delay > 0: options['credit'] = 1 client.subscribe(topic_pattern=self._topic_pattern, share=self._share, options=options, on_subscribed=self._subscribed, on_message=self._message) def _subscribed(self, client, err, pattern, share): """ Subscribe callback. Indicates the subscription has been completed. If not error is report then it is open to receive messages. :param client the associate client for this subscription :param err issues detected while making the subscription, None means successful :param pattern the topic or pattern for the subscription :param share is the subscription is shared. """ if err is not None: self._close(client, 'problem with subscribe request {0}'.format(err)) if pattern: if share: self._output('Subscribed to share: {0}, pattern: {1}'.format( share, pattern)) else: self._output('Subscribed to pattern: {0}'.format(pattern)) def _write_to_file(self, filename, data): """ Write the contains of the data to the named file :param filename the filepath to write the data to :data data the data to be written """ self._output('Writing message data to {0}'.format(self._file)) try: with open(self._file, 'wb') as f: if PYTHON2: if isinstance(data, str) or isinstance(data, unicode): f.write(data) else: f.write(''.join(chr(byte) for byte in data)) else: if isinstance(data, str): f.write(data) else: f.write(bytes(data)) except IOError as e: self._close( None, 'Failed to write message to {0} ' 'because {1}'.format(self._file, e)) def _message(self, message_type, data, delivery): """ Message callback. This indicate there is a message has been receive and ready for processing. :param client the reference to the connected client :param message_type, the type of message received (message or malformed) :param data, the actual message recevied :param delivery, the associated delivery information for the message """ if message_type == mqlight.MALFORMED: malformed_message = ('*** received malformed message ***\n' 'Data: {0}\nDelivery: {1}'.format( data, delivery)) print(malformed_message, file=sys.stderr) else: if self._verbose: self._output('# received message {0}'.format( self._num_of_messages.add())) if self._file: self._write_to_file(self._file, data) delivery['message']['confirm_delivery']() else: print('{0}{1}\n'.format(data[:50], (' ...' if len(data) > 50 else '')), end="") sys.stdout.flush() if self._verbose: self._output(delivery) if self._delay > 0: sleep(self._delay) if self._client.get_state() == 'started': delivery['message']['confirm_delivery']() def _state_changed(self, client, new_state, error): """ state_change callback. Indicates that there has been a change of connection state to the server. :param client the client reference whoses state has changed :param new_state the new state for the client :param any assoicated error with the state change. None means no error """ if error: if new_state in ('stopping', 'stopped', 'retrying'): self._close(client, error) def run(self): """ Start the sample by performing a client connection. The associated callback will handle the subsequent stages of receiving messages """ if not self._exit.isSet(): try: self._client = mqlight.Client( service=self._service, client_id=self._client_id, security_options=self._security_options, on_started=self._started, on_state_changed=self._state_changed) except Exception as exc: self._close(self._client, exc) # waiting for finish try: while not self._exit.wait(1.0): pass except KeyboardInterrupt: self._output('Closing ...') self._close(self._client)
class RawServer: def __init__(self, doneflag, timeout_check_interval, timeout, noisy = True, ipv6_enable = True, failfunc = lambda x: None, errorfunc = None, sockethandler = None, excflag = Event(), max_socket_connects = 1000): self.timeout_check_interval = timeout_check_interval self.timeout = timeout self.servers = {} self.single_sockets = {} self.dead_from_write = [] self.doneflag = doneflag self.noisy = noisy self.failfunc = failfunc self.errorfunc = errorfunc self.exccount = 0 self.funcs = [] self.externally_added = [] self.finished = Event() self.tasks_to_kill = [] self.excflag = excflag self.lock = RLock() if DEBUG2: log('rawserver::__init__: timeout_check_interval', timeout_check_interval, 'timeout', timeout, 'ipv6_enable', ipv6_enable) if sockethandler is None: if DEBUG2: log('rawserver::__init__: create SocketHandler: max_socket_connects', max_socket_connects) sockethandler = SocketHandler(timeout, ipv6_enable, READSIZE, max_socket_connects) self.sockethandler = sockethandler self.thread_ident = None self.interrupt_socket = sockethandler.get_interrupt_socket() self.add_task(self.scan_for_timeouts, timeout_check_interval) def get_exception_flag(self): return self.excflag def _add_task(self, func, delay, id = None): if delay < 0: delay = 0 insort(self.funcs, (clock() + delay, func, id)) def add_task(self, func, delay = 0, id = None): if DEBUG_TASKS: log('rawserver::add_task: func', func, 'delay', delay) if delay < 0: delay = 0 self.lock.acquire() self.externally_added.append((func, delay, id)) if self.thread_ident != get_ident(): self.interrupt_socket.interrupt() self.lock.release() def scan_for_timeouts(self): self.add_task(self.scan_for_timeouts, self.timeout_check_interval) self.sockethandler.scan_for_timeouts() def bind(self, port, bind = '', reuse = False, ipv6_socket_style = 1): result = self.sockethandler.bind(port, bind, reuse, ipv6_socket_style) return result def find_and_bind(self, first_try, minport, maxport, bind = '', reuse = False, ipv6_socket_style = 1, randomizer = False): result = self.sockethandler.find_and_bind(first_try, minport, maxport, bind, reuse, ipv6_socket_style, randomizer) return result def start_connection_raw(self, dns, socktype, handler = None): if DEBUG2: log('rawserver::start_connection_raw: dns', dns, 'socktype', socktype, 'handler', handler) return self.sockethandler.start_connection_raw(dns, socktype, handler) def start_connection(self, dns, handler = None, randomize = False): if DEBUG2: log('rawserver::start_connection: dns', dns, 'randomize', randomize, 'handler', handler) return self.sockethandler.start_connection(dns, handler, randomize) def get_stats(self): return self.sockethandler.get_stats() def pop_external(self): self.lock.acquire() while self.externally_added: a, b, c = self.externally_added.pop(0) self._add_task(a, b, c) self.lock.release() def listen_forever(self, handler): if DEBUG: log('rawserver::listen_forever: handler', handler) self.thread_ident = get_ident() self.sockethandler.set_handler(handler) try: while not self.doneflag.isSet(): try: self.pop_external() self._kill_tasks() if self.funcs: period = self.funcs[0][0] + 0.001 - clock() else: period = 1073741824 if period < 0: period = 0 events = self.sockethandler.do_poll(period) if self.doneflag.isSet(): if DEBUG: log('rawserver::listen_forever: stopping because done flag set') return while self.funcs and self.funcs[0][0] <= clock(): garbage1, func, id = self.funcs.pop(0) if id in self.tasks_to_kill: pass try: if DEBUG_TASKS: if func.func_name != '_bgalloc': log('rawserver::listen_forever: run func:', func.func_name) st = time.time() func() if DEBUG_TASKS: et = time.time() diff = et - st log('rawserver::listen_forever:', func.func_name, 'took %.5f' % diff) except (SystemError, MemoryError) as e: self.failfunc(e) return except KeyboardInterrupt as e: return except error: log('rawserver::listen_forever: func exception') print_exc() except Exception as e: raise self.sockethandler.close_dead() self.sockethandler.handle_events(events) if self.doneflag.isSet(): if DEBUG: log('rawserver::listen_forever: stopping because done flag set2') return self.sockethandler.close_dead() except (SystemError, MemoryError) as e: if DEBUG: log('rawserver::listen_forever: SYS/MEM exception', e) self.failfunc(e) return except error: if DEBUG: log('rawserver::listen_forever: ERROR exception') print_exc() if self.doneflag.isSet(): return except KeyboardInterrupt as e: self.failfunc(e) return except Exception as e: raise finally: self.finished.set() def is_finished(self): return self.finished.isSet() def wait_until_finished(self): self.finished.wait() def _kill_tasks(self): if self.tasks_to_kill: new_funcs = [] for t, func, id in self.funcs: if id not in self.tasks_to_kill: new_funcs.append((t, func, id)) self.funcs = new_funcs self.tasks_to_kill = [] def kill_tasks(self, id): self.tasks_to_kill.append(id) def exception(self, e, kbint = False): if not kbint: self.excflag.set() self.exccount += 1 if self.errorfunc is None: print_exc() elif not kbint: self.errorfunc(e) def shutdown(self): if DEBUG: log('rawserver::shuwdown: ---') self.sockethandler.shutdown() def create_udpsocket(self, port, host): if DEBUG: log('rawserver::create_udpsocket: host', host, 'port', port) return self.sockethandler.create_udpsocket(port, host) def start_listening_udp(self, serversocket, handler): if DEBUG: log('rawserver::start_listening_udp: serversocket', serversocket, 'handler', handler) self.sockethandler.start_listening_udp(serversocket, handler) def stop_listening_udp(self, serversocket): if DEBUG: log('rawserver::stop_listening_udp: serversocket', serversocket) self.sockethandler.stop_listening_udp(serversocket)
class CoversWidget(QWidget): # {{{ chosen = pyqtSignal() finished = pyqtSignal() def __init__(self, log, current_cover, parent=None): QWidget.__init__(self, parent) self.log = log self.abort = Event() self.l = l = QGridLayout() self.setLayout(l) self.msg = QLabel() self.msg.setWordWrap(True) l.addWidget(self.msg, 0, 0) self.covers_view = CoversView(current_cover, self) self.covers_view.chosen.connect(self.chosen) l.addWidget(self.covers_view, 1, 0) self.continue_processing = True def reset_covers(self): self.covers_view.reset_covers() def start(self, book, current_cover, title, authors, caches): self.continue_processing = True self.abort.clear() self.book, self.current_cover = book, current_cover self.title, self.authors = title, authors self.log('Starting cover download for:', book.title) self.log('Query:', title, authors, self.book.identifiers) self.msg.setText('<p>'+ _('Downloading covers for <b>%s</b>, please wait...')%book.title) self.covers_view.start() self.worker = CoverWorker(self.log, self.abort, self.title, self.authors, book.identifiers, caches) self.worker.start() QTimer.singleShot(50, self.check) self.covers_view.setFocus(Qt.OtherFocusReason) def check(self): if self.worker.is_alive() and not self.abort.is_set(): QTimer.singleShot(50, self.check) try: self.process_result(self.worker.rq.get_nowait()) except Empty: pass else: self.process_results() def process_results(self): while self.continue_processing: try: self.process_result(self.worker.rq.get_nowait()) except Empty: break if self.continue_processing: self.covers_view.clear_failed() if self.worker.error and self.worker.error.strip(): error_dialog(self, _('Download failed'), _('Failed to download any covers, click' ' "Show details" for details.'), det_msg=self.worker.error, show=True) num = self.covers_view.model().rowCount() if num < 2: txt = _('Could not find any covers for <b>%s</b>')%self.book.title else: txt = _('Found <b>%(num)d</b> possible covers for %(title)s. ' 'When the download completes, the covers will be sorted by size.')%dict(num=num-1, title=self.title) self.msg.setText(txt) self.msg.setWordWrap(True) self.covers_view.stop() self.finished.emit() def process_result(self, result): if not self.continue_processing: return plugin_name, width, height, fmt, data = result self.covers_view.model().update_result(plugin_name, width, height, data) def cleanup(self): self.covers_view.delegate.stop_animation() self.continue_processing = False def cancel(self): self.cleanup() self.abort.set() def cover_pixmap(self): idx = None for i in self.covers_view.selectionModel().selectedIndexes(): if i.isValid(): idx = i break if idx is None: idx = self.covers_view.currentIndex() return self.covers_view.model().cover_pixmap(idx)
class QuantumProgram(object): """Quantum Program Class. Class internal properties. Elements that are not python identifiers or string constants are denoted by "--description (type)--". For example, a circuit's name is denoted by "--circuit name (string)--" and might have the value "teleport". Internal:: __quantum_registers (list[dic]): An dictionary of quantum registers used in the quantum program. __quantum_registers = { --register name (string)--: QuantumRegistor, } __classical_registers (list[dic]): An ordered list of classical registers used in the quantum program. __classical_registers = { --register name (string)--: ClassicalRegistor, } __quantum_program (dic): An dictionary of quantum circuits __quantum_program = { --circuit name (string)--: --circuit object --, } __init_circuit (obj): A quantum circuit object for the initial quantum circuit __ONLINE_BACKENDS (list[str]): A list of online backends __LOCAL_BACKENDS (list[str]): A list of local backends """ # -- FUTURE IMPROVEMENTS -- # TODO: for status results make ALL_CAPS (check) or some unified method # TODO: Jay: coupling_map, basis_gates will move into a config object # only exists once you set the api to use the online backends __api = {} __api_config = {} def __init__(self, specs=None): self.__quantum_registers = {} self.__classical_registers = {} self.__quantum_program = {} # stores all the quantum programs self.__init_circuit = None # stores the intial quantum circuit of the # program self.__ONLINE_BACKENDS = [] # pylint: disable=invalid-name self.__LOCAL_BACKENDS = qiskit.backends.local_backends() # pylint: disable=invalid-name self.mapper = mapper if specs: self.__init_specs(specs) self.callback = None self.jobs_results = [] self.jobs_results_ready_event = Event() self.are_multiple_results = False # are we expecting multiple results? def enable_logs(self, level=logging.INFO): """Enable the console output of the logging messages. Enable the output of logging messages (above level `level`) to the console, by configuring the `qiskit` logger accordingly. Params: level (int): minimum severity of the messages that are displayed. Note: This is a convenience method over the standard Python logging facilities, and modifies the configuration of the 'qiskit.*' loggers. If finer control over the logging configuration is needed, it is encouraged to bypass this method. """ # Update the handlers and formatters. set_qiskit_logger() # Set the logger level. logging.getLogger('qiskit').setLevel(level) def disable_logs(self): """Disable the console output of the logging messages. Disable the output of logging messages (above level `level`) to the console, by removing the handlers from the `qiskit` logger. Note: This is a convenience method over the standard Python logging facilities, and modifies the configuration of the 'qiskit.*' loggers. If finer control over the logging configuration is needed, it is encouraged to bypass this method. """ unset_qiskit_logger() ############################################################### # methods to initiate an build a quantum program ############################################################### def __init_specs(self, specs): """Populate the Quantum Program Object with initial Specs. Args: specs (dict): Q_SPECS = { "circuits": [{ "name": "Circuit", "quantum_registers": [{ "name": "qr", "size": 4 }], "classical_registers": [{ "name": "cr", "size": 4 }] }], """ quantumr = [] classicalr = [] if "circuits" in specs: for circuit in specs["circuits"]: quantumr = self.create_quantum_registers( circuit["quantum_registers"]) classicalr = self.create_classical_registers( circuit["classical_registers"]) self.create_circuit(name=circuit["name"], qregisters=quantumr, cregisters=classicalr) # TODO: Jay: I think we should return function handles for the # registers and circuit. So that we dont need to get them after we # create them with get_quantum_register etc def create_quantum_register(self, name, size): """Create a new Quantum Register. Args: name (str): the name of the quantum register size (int): the size of the quantum register Returns: QuantumRegister: internal reference to a quantum register in __quantum_registers Raises: QISKitError: if the register already exists in the program. """ if name in self.__quantum_registers: if size != len(self.__quantum_registers[name]): raise QISKitError("Can't make this register: Already in" " program with different size") logger.info(">> quantum_register exists: %s %s", name, size) else: self.__quantum_registers[name] = QuantumRegister(name, size) logger.info(">> new quantum_register created: %s %s", name, size) return self.__quantum_registers[name] def create_quantum_registers(self, register_array): """Create a new set of Quantum Registers based on a array of them. Args: register_array (list[dict]): An array of quantum registers in dictionay format:: "quantum_registers": [ { "name": "qr", "size": 4 }, ... ] Returns: list(QuantumRegister): Array of quantum registers objects """ new_registers = [] for register in register_array: register = self.create_quantum_register(register["name"], register["size"]) new_registers.append(register) return new_registers def create_classical_register(self, name, size): """Create a new Classical Register. Args: name (str): the name of the classical register size (int): the size of the classical register Returns: ClassicalRegister: internal reference to a classical register in __classical_registers Raises: QISKitError: if the register already exists in the program. """ if name in self.__classical_registers: if size != len(self.__classical_registers[name]): raise QISKitError("Can't make this register: Already in" " program with different size") logger.info(">> classical register exists: %s %s", name, size) else: logger.info(">> new classical register created: %s %s", name, size) self.__classical_registers[name] = ClassicalRegister(name, size) return self.__classical_registers[name] def create_classical_registers(self, registers_array): """Create a new set of Classical Registers based on a array of them. Args: registers_array (list[dict]): An array of classical registers in dictionay fromat:: "classical_registers": [ { "name": "qr", "size": 4 }, ... ] Returns: list(ClassicalRegister): Array of clasical registers objects """ new_registers = [] for register in registers_array: new_registers.append( self.create_classical_register(register["name"], register["size"])) return new_registers def create_circuit(self, name, qregisters=None, cregisters=None): """Create a empty Quantum Circuit in the Quantum Program. Args: name (str): the name of the circuit. qregisters (list(QuantumRegister)): is an Array of Quantum Registers by object reference cregisters (list(ClassicalRegister)): is an Array of Classical Registers by object reference Returns: QuantumCircuit: A quantum circuit is created and added to the Quantum Program """ if not qregisters: qregisters = [] if not cregisters: cregisters = [] quantum_circuit = QuantumCircuit() if not self.__init_circuit: self.__init_circuit = quantum_circuit for register in qregisters: quantum_circuit.add(register) for register in cregisters: quantum_circuit.add(register) self.add_circuit(name, quantum_circuit) return self.__quantum_program[name] def add_circuit(self, name, quantum_circuit): """Add a new circuit based on an Object representation. Args: name (str): the name of the circuit to add. quantum_circuit (QuantumCircuit): a quantum circuit to add to the program-name """ for qname, qreg in quantum_circuit.get_qregs().items(): self.create_quantum_register(qname, len(qreg)) for cname, creg in quantum_circuit.get_cregs().items(): self.create_classical_register(cname, len(creg)) self.__quantum_program[name] = quantum_circuit def load_qasm_file(self, qasm_file, name=None, basis_gates='u1,u2,u3,cx,id'): """ Load qasm file into the quantum program. Args: qasm_file (str): a string for the filename including its location. name (str or None): the name of the quantum circuit after loading qasm text into it. If no name is give the name is of the text file. basis_gates (str): basis gates for the quantum circuit. Returns: str: Adds a quantum circuit with the gates given in the qasm file to the quantum program and returns the name to be used to get this circuit Raises: QISKitError: if the file cannot be read. """ if not os.path.exists(qasm_file): raise QISKitError('qasm file "{0}" not found'.format(qasm_file)) if not name: name = os.path.splitext(os.path.basename(qasm_file))[0] node_circuit = qasm.Qasm(filename=qasm_file).parse() # Node (AST) logger.info("circuit name: " + name) logger.info("******************************") logger.info(node_circuit.qasm()) # current method to turn it a DAG quantum circuit. unrolled_circuit = unroll.Unroller( node_circuit, unroll.CircuitBackend(basis_gates.split(","))) circuit_unrolled = unrolled_circuit.execute() self.add_circuit(name, circuit_unrolled) return name def load_qasm_text(self, qasm_string, name=None, basis_gates='u1,u2,u3,cx,id'): """ Load qasm string in the quantum program. Args: qasm_string (str): a string for the file name. name (str or None): the name of the quantum circuit after loading qasm text into it. If no name is give the name is of the text file. basis_gates (str): basis gates for the quantum circuit. Returns: str: Adds a quantum circuit with the gates given in the qasm string to the quantum program. """ node_circuit = qasm.Qasm(data=qasm_string).parse() # Node (AST) if not name: # Get a random name if none is given name = "".join([ random.choice(string.ascii_letters + string.digits) for n in range(10) ]) logger.info("circuit name: " + name) logger.info("******************************") logger.info(node_circuit.qasm()) # current method to turn it a DAG quantum circuit. unrolled_circuit = unroll.Unroller( node_circuit, unroll.CircuitBackend(basis_gates.split(","))) circuit_unrolled = unrolled_circuit.execute() self.add_circuit(name, circuit_unrolled) return name ############################################################### # methods to get elements from a QuantumProgram ############################################################### def get_quantum_register(self, name): """Return a Quantum Register by name. Args: name (str): the name of the quantum register Returns: QuantumRegister: The quantum register with this name Raises: KeyError: if the quantum register is not on the quantum program. """ try: return self.__quantum_registers[name] except KeyError: raise KeyError('No quantum register "{0}"'.format(name)) def get_classical_register(self, name): """Return a Classical Register by name. Args: name (str): the name of the classical register Returns: ClassicalRegister: The classical register with this name Raises: KeyError: if the classical register is not on the quantum program. """ try: return self.__classical_registers[name] except KeyError: raise KeyError('No classical register "{0}"'.format(name)) def get_quantum_register_names(self): """Return all the names of the quantum Registers.""" return self.__quantum_registers.keys() def get_classical_register_names(self): """Return all the names of the classical Registers.""" return self.__classical_registers.keys() def get_circuit(self, name): """Return a Circuit Object by name Args: name (str): the name of the quantum circuit Returns: QuantumCircuit: The quantum circuit with this name Raises: KeyError: if the circuit is not on the quantum program. """ try: return self.__quantum_program[name] except KeyError: raise KeyError('No quantum circuit "{0}"'.format(name)) def get_circuit_names(self): """Return all the names of the quantum circuits.""" return self.__quantum_program.keys() def get_qasm(self, name): """Get qasm format of circuit by name. Args: name (str): name of the circuit Returns: str: The quantum circuit in qasm format """ quantum_circuit = self.get_circuit(name) return quantum_circuit.qasm() def get_qasms(self, list_circuit_name): """Get qasm format of circuit by list of names. Args: list_circuit_name (list[str]): names of the circuit Returns: list(QuantumCircuit): List of quantum circuit in qasm format """ qasm_source = [] for name in list_circuit_name: qasm_source.append(self.get_qasm(name)) return qasm_source def get_initial_circuit(self): """Return the initialization Circuit.""" return self.__init_circuit ############################################################### # methods for working with backends ############################################################### def set_api(self, token, url, verify=True): """ Setup the API. Fills the __ONLINE_BACKENDS, __api, and __api_config variables. Does not catch exceptions from IBMQuantumExperience. Args: token (str): The token used to register on the online backend such as the quantum experience. url (str): The url used for online backend such as the quantum experience. verify (bool): If False, ignores SSL certificates errors. Raises: ConnectionError: if the API instantiation failed. """ try: self.__api = IBMQuantumExperience(token, {"url": url}, verify) except Exception as ex: raise ConnectionError( "Couldn't connect to IBMQuantumExperience server: {0}".format( ex)) qiskit.backends.discover_remote_backends(self.__api) self.__ONLINE_BACKENDS = self.online_backends() self.__api_config["token"] = token self.__api_config["url"] = {"url": url} def get_api_config(self): """Return the program specs.""" return self.__api_config def get_api(self): """Returns a function handle to the API.""" return self.__api def save(self, file_name=None, beauty=False): """ Save Quantum Program in a Json file. Args: file_name (str): file name and path. beauty (boolean): save the text with indent 4 to make it readable. Returns: dict: The dictionary with the status and result of the operation Raises: LookupError: if the file_name is not correct, or writing to the file resulted in an error. """ if file_name is None: error = {"status": "Error", "result": "Not filename provided"} raise LookupError(error['result']) if beauty: indent = 4 else: indent = 0 elemements_to_save = self.__quantum_program elements_saved = {} for circuit in elemements_to_save: elements_saved[circuit] = {} elements_saved[circuit]["qasm"] = elemements_to_save[circuit].qasm( ) try: with open(file_name, 'w') as save_file: json.dump(elements_saved, save_file, indent=indent) return {'status': 'Done', 'result': elemements_to_save} except ValueError: error = { 'status': 'Error', 'result': 'Some Problem happened to save the file' } raise LookupError(error['result']) def load(self, file_name=None): """ Load Quantum Program Json file into the Quantum Program object. Args: file_name (str): file name and path. Returns: dict: The dictionary with the status and result of the operation Raises: LookupError: if the file_name is not correct, or reading from the file resulted in an error. """ if file_name is None: error = {"status": "Error", "result": "Not filename provided"} raise LookupError(error['result']) try: with open(file_name, 'r') as load_file: elemements_loaded = json.load(load_file) for circuit in elemements_loaded: circuit_qasm = elemements_loaded[circuit]["qasm"] elemements_loaded[circuit] = qasm.Qasm( data=circuit_qasm).parse() self.__quantum_program = elemements_loaded return {"status": 'Done', 'result': self.__quantum_program} except ValueError: error = { 'status': 'Error', 'result': 'Some Problem happened to load the file' } raise LookupError(error['result']) def available_backends(self): """All the backends that are seen by QISKIT.""" return self.__ONLINE_BACKENDS + self.__LOCAL_BACKENDS def online_backends(self): """Get the online backends. Queries network API if it exists and gets the backends that are online. Returns: list(str): List of online backends names if the online api has been set or an empty list if it has not been set. Raises: ConnectionError: if the API call failed. """ if self.get_api(): try: backends = self.__api.available_backends() except Exception as ex: raise ConnectionError( "Couldn't get available backend list: {0}".format(ex)) return [backend['name'] for backend in backends] return [] def online_simulators(self): """Gets online simulators via QX API calls. Returns: list(str): List of online simulator names. Raises: ConnectionError: if the API call failed. """ online_simulators_list = [] if self.get_api(): try: backends = self.__api.available_backends() except Exception as ex: raise ConnectionError( "Couldn't get available backend list: {0}".format(ex)) for backend in backends: if backend['simulator']: online_simulators_list.append(backend['name']) return online_simulators_list def online_devices(self): """Gets online devices via QX API calls. Returns: list(str): List of online devices names. Raises: ConnectionError: if the API call failed. """ devices = [] if self.get_api(): try: backends = self.__api.available_backends() except Exception as ex: raise ConnectionError( "Couldn't get available backend list: {0}".format(ex)) for backend in backends: if not backend['simulator']: devices.append(backend['name']) return devices def get_backend_status(self, backend): """Return the online backend status. It uses QX API call or by local backend is the name of the local or online simulator or experiment. Args: backend (str): The backend to check Returns: dict: {'available': True} Raises: ConnectionError: if the API call failed. ValueError: if the backend is not available. """ if backend in self.__ONLINE_BACKENDS: try: return self.__api.backend_status(backend) except Exception as ex: raise ConnectionError( "Couldn't get backend status: {0}".format(ex)) elif backend in self.__LOCAL_BACKENDS: return {'available': True} else: raise ValueError( 'the backend "{0}" is not available'.format(backend)) def get_backend_configuration(self, backend, list_format=False): """Return the configuration of the backend. The return is via QX API call. Args: backend (str): Name of the backend. list_format (bool): Struct used for the configuration coupling map: dict (if False) or list (if True). Returns: dict: The configuration of the named backend. Raises: ConnectionError: if the API call failed. LookupError: if a configuration for the named backend can't be found. """ if self.get_api(): configuration_edit = {} try: backends = self.__api.available_backends() except Exception as ex: raise ConnectionError( "Couldn't get available backend list: {0}".format(ex)) for configuration in backends: if configuration['name'] == backend: for key in configuration: new_key = convert(key) # TODO: removed these from the API code if new_key not in [ 'id', 'serial_number', 'topology_id', 'status', 'coupling_map' ]: configuration_edit[new_key] = configuration[key] if new_key == 'coupling_map': if configuration[key] == 'all-to-all': configuration_edit[new_key] = \ configuration[key] else: if not list_format: cmap = mapper.coupling_list2dict( configuration[key]) else: cmap = configuration[key] configuration_edit[new_key] = cmap return configuration_edit else: return qiskit.backends.get_backend_configuration(backend) def get_backend_calibration(self, backend): """Return the online backend calibrations. The return is via QX API call. Args: backend (str): Name of the backend. Returns: dict: The calibration of the named backend. Raises: ConnectionError: if the API call failed. LookupError: If a configuration for the named backend can't be found. """ if backend in self.__ONLINE_BACKENDS: try: calibrations = self.__api.backend_calibration(backend) except Exception as ex: raise ConnectionError( "Couldn't get backend calibration: {0}".format(ex)) calibrations_edit = {} for key, vals in calibrations.items(): new_key = convert(key) calibrations_edit[new_key] = vals return calibrations_edit elif backend in self.__LOCAL_BACKENDS: return {'backend': backend, 'calibrations': None} else: raise LookupError( 'backend calibration for "{0}" not found'.format(backend)) def get_backend_parameters(self, backend): """Return the online backend parameters. The return is via QX API call. Args: backend (str): Name of the backend. Returns: dict: The configuration of the named backend. Raises: ConnectionError: if the API call failed. LookupError: If a configuration for the named backend can't be found. """ if backend in self.__ONLINE_BACKENDS: try: parameters = self.__api.backend_parameters(backend) except Exception as ex: raise ConnectionError( "Couldn't get backend paramters: {0}".format(ex)) parameters_edit = {} for key, vals in parameters.items(): new_key = convert(key) parameters_edit[new_key] = vals return parameters_edit elif backend in self.__LOCAL_BACKENDS: return {'backend': backend, 'parameters': None} else: raise LookupError( 'backend parameters for "{0}" not found'.format(backend)) ############################################################### # methods to compile quantum programs into qobj ############################################################### def compile(self, name_of_circuits, backend="local_qasm_simulator", config=None, basis_gates=None, coupling_map=None, initial_layout=None, shots=1024, max_credits=3, seed=None, qobj_id=None): """Compile the circuits into the exectution list. This builds the internal "to execute" list which is list of quantum circuits to run on different backends. Args: name_of_circuits (list[str]): circuit names to be compiled. backend (str): a string representing the backend to compile to config (dict): a dictionary of configurations parameters for the compiler basis_gates (str): a comma seperated string and are the base gates, which by default are: u1,u2,u3,cx,id coupling_map (dict): A directed graph of coupling:: { control(int): [ target1(int), target2(int), , ... ], ... } eg. {0: [2], 1: [2], 3: [2]} initial_layout (dict): A mapping of qubit to qubit:: { ("q", strart(int)): ("q", final(int)), ... } eg. { ("q", 0): ("q", 0), ("q", 1): ("q", 1), ("q", 2): ("q", 2), ("q", 3): ("q", 3) } shots (int): the number of shots max_credits (int): the max credits to use 3, or 5 seed (int): the intial seed the simulatros use qobj_id (str): identifier of the qobj. Returns: dict: the job id and populates the qobj:: qobj = { id: --job id (string), config: -- dictionary of config settings (dict)--, { "max_credits" (online only): -- credits (int) --, "shots": -- number of shots (int) --. "backend": -- backend name (str) -- } circuits: [ { "name": --circuit name (string)--, "compiled_circuit": --compiled quantum circuit (JSON format)--, "compiled_circuit_qasm": --compiled quantum circuit (QASM format)--, "config": --dictionary of additional config settings (dict)--, { "coupling_map": --adjacency list (dict)--, "basis_gates": --comma separated gate names (string)--, "layout": --layout computed by mapper (dict)--, "seed": (simulator only)--initial seed for the simulator (int)--, } }, ... ] } Raises: ValueError: if no names of the circuits have been specified. QISKitError: if any of the circuit names cannot be found on the Quantum Program. """ # TODO: Jay: currently basis_gates, coupling_map, initial_layout, # shots, max_credits and seed are extra inputs but I would like # them to go into the config. qobj = {} if not qobj_id: qobj_id = "".join([ random.choice(string.ascii_letters + string.digits) for n in range(30) ]) qobj['id'] = qobj_id qobj["config"] = { "max_credits": max_credits, 'backend': backend, "shots": shots } qobj["circuits"] = [] if not name_of_circuits: raise ValueError('"name_of_circuits" must be specified') if isinstance(name_of_circuits, str): name_of_circuits = [name_of_circuits] for name in name_of_circuits: if name not in self.__quantum_program: raise QISKitError( 'circuit "{0}" not found in program'.format(name)) if not basis_gates: basis_gates = "u1,u2,u3,cx,id" # QE target basis # TODO: The circuit object going into this is to have .qasm() method (be careful) circuit = self.__quantum_program[name] dag_circuit, final_layout = openquantumcompiler.compile( circuit.qasm(), basis_gates=basis_gates, coupling_map=coupling_map, initial_layout=initial_layout, get_layout=True) # making the job to be added to qobj job = {} job["name"] = name # config parameters used by the runner if config is None: config = {} # default to empty config dict job["config"] = copy.deepcopy(config) job["config"]["coupling_map"] = mapper.coupling_dict2list( coupling_map) # TODO: Jay: make config options optional for different backends # Map the layout to a format that can be json encoded list_layout = None if final_layout: list_layout = [[k, v] for k, v in final_layout.items()] job["config"]["layout"] = list_layout job["config"]["basis_gates"] = basis_gates if seed is None: job["config"]["seed"] = None else: job["config"]["seed"] = seed # the compiled circuit to be run saved as a dag job["compiled_circuit"] = openquantumcompiler.dag2json( dag_circuit, basis_gates=basis_gates) job["compiled_circuit_qasm"] = dag_circuit.qasm(qeflag=True) # add job to the qobj qobj["circuits"].append(job) return qobj def reconfig(self, qobj, backend=None, config=None, shots=None, max_credits=None, seed=None): """Change configuration parameters for a compile qobj. Only parameters which don't affect the circuit compilation can change, e.g., the coupling_map cannot be changed here! Notes: If the inputs are left as None then the qobj is not updated Args: qobj (dict): already compile qobj backend (str): see .compile config (dict): see .compile shots (int): see .compile max_credits (int): see .compile seed (int): see .compile Returns: qobj: updated qobj """ if backend is not None: qobj['config']['backend'] = backend if shots is not None: qobj['config']['shots'] = shots if max_credits is not None: qobj['config']['max_credits'] = max_credits for circuits in qobj['circuits']: if seed is not None: circuits['seed'] = seed if config is not None: circuits['config'].update(config) return qobj def get_execution_list(self, qobj): """Print the compiled circuits that are ready to run. Note: This method is intended to be used during interactive sessions, and prints directly to stdout instead of using the logger. Returns: list(str): names of the circuits in `qobj` """ if not qobj: print("no executions to run") execution_list = [] print("id: %s" % qobj['id']) print("backend: %s" % qobj['config']['backend']) print("qobj config:") for key in qobj['config']: if key != 'backend': print(' ' + key + ': ' + str(qobj['config'][key])) for circuit in qobj['circuits']: execution_list.append(circuit["name"]) print(' circuit name: ' + circuit["name"]) print(' circuit config:') for key in circuit['config']: print(' ' + key + ': ' + str(circuit['config'][key])) return execution_list def get_compiled_configuration(self, qobj, name): """Get the compiled layout for the named circuit and backend. Args: name (str): the circuit name qobj (dict): the qobj Returns: dict: the config of the circuit. Raises: QISKitError: if the circuit has no configurations """ try: for index in range(len(qobj["circuits"])): if qobj["circuits"][index]['name'] == name: return qobj["circuits"][index]["config"] except KeyError: raise QISKitError( 'No compiled configurations for circuit "{0}"'.format(name)) def get_compiled_qasm(self, qobj, name): """Return the compiled cricuit in qasm format. Args: qobj (dict): the qobj name (str): name of the quantum circuit Returns: str: the QASM of the compiled circuit. Raises: QISKitError: if the circuit has no configurations """ try: for index in range(len(qobj["circuits"])): if qobj["circuits"][index]['name'] == name: return qobj["circuits"][index]["compiled_circuit_qasm"] except KeyError: raise QISKitError( 'No compiled qasm for circuit "{0}"'.format(name)) ############################################################### # methods to run quantum programs ############################################################### def run(self, qobj, wait=5, timeout=60): """Run a program (a pre-compiled quantum program). This function will block until the Job is processed. The program to run is extracted from the qobj parameter. Args: qobj (dict): the dictionary of the quantum object to run. wait (int): Time interval to wait between requests for results timeout (int): Total time to wait until the execution stops Returns: Result: A Result (class). """ self.callback = None self._run_internal([qobj], wait=wait, timeout=timeout) self.wait_for_results(timeout) return self.jobs_results[0] def run_batch(self, qobj_list, wait=5, timeout=120): """Run various programs (a list of pre-compiled quantum programs). This function will block until all programs are processed. The programs to run are extracted from qobj elements of the list. Args: qobj_list (list(dict)): The list of quantum objects to run. wait (int): Time interval to wait between requests for results timeout (int): Total time to wait until the execution stops Returns: list(Result): A list of Result (class). The list will contain one Result object per qobj in the input list. """ self._run_internal(qobj_list, wait=wait, timeout=timeout, are_multiple_results=True) self.wait_for_results(timeout) return self.jobs_results def run_async(self, qobj, wait=5, timeout=60, callback=None): """Run a program (a pre-compiled quantum program) asynchronously. This is a non-blocking function, so it will return inmediately. All input for run comes from qobj. Args: qobj(dict): the dictionary of the quantum object to run or list of qobj. wait (int): Time interval to wait between requests for results timeout (int): Total time to wait until the execution stops callback (fn(result)): A function with signature: fn(result): The result param will be a Result object. """ self._run_internal([qobj], wait=wait, timeout=timeout, callback=callback) def run_batch_async(self, qobj_list, wait=5, timeout=120, callback=None): """Run various programs (a list of pre-compiled quantum program) asynchronously. This is a non-blocking function, so it will return inmediately. All input for run comes from qobj. Args: qobj_list (list(dict)): The list of quantum objects to run. wait (int): Time interval to wait between requests for results timeout (int): Total time to wait until the execution stops callback (fn(results)): A function with signature: fn(results): The results param will be a list of Result objects, one Result per qobj in the input list. """ self._run_internal(qobj_list, wait=wait, timeout=timeout, callback=callback, are_multiple_results=True) def _run_internal(self, qobj_list, wait=5, timeout=60, callback=None, are_multiple_results=False): self.callback = callback self.are_multiple_results = are_multiple_results q_job_list = [] for qobj in qobj_list: q_job = QuantumJob(qobj, preformatted=True) q_job_list.append(q_job) job_processor = JobProcessor(q_job_list, max_workers=5, callback=self._jobs_done_callback) job_processor.submit() def _jobs_done_callback(self, jobs_results): """ This internal callback will be called once all Jobs submitted have finished. NOT every time a job has finished. Args: jobs_results (list): list of Result objects """ if self.callback is None: # We are calling from blocking functions (run, run_batch...) self.jobs_results = jobs_results self.jobs_results_ready_event.set() return if self.are_multiple_results: self.callback(jobs_results) # for run_batch_async() callback else: self.callback(jobs_results[0]) # for run_async() callback def wait_for_results(self, timeout): """Wait for all the results to be ready during an execution.""" is_ok = self.jobs_results_ready_event.wait(timeout) self.jobs_results_ready_event.clear() if not is_ok: raise QISKitError( 'Error waiting for Job results: Timeout after {0} ' 'seconds.'.format(timeout)) def execute(self, name_of_circuits, backend="local_qasm_simulator", config=None, wait=5, timeout=60, basis_gates=None, coupling_map=None, initial_layout=None, shots=1024, max_credits=3, seed=None): """Execute, compile, and run an array of quantum circuits). This builds the internal "to execute" list which is list of quantum circuits to run on different backends. Args: name_of_circuits (list[str]): circuit names to be compiled. backend (str): a string representing the backend to compile to config (dict): a dictionary of configurations parameters for the compiler wait (int): Time interval to wait between requests for results timeout (int): Total time to wait until the execution stops basis_gates (str): a comma seperated string and are the base gates, which by default are: u1,u2,u3,cx,id coupling_map (dict): A directed graph of coupling:: { control(int): [ target1(int), target2(int), , ... ], ... } eg. {0: [2], 1: [2], 3: [2]} initial_layout (dict): A mapping of qubit to qubit { ("q", strart(int)): ("q", final(int)), ... } eg. { ("q", 0): ("q", 0), ("q", 1): ("q", 1), ("q", 2): ("q", 2), ("q", 3): ("q", 3) } shots (int): the number of shots max_credits (int): the max credits to use 3, or 5 seed (int): the intial seed the simulatros use Returns: Result: status done and populates the internal __quantum_program with the data """ # TODO: Jay: currently basis_gates, coupling_map, intial_layout, shots, # max_credits, and seed are extra inputs but I would like them to go # into the config qobj = self.compile(name_of_circuits, backend=backend, config=config, basis_gates=basis_gates, coupling_map=coupling_map, initial_layout=initial_layout, shots=shots, max_credits=max_credits, seed=seed) result = self.run(qobj, wait=wait, timeout=timeout) return result
class IOLoopThread(Thread): """Run a pyzmq ioloop in a thread to send and receive messages """ _exiting = False ioloop = None def __init__(self): super(IOLoopThread, self).__init__() self.daemon = True @staticmethod @atexit.register def _notice_exit(): # Class definitions can be torn down during interpreter shutdown. # We only need to set _exiting flag if this hasn't happened. if IOLoopThread is not None: IOLoopThread._exiting = True def start(self): """Start the IOLoop thread Don't return until self.ioloop is defined, which is created in the thread """ self._start_event = Event() Thread.start(self) self._start_event.wait() def run(self): """Run my loop, ignoring EINTR events in the poller""" if 'asyncio' in sys.modules: # tornado may be using asyncio, # ensure an eventloop exists for this thread import asyncio asyncio.set_event_loop(asyncio.new_event_loop()) self.ioloop = ioloop.IOLoop() # signal that self.ioloop is defined self._start_event.set() while True: try: self.ioloop.start() except ZMQError as e: if e.errno == errno.EINTR: continue else: raise except Exception: if self._exiting: break else: raise else: break def stop(self): """Stop the channel's event loop and join its thread. This calls :meth:`~threading.Thread.join` and returns when the thread terminates. :class:`RuntimeError` will be raised if :meth:`~threading.Thread.start` is called again. """ if self.ioloop is not None: self.ioloop.add_callback(self.ioloop.stop) self.join() self.close() self.ioloop = None def close(self): if self.ioloop is not None: try: self.ioloop.close(all_fds=True) except Exception: pass
class SmuffPlugin(octoprint.plugin.SettingsPlugin, octoprint.plugin.AssetPlugin, octoprint.plugin.TemplatePlugin, octoprint.plugin.StartupPlugin, octoprint.plugin.EventHandlerPlugin, octoprint.plugin.ShutdownPlugin): def __init__(self, _logger, _lock): self._serial = None # serial instance to communicate with the SMuFF self._serlock = _lock # lock object for reading/writing self._serevent = Event( ) # event raised when a valid response has been received self._fw_info = "?" # SMuFFs firmware info self._cur_tool = "-1" # the current tool self._pre_tool = "-1" # the previous tool self._pending_tool = "?" # the tool on a pending tool change self._selector = False # status of the Selector endstop self._revolver = False # status of the Revolver endstop self._feeder = False # status of the Feeder endstop self._feeder2 = False # status of the 2nd Feeder endstop self._is_busy = False # flag set when SMuFF signals "Busy" self._is_error = False # flag set when SMuFF signals "Error" self._is_aligned = False # flag set when Feeder endstop is reached (not used yet) self._response = None # the response string from SMuFF self._wait_requested = False # set when SMuFF requested a "Wait" (in case of jams or similar) self._abort_requested = False # set when SMuFF requested a "Abort" ##~~ ShutdownPlugin mixin def on_startup(self, host, port): self._logger.info("Yeah... starting up...") def on_shutdown(self): close_SMuFF_serial() self._logger.debug("Booo... shutting down...") ##~~ StartupPlugin mixin def on_after_startup(self): port = self._settings.get(["tty"]) baud = self._settings.get_int(["baudrate"]) self._logger.debug("Opening serial {0} with {1}".format(port, baud)) if open_SMuFF_serial(port, baud): self._serial = __ser0__ start_reader_thread() # request firmware info from SMuFF if self._serial.is_open: self._fw_info = self.send_SMuFF_and_wait(M115) self._logger.debug("FW-Info: {}".format(self._fw_info)) pass ##~~ EventHandler mixin def on_event(self, event, payload): #self._logger.debug("Event: [" + event + ", {0}".format(payload) + "]") if event == Events.SHUTDOWN: self._logger.debug("Shutting down, closing serial") close_SMuFF_serial() ##~~ SettingsPlugin mixin def get_settings_defaults(self): self._logger.debug("SMuFF plugin loaded, getting defaults") params = dict(firmware_info="No data. Please check connection!", baudrate=SERBAUD, tty=SERDEV, tool=self._cur_tool, selector_end=self._selector, revolver_end=self._revolver, feeder_end=self._feeder, feeder2_end=self._feeder) # look up the serial port driver #if sys.platform == "win32": # if SERDEV.startswith("tty"): # params['tty'] = "Wrong device on WIN32 ({})".format(SERDEV) # else: # params['tty'] = SERDEV #else: # drvr = self.find_file(SERDEV, "/dev") # if len(drvr) > 0: # params['tty'] = "/dev/{}".format(SERDEV) return params def on_settings_save(self, data): global __ser0__ baud = self._settings.get_int(["baudrate"]) port = self._settings.get(["tty"]) octoprint.plugin.SettingsPlugin.on_settings_save(self, data) baud_new = self._settings.get_int(["baudrate"]) port_new = self._settings.get(["tty"]) self._logger.debug("Settings saved: {0}/{1} {2}/{3}".format( baud, baud_new, port, port_new)) # did the settings change? if not port_new == port or not baud_new == baud: # close the previous port, re-open it, start the reader thread and get SMuFF's firmware information close_SMuFF_serial() open_SMuFF_serial(port_new, baud_new) self._serial = __ser0__ start_reader_thread() self._fw_info = self.send_SMuFF_and_wait(M115) def get_template_configs(self): # self._logger.debug("Settings-Template was requested") return [ dict(type="settings", custom_bindings=True, template='SMuFF_settings.jinja2'), dict(type="navbar", custom_bindings=True, template='SMuFF_navbar.jinja2') ] ##~~ AssetPlugin mixin def get_assets(self): return dict(js=["js/SMuFF.js"], css=["css/SMuFF.css"], less=["less/SMuFF.less"]) ##~~ Softwareupdate hook def get_update_information(self): return dict(SMuFF=dict( displayName="SMuFF Plugin", displayVersion=self._plugin_version, # version check: github repository type="github_release", user="******", repo="OctoPrint-Smuff", current=self._plugin_version, # update method: pip pip= "https://github.com/technik-gegg/OctoPrint-Smuff/archive/{target_version}.zip" )) ##~~ GCode hooks def extend_tool_queuing(self, comm_instance, phase, cmd, cmd_type, gcode, subcode, tags, *args, **kwargs): # self._logger.debug("Processing queuing: [" + cmd + "," + str(cmd_type)+ "," + str(tags) + "]") if gcode and gcode.startswith(TOOL): self._logger.debug("OctoPrint current tool: {0}".format( comm_instance._currentTool)) # if the tool that's already loaded is addressed, ignore the filament change if cmd == self._cur_tool and self._feeder: self._logger.info(cmd + " equals " + self._cur_tool + " -- no tool change needed") return "M117 Tool already selected" self._is_aligned = False # replace the tool change command return [AT_SMUFF + " " + cmd] if cmd and cmd.startswith(AT_SMUFF): v1 = None v2 = None spd = 300 action = None tmp = cmd.split() if len(tmp): action = tmp[1] if len(tmp) > 2: v1 = int(tmp[2]) if len(tmp) > 3: v2 = int(tmp[3]) if len(tmp) > 4: spd = int(tmp[4]) # self._logger.debug("1>> " + cmd + " action: " + str(action) + " v1,v2: " + str(v1) + ", " + str(v2)) # @SMuFF MOTORS if action and action == MOTORS: # send a servo command to SMuFF self.send_SMuFF_and_wait(M18) return "" # @SMuFF FAN if action and action == FAN: # send a fan command to SMuFF if v1 == 1: self.send_SMuFF_and_wait(M106) elif v1 == 0: self.send_SMuFF_and_wait(M107) return "" def extend_tool_sending(self, comm_instance, phase, cmd, cmd_type, gcode, subcode, tags, *args, **kwargs): if gcode and gcode.startswith(TOOL): return "" # is this the replaced tool change command? if cmd and cmd.startswith(AT_SMUFF): v1 = None v2 = None spd = 300 action = None tmp = cmd.split() if len(tmp): action = tmp[1] if len(tmp) > 2: v1 = int(tmp[2]) if len(tmp) > 3: v2 = int(tmp[3]) if len(tmp) > 4: spd = int(tmp[4]) self._logger.debug("2>> " + cmd + " action: " + str(action) + " v1,v2: " + str(v1) + ", " + str(v2)) # @SMuFF T0...T99 if action and action.startswith(TOOL): if self._printer.set_job_on_hold(True, False): try: # store the new tool for later self._pending_tool = str(action) # check if there's filament loaded if self._feeder: self._logger.debug( "2>> calling script 'beforeToolChange'") # if so, send the OctoPrints default "Before Tool Change" script to the printer self._printer.script("beforeToolChange") else: self._logger.debug("2>> calling SMuFF LOAD") # not loaded, nothing to retract, so send the tool change to the SMuFF self._printer.commands(AT_SMUFF + " " + LOAD) except UnknownScript: self._logger.error( "Script 'beforeToolChange' not found!") # Notice: set_job_on_hold(False) must not be set yet since the # whole tool change procedure isn't finished yet. Setting it now # will unpause OctoPrint and it'll continue printing without filament! #finally: #self._printer.set_job_on_hold(False) # @SMuFF SERVO if action and action == SERVO: # send a servo command to SMuFF self.send_SMuFF_and_wait(M280 + str(v1) + " S" + str(v2)) return "" # @SMuFF SERVOOPEN if action and action == SERVOOPEN: # send a servo open command to SMuFF self.send_SMuFF_and_wait(M280 + str(v1) + " R0") return "" # @SMuFF SERVOCLOSE if action and action == SERVOCLOSE: # send a servo close command to SMuFF self.send_SMuFF_and_wait(M280 + str(v1) + " R1") return "" # @SMuFF WIPE if action and action == WIPE: # send a servo wipe command to SMuFF self.send_SMuFF_and_wait(G12) # @SMuFF LOAD if action and action == LOAD: with self._printer.job_on_hold(): try: self._logger.debug("1>> LOAD: Feeder: " + str(self._feeder) + ", Pending: " + str(self._pending_tool) + ", Current: " + str(self._cur_tool)) retry = 3 # retry up to 3 times while retry > 0: # send a tool change command to SMuFF res = self.send_SMuFF_and_wait(self._pending_tool) # do we have the tool requested now? if str(res) == str(self._pending_tool): self._pre_tool = self._cur_tool self._cur_tool = self._pending_tool comm_instance._currentTool = self.parse_tool_number( self._cur_tool) self._logger.debug( "2>> calling script 'afterToolChange'") # send the default OctoPrint "After Tool Change" script to the printer self._printer.script("afterToolChange") retry = 0 else: # not the result expected, do it all again self._logger.warning( "Tool change failed, retrying (<{0}> not <{1}>)" .format(res, self._pending_tool)) if self._abort_requested: retry = 0 if not self._wait_requested: retry -= 1 except UnknownScript: # shouldn't happen at all, since we're using default OctoPrint scripts # but you never know self._logger.error( "Script 'afterToolChange' not found!") finally: # now is the time to release the hold and continue printing self._printer.set_job_on_hold(False) def extend_script_variables(self, comm_instance, script_type, script_name, *args, **kwargs): # This section was supposed to pass the current endstop states to OctoPrint scripts when requested. # Unfortunatelly, OctoPrint caches the variables after the very first call and doesn't update # them on consecutive calls while in the same script. # This renders my attempt aligning the position of the filament based on the Feeder endstop trigger # unusable until OctoPrint updates this feature someday in the future. if script_type and script_type == "gcode": variables = dict(feeder="on" if self._feeder else "off", feeder2="on" if self._feeder2 else "off", tool=self._cur_tool, aligned="yes" if self._is_aligned else "no") #self._logger.debug(" >> Script vars query: [" + str(script_type) + "," + str(script_name) + "] {0}".format(variables)) return None, None, variables return None def extend_gcode_received(self, comm_instance, line, *args, **kwargs): # Refresh the current tool in OctoPrint on each command coming from the printer - just in case # This is needed because OctoPrint manages the current tool itself and it might try to swap # tools because of the wrong information. comm_instance._currentTool = self.parse_tool_number(self._cur_tool) # don't process any of the GCodes received return line ##~~ helper functions # sending data to SMuFF def send_SMuFF(self, data): if self._serial and self._serial.is_open: try: b = bytearray(80) # not expecting commands longer then that b = "{}\n".format(data).encode("ascii") # lock down the reader thread just in case # (shouldn't be needed at all, since simultanous read/write operations should # be no big deal) self._serlock.acquire() n = self._serial.write(b) self._serlock.release() self._logger.debug("Sending {1} bytes: [{0}]".format(b, n)) return True except (OSError, serial.SerialException): self._serlock.release() self._logger.error("Can't send command to SMuFF") return False else: self._logger.error("Serial not open") return False # sending data to SMuFF and waiting for a response def send_SMuFF_and_wait(self, data): if self.send_SMuFF(data) == False: return None timeout = 10 # wait max. 10 seconds for a response done = False resp = None self.set_busy(False) # reset busy and self.set_error(False) # error flags while not done: self._serevent.clear() is_set = self._serevent.wait(timeout) if is_set: self._logger.info( "To [{0}] SMuFF says [{1}] (is_error = {2})".format( data, self._response, self._is_error)) resp = self._response if self._response == None or self._is_error: done = True elif not self._response.startswith('echo:'): done = True self._response = None else: self._logger.info("No event received... aborting") if self._is_busy == False: done = True return resp def find_file(self, pattern, path): result = [] for root, dirs, files in os.walk(path): for name in files: if fnmatch.fnmatch(name, pattern): result.append(os.path.join(root, name)) return result def set_busy(self, busy): self._is_busy = busy def set_error(self, error): self._is_error = error def set_response(self, response): self._response = response def parse_states(self, states): #self._logger.debug("States received: [" + states + "]") if len(states) == 0: return False # Note: SMuFF sends periodically states in this notation: # "echo: states: T: T4 S: off R: off F: off F2: off" m = re.search( r'^((\w+:.)(\w+:))\s([T]:\s)(\w+)\s([S]:\s)(\w+)\s([R]:\s)(\w+)\s([F]:\s)(\w+)\s([F,2]+:\s)(\w+)', states) if m: self._cur_tool = m.group(5).strip() self._selector = m.group(7).strip() == ESTOP_ON self._revolver = m.group(9).strip() == ESTOP_ON self._feeder = m.group(11).strip() == ESTOP_ON self._feeder2 = m.group(13).strip() == ESTOP_ON if hasattr(self, "_plugin_manager"): self._plugin_manager.send_plugin_message( self._identifier, { 'type': 'status', 'tool': self._cur_tool, 'feeder': self._feeder, 'feeder2': self._feeder2, 'fw_info': self._fw_info, 'conn': self._serial.is_open }) return True else: if hasattr(self, "_logger"): self._logger.error("No match in parse_states: [" + states + "]") return False def parse_tool_number(self, tool): try: return int(re.findall(r'[-\d]+', tool)[0]) except Exception: self._logger.error("Can't parse tool number in {0}".format(tool)) return -1 def hex_dump(self, s): self._logger.debug(":".join("{:02x}".format(ord(c)) for c in s)) # parse the response we've got from the SMuFF def parse_serial_data(self, data): if self == None: return #self._logger.debug("Raw data: [{0}]".format(data.rstrip("\n"))) global last_response self._serevent.clear() # after first connect the response from the SMuFF is supposed to be 'start' if data.startswith('start\n'): self._logger.debug("SMuFF has sent \"start\" response") return if data.startswith("echo:"): # don't process any general debug messages if data[6:].startswith("dbg:"): self._logger.debug("SMuFF has sent a debug response: [" + data.rstrip() + "]") # but do process the tool/endstop states elif data[6:].startswith("states:"): last_response = None self.parse_states(data.rstrip()) # and register whether SMuFF is busy elif data[6:].startswith("busy"): self._logger.debug("SMuFF has sent a busy response: [" + data.rstrip() + "]") self.set_busy(True) return if data.startswith("error:"): self._logger.info("SMuFF has sent a error response: [" + data.rstrip() + "]") # maybe the SMuFF has received garbage if data[7:].startswith("Unknown command:"): self._serial.flushOutput() self.set_error(True) return if data.startswith(ACTION_CMD): self._logger.info("SMuFF has sent an action request: [" + data.rstrip() + "]") # what action is it? is it a tool change? if data[10:].startswith("T"): tool = self.parse_tool_number(data[10:]) # only if the printer isn't printing state = self._printer.get_state_id() self._logger.debug( "SMuFF requested an action while printer in state '{}'". format(state)) if state == "OPERATIONAL": # query the nozzle temp temps = self._printer.get_current_temperatures() try: if temps['tool0']['actual'] > 160: self._logger.debug("Nozzle temp. > 160") self._printer.change_tool("tool{}".format(tool)) self.send_SMuFF("{0} T: OK".format(ACTION_CMD)) else: self._logger.error( "Can't change to tool {}, nozzle too cold". format(tool)) self.send_SMuFF( "{0} T: \"Nozzle too cold\"".format( ACTION_CMD)) except: self._logger.debug( "Can't query temperatures. Aborting.") self.send_SMuFF( "{0} T: \"No nozzle temp. avail.\"".format( ACTION_CMD)) else: self._logger.error( "Can't change to tool {}, printer not ready or printing" .format(tool)) self.send_SMuFF( "{0} T: \"Printer not ready\"".format(ACTION_CMD)) if data[10:].startswith(ACTION_WAIT): self._wait_requested = True self._logger.debug("waiting for SMuFF to come clear...") if data[10:].startswith(ACTION_CONTINUE): self._wait_requested = False self._abort_requested = False self._logger.debug("continuing after SMuFF cleared...") if data[10:].startswith(ACTION_ABORT): self._wait_requested = False self._abort_requested = True self._logger.debug("aborting operation...") return if data.startswith("ok\n"): if self._is_error: self.set_response(None) else: self.set_response(last_response) self._serevent.set() return # store the last response before the "ok" last_response = data.rstrip("\n") self._logger.debug("Received response: [{0}]".format(last_response))
class Tegrastats: """ - Subprocess read: https://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python/4896288#4896288 - Property https://www.programiz.com/python-programming/property """ def __init__(self, callback, tegrastats_path): self._running = Event() # Error message from thread self._error = None # Start process tegrastats self.path = locate_commands("tegrastats", tegrastats_path) # Define Tegrastats process self._thread = None # Initialize callback self.callback = callback def _decode(self, text): # Find and parse all single values stats = VALS(text) # Parse if exist MTS mts = MTS(text) if mts: stats['MTS'] = mts # Parse RAM stats['RAM'] = RAM(text) # If exists parse SWAP swap = SWAP(text) if swap: stats['SWAP'] = swap # If exists parse IRAM iram = IRAM(text) if iram: stats['IRAM'] = iram # Parse CPU status stats['CPU'] = CPUS(text) # Parse temperatures stats['TEMP'] = TEMPS(text) # Parse Watts stats['WATT'] = WATTS(text) return stats def _read_tegrastats(self, interval, running): pts = sp.Popen( [self.path, '--interval', str(interval)], stdout=sp.PIPE) try: # Reading loop while running.is_set(): if pts.poll() is not None: continue out = pts.stdout if out is not None: # Read line process output line = out.readline() # Decode line in UTF-8 tegrastats_data = line.decode("utf-8") # Decode and store stats = self._decode(tegrastats_data) # Launch callback self.callback(stats) except AttributeError: pass except IOError: pass except Exception: # Write error message self._error = sys.exc_info() finally: # Kill process try: pts.kill() except OSError: pass def open(self, interval=0.5): if self._thread is not None: return False # Set timeout interval = int(interval * 1000) # Check if thread or process exist self._running.set() # Start thread Service client self._thread = Thread(target=self._read_tegrastats, args=( interval, self._running, )) self._thread.start() return True def close(self, timeout=None): # Catch exception if exist if self._error: # Extract exception and raise ex_type, ex_value, tb_str = self._error ex_value.__traceback__ = tb_str raise ex_value # Check if thread and process are already empty self._running.clear() if self._thread is not None: self._thread.join(timeout) self._thread = None return True
class Service(object): """Celery periodic task service.""" scheduler_cls = PersistentScheduler def __init__(self, app, max_interval=None, schedule_filename=None, scheduler_cls=None): self.app = app self.max_interval = (max_interval or app.conf.beat_max_loop_interval) self.scheduler_cls = scheduler_cls or self.scheduler_cls self.schedule_filename = (schedule_filename or app.conf.beat_schedule_filename) self._is_shutdown = Event() self._is_stopped = Event() def __reduce__(self): return self.__class__, (self.max_interval, self.schedule_filename, self.scheduler_cls, self.app) def start(self, embedded_process=False): info('beat: Starting...') debug('beat: Ticking with max interval->%s', humanize_seconds(self.scheduler.max_interval)) signals.beat_init.send(sender=self) if embedded_process: signals.beat_embedded_init.send(sender=self) platforms.set_process_title('celery beat') try: while not self._is_shutdown.is_set(): interval = self.scheduler.tick() if interval and interval > 0.0: debug('beat: Waking up %s.', humanize_seconds(interval, prefix='in ')) time.sleep(interval) if self.scheduler.should_sync(): self.scheduler._do_sync() except (KeyboardInterrupt, SystemExit): self._is_shutdown.set() finally: self.sync() def sync(self): self.scheduler.close() self._is_stopped.set() def stop(self, wait=False): info('beat: Shutting down...') self._is_shutdown.set() wait and self._is_stopped.wait() # block until shutdown done. def get_scheduler(self, lazy=False, extension_namespace='celery.beat_schedulers'): filename = self.schedule_filename aliases = dict(load_extension_class_names(extension_namespace) or {}) return symbol_by_name(self.scheduler_cls, aliases=aliases)( app=self.app, schedule_filename=filename, max_interval=self.max_interval, lazy=lazy, ) @cached_property def scheduler(self): return self.get_scheduler()
class Dispatcher(object): """This class dispatches all kinds of updates to its registered handlers. Attributes: bot (:class:`telegram.Bot`): The bot object that should be passed to the handlers. update_queue (:obj:`Queue`): The synchronized queue that will contain the updates. job_queue (:class:`telegram.ext.JobQueue`): Optional. The :class:`telegram.ext.JobQueue` instance to pass onto handler callbacks. workers (:obj:`int`): Number of maximum concurrent worker threads for the ``@run_async`` decorator. user_data (:obj:`defaultdict`): A dictionary handlers can use to store data for the user. chat_data (:obj:`defaultdict`): A dictionary handlers can use to store data for the chat. persistence (:class:`telegram.ext.BasePersistence`): Optional. The persistence class to store data that should be persistent over restarts Args: bot (:class:`telegram.Bot`): The bot object that should be passed to the handlers. update_queue (:obj:`Queue`): The synchronized queue that will contain the updates. job_queue (:class:`telegram.ext.JobQueue`, optional): The :class:`telegram.ext.JobQueue` instance to pass onto handler callbacks. workers (:obj:`int`, optional): Number of maximum concurrent worker threads for the ``@run_async`` decorator. defaults to 4. persistence (:class:`telegram.ext.BasePersistence`, optional): The persistence class to store data that should be persistent over restarts use_context (:obj:`bool`, optional): If set to ``True`` Use the context based callback API. During the deprecation period of the old API the default is ``False``. **New users**: set this to ``True``. """ __singleton_lock = Lock() __singleton_semaphore = BoundedSemaphore() __singleton = None logger = logging.getLogger(__name__) def __init__(self, bot, update_queue, workers=4, exception_event=None, job_queue=None, persistence=None, use_context=False): self.bot = bot self.update_queue = update_queue self.job_queue = job_queue self.workers = workers self.use_context = use_context if not use_context: warnings.warn( 'Old Handler API is deprecated - see https://git.io/fxJuV for details', TelegramDeprecationWarning, stacklevel=3) self.user_data = defaultdict(dict) """:obj:`dict`: A dictionary handlers can use to store data for the user.""" self.chat_data = defaultdict(dict) if persistence: if not isinstance(persistence, BasePersistence): raise TypeError( "persistence should be based on telegram.ext.BasePersistence" ) self.persistence = persistence if self.persistence.store_user_data: self.user_data = self.persistence.get_user_data() if not isinstance(self.user_data, defaultdict): raise ValueError("user_data must be of type defaultdict") if self.persistence.store_chat_data: self.chat_data = self.persistence.get_chat_data() if not isinstance(self.chat_data, defaultdict): raise ValueError("chat_data must be of type defaultdict") else: self.persistence = None self.handlers = {} """Dict[:obj:`int`, List[:class:`telegram.ext.Handler`]]: Holds the handlers per group.""" self.groups = [] """List[:obj:`int`]: A list with all groups.""" self.error_handlers = [] """List[:obj:`callable`]: A list of errorHandlers.""" self.running = False """:obj:`bool`: Indicates if this dispatcher is running.""" self.__stop_event = Event() self.__exception_event = exception_event or Event() self.__async_queue = Queue() self.__async_threads = set() # For backward compatibility, we allow a "singleton" mode for the dispatcher. When there's # only one instance of Dispatcher, it will be possible to use the `run_async` decorator. with self.__singleton_lock: if self.__singleton_semaphore.acquire(blocking=0): self._set_singleton(self) else: self._set_singleton(None) @property def exception_event(self): return self.__exception_event def _init_async_threads(self, base_name, workers): base_name = '{}_'.format(base_name) if base_name else '' for i in range(workers): thread = Thread(target=self._pooled, name='Bot:{}:worker:{}{}'.format( self.bot.id, base_name, i)) self.__async_threads.add(thread) thread.start() @classmethod def _set_singleton(cls, val): cls.logger.debug('Setting singleton dispatcher as %s', val) cls.__singleton = weakref.ref(val) if val else None @classmethod def get_instance(cls): """Get the singleton instance of this class. Returns: :class:`telegram.ext.Dispatcher` Raises: RuntimeError """ if cls.__singleton is not None: return cls.__singleton() # pylint: disable=not-callable else: raise RuntimeError( '{} not initialized or multiple instances exist'.format( cls.__name__)) def _pooled(self): thr_name = current_thread().getName() while 1: promise = self.__async_queue.get() # If unpacking fails, the thread pool is being closed from Updater._join_async_threads if not isinstance(promise, Promise): self.logger.debug("Closing run_async thread %s/%d", thr_name, len(self.__async_threads)) break promise.run() if isinstance(promise.exception, DispatcherHandlerStop): self.logger.warning( 'DispatcherHandlerStop is not supported with async functions; func: %s', promise.pooled_function.__name__) def run_async(self, func, *args, **kwargs): """Queue a function (with given args/kwargs) to be run asynchronously. Warning: If you're using @run_async you cannot rely on adding custom attributes to :class:`telegram.ext.CallbackContext`. See its docs for more info. Args: func (:obj:`callable`): The function to run in the thread. *args (:obj:`tuple`, optional): Arguments to `func`. **kwargs (:obj:`dict`, optional): Keyword arguments to `func`. Returns: Promise """ # TODO: handle exception in async threads # set a threading.Event to notify caller thread promise = Promise(func, args, kwargs) self.__async_queue.put(promise) return promise def start(self, ready=None): """Thread target of thread 'dispatcher'. Runs in background and processes the update queue. Args: ready (:obj:`threading.Event`, optional): If specified, the event will be set once the dispatcher is ready. """ if self.running: self.logger.warning('already running') if ready is not None: ready.set() return if self.__exception_event.is_set(): msg = 'reusing dispatcher after exception event is forbidden' self.logger.error(msg) raise TelegramError(msg) self._init_async_threads(uuid4(), self.workers) self.running = True self.logger.debug('Dispatcher started') if ready is not None: ready.set() while 1: try: # Pop update from update queue. update = self.update_queue.get(True, 1) except Empty: if self.__stop_event.is_set(): self.logger.debug('orderly stopping') break elif self.__exception_event.is_set(): self.logger.critical( 'stopping due to exception in another thread') break continue self.logger.debug('Processing Update: %s' % update) self.process_update(update) self.update_queue.task_done() self.running = False self.logger.debug('Dispatcher thread stopped') def stop(self): """Stops the thread.""" if self.running: self.__stop_event.set() while self.running: sleep(0.1) self.__stop_event.clear() # async threads must be join()ed only after the dispatcher thread was joined, # otherwise we can still have new async threads dispatched threads = list(self.__async_threads) total = len(threads) # Stop all threads in the thread pool by put()ting one non-tuple per thread for i in range(total): self.__async_queue.put(None) for i, thr in enumerate(threads): self.logger.debug('Waiting for async thread {0}/{1} to end'.format( i + 1, total)) thr.join() self.__async_threads.remove(thr) self.logger.debug('async thread {0}/{1} has ended'.format( i + 1, total)) @property def has_running_threads(self): return self.running or bool(self.__async_threads) def process_update(self, update): """Processes a single update. Args: update (:obj:`str` | :class:`telegram.Update` | :class:`telegram.TelegramError`): The update to process. """ def persist_update(update): """Persist a single update. Args: update (:class:`telegram.Update`): The update to process. """ if self.persistence and isinstance(update, Update): if self.persistence.store_chat_data and update.effective_chat: chat_id = update.effective_chat.id try: self.persistence.update_chat_data( chat_id, self.chat_data[chat_id]) except Exception as e: try: self.dispatch_error(update, e) except Exception: message = 'Saving chat data raised an error and an ' \ 'uncaught error was raised while handling ' \ 'the error with an error_handler' self.logger.exception(message) if self.persistence.store_user_data and update.effective_user: user_id = update.effective_user.id try: self.persistence.update_user_data( user_id, self.user_data[user_id]) except Exception as e: try: self.dispatch_error(update, e) except Exception: message = 'Saving user data raised an error and an ' \ 'uncaught error was raised while handling ' \ 'the error with an error_handler' self.logger.exception(message) # An error happened while polling if isinstance(update, TelegramError): try: self.dispatch_error(None, update) except Exception: self.logger.exception( 'An uncaught error was raised while handling the error') return context = None for group in self.groups: try: for handler in self.handlers[group]: check = handler.check_update(update) if check is not None and check is not False: if not context and self.use_context: context = CallbackContext.from_update(update, self) handler.handle_update(update, self, check, context) persist_update(update) break # Stop processing with any other handler. except DispatcherHandlerStop: self.logger.debug( 'Stopping further handlers due to DispatcherHandlerStop') persist_update(update) break # Dispatch any error. except Exception as e: try: self.dispatch_error(update, e) except DispatcherHandlerStop: self.logger.debug('Error handler stopped further handlers') break # Errors should not stop the thread. except Exception: self.logger.exception( 'An error was raised while processing the update and an ' 'uncaught error was raised while handling the error ' 'with an error_handler') def add_handler(self, handler, group=DEFAULT_GROUP): """Register a handler. TL;DR: Order and priority counts. 0 or 1 handlers per group will be used. A handler must be an instance of a subclass of :class:`telegram.ext.Handler`. All handlers are organized in groups with a numeric value. The default group is 0. All groups will be evaluated for handling an update, but only 0 or 1 handler per group will be used. If :class:`telegram.ext.DispatcherHandlerStop` is raised from one of the handlers, no further handlers (regardless of the group) will be called. The priority/order of handlers is determined as follows: * Priority of the group (lower group number == higher priority) * The first handler in a group which should handle an update (see :attr:`telegram.ext.Handler.check_update`) will be used. Other handlers from the group will not be used. The order in which handlers were added to the group defines the priority. Args: handler (:class:`telegram.ext.Handler`): A Handler instance. group (:obj:`int`, optional): The group identifier. Default is 0. """ # Unfortunately due to circular imports this has to be here from .conversationhandler import ConversationHandler if not isinstance(handler, Handler): raise TypeError('handler is not an instance of {0}'.format( Handler.__name__)) if not isinstance(group, int): raise TypeError('group is not int') if isinstance(handler, ConversationHandler) and handler.persistent: if not self.persistence: raise ValueError( "Conversationhandler {} can not be persistent if dispatcher has no " "persistence".format(handler.name)) handler.conversations = self.persistence.get_conversations( handler.name) handler.persistence = self.persistence if group not in self.handlers: self.handlers[group] = list() self.groups.append(group) self.groups = sorted(self.groups) self.handlers[group].append(handler) def remove_handler(self, handler, group=DEFAULT_GROUP): """Remove a handler from the specified group. Args: handler (:class:`telegram.ext.Handler`): A Handler instance. group (:obj:`object`, optional): The group identifier. Default is 0. """ if handler in self.handlers[group]: self.handlers[group].remove(handler) if not self.handlers[group]: del self.handlers[group] self.groups.remove(group) def update_persistence(self): """Update :attr:`user_data` and :attr:`chat_data` in :attr:`persistence`. """ if self.persistence: if self.persistence.store_chat_data: for chat_id in self.chat_data: self.persistence.update_chat_data(chat_id, self.chat_data[chat_id]) if self.persistence.store_user_data: for user_id in self.user_data: self.persistence.update_user_data(user_id, self.user_data[user_id]) def add_error_handler(self, callback): """Registers an error handler in the Dispatcher. This handler will receive every error which happens in your bot. Warning: The errors handled within these handlers won't show up in the logger, so you need to make sure that you reraise the error. Args: callback (:obj:`callable`): The callback function for this error handler. Will be called when an error is raised. Callback signature for context based API: ``def callback(update: Update, context: CallbackContext)`` The error that happened will be present in context.error. Note: See https://git.io/fxJuV for more info about switching to context based API. """ self.error_handlers.append(callback) def remove_error_handler(self, callback): """Removes an error handler. Args: callback (:obj:`callable`): The error handler to remove. """ if callback in self.error_handlers: self.error_handlers.remove(callback) def dispatch_error(self, update, error): """Dispatches an error. Args: update (:obj:`str` | :class:`telegram.Update` | None): The update that caused the error error (:obj:`Exception`): The error that was raised. """ if self.error_handlers: for callback in self.error_handlers: if self.use_context: callback(update, CallbackContext.from_error(update, error, self)) else: callback(self.bot, update, error) else: self.logger.exception( 'No error handlers are registered, logging exception.', exc_info=error)
class Reporter(InterfaceBase, AbstractContextManager, SetupUploadMixin, AsyncManagerMixin): """ A simple metrics reporter class. This class caches reports and supports both a explicit flushing and context-based flushing. To ensure reports are sent to the backend, please use (assuming an instance of Reporter named 'reporter'): - use the context manager feature (which will automatically flush when exiting the context): with reporter: reporter.report... ... - explicitly call flush: reporter.report... ... reporter.flush() """ def __init__(self, metrics, flush_threshold=10, async_enable=False): """ Create a reporter :param metrics: A Metrics manager instance that handles actual reporting, uploads etc. :type metrics: .backend_interface.metrics.Metrics :param flush_threshold: Events flush threshold. This determines the threshold over which cached reported events are flushed and sent to the backend. :type flush_threshold: int """ log = metrics.log.getChild('reporter') log.setLevel(log.level) super(Reporter, self).__init__(session=metrics.session, log=log) self._metrics = metrics self._flush_threshold = flush_threshold self._events = [] self._bucket_config = None self._storage_uri = None self._async_enable = async_enable self._flush_frequency = 30.0 self._exit_flag = False self._flush_event = Event() self._flush_event.clear() self._thread = Thread(target=self._daemon) self._thread.daemon = True self._thread.start() self._max_iteration = 0 def _set_storage_uri(self, value): value = '/'.join(x for x in (value.rstrip('/'), self._metrics.storage_key_prefix) if x) self._storage_uri = value storage_uri = property(None, _set_storage_uri) @property def flush_threshold(self): return self._flush_threshold @flush_threshold.setter def flush_threshold(self, value): self._flush_threshold = max(0, value) @property def async_enable(self): return self._async_enable @async_enable.setter def async_enable(self, value): self._async_enable = bool(value) @property def max_iteration(self): return self._max_iteration def _daemon(self): while not self._exit_flag: self._flush_event.wait(self._flush_frequency) self._flush_event.clear() self._write() # wait for all reports if self.get_num_results() > 0: self.wait_for_results() # make sure we flushed everything self._async_enable = False self._write() if self.get_num_results() > 0: self.wait_for_results() def _report(self, ev): ev_iteration = ev.get_iteration() if ev_iteration is not None: # we have to manually add get_iteration_offset() because event hasn't reached the Metric manager self._max_iteration = max( self._max_iteration, ev_iteration + self._metrics.get_iteration_offset()) self._events.append(ev) if len(self._events) >= self._flush_threshold: self.flush() def _write(self): if not self._events: return # print('reporting %d events' % len(self._events)) res = self._metrics.write_events(self._events, async_enable=self._async_enable, storage_uri=self._storage_uri) if self._async_enable: self._add_async_result(res) self._events = [] def flush(self): """ Flush cached reports to backend. """ self._flush_event.set() def stop(self): self._exit_flag = True self._flush_event.set() self._thread.join() def report_scalar(self, title, series, value, iter): """ Report a scalar value :param title: Title (AKA metric) :type title: str :param series: Series (AKA variant) :type series: str :param value: Reported value :type value: float :param iter: Iteration number :type value: int """ ev = ScalarEvent(metric=self._normalize_name(title), variant=self._normalize_name(series), value=value, iter=iter) self._report(ev) def report_vector(self, title, series, values, iter): """ Report a vector of values :param title: Title (AKA metric) :type title: str :param series: Series (AKA variant) :type series: str :param values: Reported values :type value: [float] :param iter: Iteration number :type value: int """ if not isinstance(values, Iterable): raise ValueError('values: expected an iterable') ev = VectorEvent(metric=self._normalize_name(title), variant=self._normalize_name(series), values=values, iter=iter) self._report(ev) def report_plot(self, title, series, plot, iter): """ Report a Plotly chart :param title: Title (AKA metric) :type title: str :param series: Series (AKA variant) :type series: str :param plot: A JSON describing a plotly chart (see https://help.plot.ly/json-chart-schema/) :type plot: str or dict :param iter: Iteration number :type value: int """ try: def default(o): if isinstance(o, np.int64): return int(o) except Exception: default = None if isinstance(plot, dict): plot = json.dumps(plot, default=default) elif not isinstance(plot, six.string_types): raise ValueError('Plot should be a string or a dict') ev = PlotEvent(metric=self._normalize_name(title), variant=self._normalize_name(series), plot_str=plot, iter=iter) self._report(ev) def report_image(self, title, series, src, iter): """ Report an image. :param title: Title (AKA metric) :type title: str :param series: Series (AKA variant) :type series: str :param src: Image source URI. This URI will be used by the webapp and workers when trying to obtain the image for presentation of processing. Currently only http(s), file and s3 schemes are supported. :type src: str :param iter: Iteration number :type value: int """ ev = ImageEventNoUpload(metric=self._normalize_name(title), variant=self._normalize_name(series), iter=iter, src=src) self._report(ev) def report_image_and_upload(self, title, series, iter, path=None, image=None, upload_uri=None, max_image_history=None, delete_after_upload=False): """ Report an image and upload its contents. Image is uploaded to a preconfigured bucket (see setup_upload()) with a key (filename) describing the task ID, title, series and iteration. :param title: Title (AKA metric) :type title: str :param series: Series (AKA variant) :type series: str :param iter: Iteration number :type iter: int :param path: A path to an image file. Required unless matrix is provided. :type path: str :param image: Image data. Required unless filename is provided. :type image: A PIL.Image.Image object or a 3D numpy.ndarray object :param max_image_history: maximum number of image to store per metric/variant combination use negative value for unlimited. default is set in global configuration (default=5) :param delete_after_upload: if True, one the file was uploaded the local copy will be deleted :type delete_after_upload: boolean """ if not self._storage_uri and not upload_uri: raise ValueError( 'Upload configuration is required (use setup_upload())') if len([x for x in (path, image) if x is not None]) != 1: raise ValueError('Expected only one of [filename, image]') kwargs = dict(metric=self._normalize_name(title), variant=self._normalize_name(series), iter=iter, image_file_history_size=max_image_history) ev = ImageEvent(image_data=image, upload_uri=upload_uri, local_image_path=path, delete_after_upload=delete_after_upload, **kwargs) self._report(ev) def report_histogram(self, title, series, histogram, iter, labels=None, xlabels=None, xtitle=None, ytitle=None, comment=None): """ Report an histogram bar plot :param title: Title (AKA metric) :type title: str :param series: Series (AKA variant) :type series: str :param histogram: The histogram data. A row for each dataset(bar in a bar group). A column for each bucket. :type histogram: numpy array :param iter: Iteration number :type value: int :param labels: The labels for each bar group. :type labels: list of strings. :param xlabels: The labels of the x axis. :type xlabels: List of strings. :param str xtitle: optional x-axis title :param str ytitle: optional y-axis title :param comment: comment underneath the title :type comment: str """ plotly_dict = create_2d_histogram_plot( np_row_wise=histogram, title=title, xtitle=xtitle, ytitle=ytitle, labels=labels, series=series, xlabels=xlabels, comment=comment, ) return self.report_plot( title=self._normalize_name(title), series=self._normalize_name(series), plot=plotly_dict, iter=iter, ) def report_table(self, title, series, table, iteration): """ Report a table plot. :param title: Title (AKA metric) :type title: str :param series: Series (AKA variant) :type series: str :param table: The table data :type table: pandas.DataFrame :param iteration: Iteration number :type iteration: int """ table_output = create_plotly_table(table, title, series) return self.report_plot( title=self._normalize_name(title), series=self._normalize_name(series), plot=table_output, iter=iteration, ) def report_line_plot(self, title, series, iter, xtitle, ytitle, mode='lines', reverse_xaxis=False, comment=None): """ Report a (possibly multiple) line plot. :param title: Title (AKA metric) :type title: str :param series: All the series' data, one for each line in the plot. :type series: An iterable of LineSeriesInfo. :param iter: Iteration number :type iter: int :param xtitle: x-axis title :type xtitle: str :param ytitle: y-axis title :type ytitle: str :param mode: 'lines' / 'markers' / 'lines+markers' :type mode: str :param reverse_xaxis: If true X axis will be displayed from high to low (reversed) :type reverse_xaxis: bool :param comment: comment underneath the title :type comment: str """ plotly_dict = create_line_plot( title=title, series=series, xtitle=xtitle, ytitle=ytitle, mode=mode, reverse_xaxis=reverse_xaxis, comment=comment, ) return self.report_plot( title=self._normalize_name(title), series='', plot=plotly_dict, iter=iter, ) def report_2d_scatter(self, title, series, data, iter, mode='lines', xtitle=None, ytitle=None, labels=None, comment=None): """ Report a 2d scatter graph (with lines) :param title: Title (AKA metric) :type title: str :param series: Series (AKA variant) :type series: str :param data: A scattered data: pairs of x,y as rows in a numpy array :type scatter: ndarray :param iter: Iteration number :type iter: int :param mode: (type str) 'lines'/'markers'/'lines+markers' :param xtitle: optional x-axis title :param ytitle: optional y-axis title :param labels: label (text) per point in the scatter (in the same order) :param comment: comment underneath the title :type comment: str """ plotly_dict = create_2d_scatter_series( np_row_wise=data, title=title, series_name=series, mode=mode, xtitle=xtitle, ytitle=ytitle, labels=labels, comment=comment, ) return self.report_plot( title=self._normalize_name(title), series=self._normalize_name(series), plot=plotly_dict, iter=iter, ) def report_3d_scatter(self, title, series, data, iter, labels=None, mode='lines', color=((217, 217, 217, 0.14), ), marker_size=5, line_width=0.8, xtitle=None, ytitle=None, ztitle=None, fill=None, comment=None): """ Report a 3d scatter graph (with markers) :param title: Title (AKA metric) :type title: str :param series: Series (AKA variant) :type series: str :param data: A scattered data: pairs of x,y,z as rows in a numpy array. or list of numpy arrays :type data: ndarray. :param iter: Iteration number :type iter: int :param labels: label (text) per point in the scatter (in the same order) :type labels: str :param mode: (type str) 'lines'/'markers'/'lines+markers' :param color: list of RGBA colors [(217, 217, 217, 0.14),] :param marker_size: marker size in px :param line_width: line width in px :param xtitle: optional x-axis title :param ytitle: optional y-axis title :param ztitle: optional z-axis title :param comment: comment underneath the title """ data_series = data if isinstance(data, list) else [data] def get_labels(i): if labels and isinstance(labels, list): try: item = labels[i] except IndexError: item = labels[-1] if isinstance(item, list): return item return labels plotly_obj = plotly_scatter3d_layout_dict( title=title, xaxis_title=xtitle, yaxis_title=ytitle, zaxis_title=ztitle, comment=comment, ) for i, values in enumerate(data_series): plotly_obj = create_3d_scatter_series( np_row_wise=values, title=title, series_name=series[i] if isinstance(series, list) else None, labels=get_labels(i), plotly_obj=plotly_obj, mode=mode, line_width=line_width, marker_size=marker_size, color=color, fill_axis=fill, ) return self.report_plot( title=self._normalize_name(title), series=self._normalize_name(series) if not isinstance(series, list) else None, plot=plotly_obj, iter=iter, ) def report_value_matrix(self, title, series, data, iter, xtitle=None, ytitle=None, xlabels=None, ylabels=None, comment=None): """ Report a heat-map matrix :param title: Title (AKA metric) :type title: str :param series: Series (AKA variant) :type series: str :param data: A heat-map matrix (example: confusion matrix) :type data: ndarray :param iter: Iteration number :type iter: int :param str xtitle: optional x-axis title :param str ytitle: optional y-axis title :param xlabels: optional label per column of the matrix :param ylabels: optional label per row of the matrix :param comment: comment underneath the title """ plotly_dict = create_value_matrix( np_value_matrix=data, title=title, xlabels=xlabels, ylabels=ylabels, series=series, comment=comment, xtitle=xtitle, ytitle=ytitle, ) return self.report_plot( title=self._normalize_name(title), series=self._normalize_name(series), plot=plotly_dict, iter=iter, ) def report_value_surface(self, title, series, data, iter, xlabels=None, ylabels=None, xtitle=None, ytitle=None, ztitle=None, camera=None, comment=None): """ Report a 3d surface (same data as heat-map matrix, only presented differently) :param title: Title (AKA metric) :type title: str :param series: Series (AKA variant) :type series: str :param data: A heat-map matrix (example: confusion matrix) :type data: ndarray :param iter: Iteration number :type iter: int :param xlabels: optional label per column of the matrix :param ylabels: optional label per row of the matrix :param xtitle: optional x-axis title :param ytitle: optional y-axis title :param ztitle: optional z-axis title :param camera: X,Y,Z camera position. def: (1,1,1) :param comment: comment underneath the title """ plotly_dict = create_3d_surface( np_value_matrix=data, title=title + '/' + series, xlabels=xlabels, ylabels=ylabels, series=series, xtitle=xtitle, ytitle=ytitle, ztitle=ztitle, camera=camera, comment=comment, ) return self.report_plot( title=self._normalize_name(title), series=self._normalize_name(series), plot=plotly_dict, iter=iter, ) def report_image_plot_and_upload(self, title, series, iter, path=None, matrix=None, upload_uri=None, max_image_history=None, delete_after_upload=False): """ Report an image as plot and upload its contents. Image is uploaded to a preconfigured bucket (see setup_upload()) with a key (filename) describing the task ID, title, series and iteration. Then a plotly object is created and registered, this plotly objects points to the uploaded image :param title: Title (AKA metric) :type title: str :param series: Series (AKA variant) :type series: str :param iter: Iteration number :type value: int :param path: A path to an image file. Required unless matrix is provided. :type path: str :param matrix: A 3D numpy.ndarray object containing image data (RGB). Required unless filename is provided. :type matrix: str :param max_image_history: maximum number of image to store per metric/variant combination use negative value for unlimited. default is set in global configuration (default=5) :param delete_after_upload: if True, one the file was uploaded the local copy will be deleted :type delete_after_upload: boolean """ if not upload_uri and not self._storage_uri: raise ValueError( 'Upload configuration is required (use setup_upload())') if len([x for x in (path, matrix) if x is not None]) != 1: raise ValueError('Expected only one of [filename, matrix]') kwargs = dict(metric=self._normalize_name(title), variant=self._normalize_name(series), iter=iter, image_file_history_size=max_image_history) ev = UploadEvent(image_data=matrix, upload_uri=upload_uri, local_image_path=path, delete_after_upload=delete_after_upload, **kwargs) _, url = ev.get_target_full_upload_uri( upload_uri or self._storage_uri, self._metrics.storage_key_prefix) # Hack: if the url doesn't start with http/s then the plotly will not be able to show it, # then we put the link under images not plots if not url.startswith('http'): return self.report_image_and_upload( title=title, series=series, iter=iter, path=path, image=matrix, upload_uri=upload_uri, max_image_history=max_image_history) self._report(ev) plotly_dict = create_image_plot( image_src=url, title=title + '/' + series, width=matrix.shape[1] if matrix is not None else 640, height=matrix.shape[0] if matrix is not None else 480, ) return self.report_plot( title=self._normalize_name(title), series=self._normalize_name(series), plot=plotly_dict, iter=iter, ) @classmethod def _normalize_name(cls, name): return name def __exit__(self, exc_type, exc_val, exc_tb): # don't flush in case an exception was raised if not exc_type: self.flush()
class virtualterminal: __stop = False __evt_stopping = Event() def __init__(self, manager): self.__evt_stopping = Event() self.__manager = manager """ Starts the worker's threads """ def start(self): self.__print_welcome() self.__spawn_shell_process() return self.__evt_stopping """ Stops the worker's threads """ def stop(self): self.__write('Terminating session...\r\n') self.__stop = True """ The body of this worker """ def __run(self, process, pin): parent = psutil.Process(process.pid) sock_reader = self.__manager.get_io().reader msg = '' errmsg = '' while not self.__stop: try: # read from the socket, terminal stdin, or terminal stdout, whichever produces data first rs, ws, es = select( [sock_reader, process.stdout, process.stderr], [], [], 1) for r in rs: if r is sock_reader: # data arrived from socket # read the data c = r.read(1) if len(c) == 0: # a data-length of 0 means that the socket failed to read, must be closed # exit the handler thread self.stop() break elif ord(c) == 4: # ord(c) of 4 means CTRL^D # # a data-length of 0 means that the socket failed to read, must be closed # # exit the handler thread logger.info( 'Received CTRL^D, terminating terminal...') self.__write('\r\n') self.stop() break else: children = parent.children(recursive=True) if len(children) == 0: # echo back to client self.__write(c.replace('\n', '\r\n')) pin.write(c) sys.stdout.flush() children = parent.children(recursive=True) self.__manager.get_io().echo(len(children) == 0) has_data = select([process.stdout, process.stderr], [], [], 0)[0] != [] if c == '\n' and not has_data and len( children) == 0: self.__write(self.__generate_promptstring()) elif r in [process.stdout, process.stderr]: # data arrived from terminal max = 100 def readAllSoFar(stream, max, retVal=''): while (select([stream], [], [], 0)[0] != [] and len(retVal) <= max): c = stream.read(1) if len(c) == 0: break else: retVal += c return retVal proc_response = readAllSoFar(r, max) if len(proc_response) == 0: self.stop() break #print 'read bytes!' children = parent.children(recursive=False) #print 'has children ' + `len(children)` proc_response = proc_response.replace('\n', '\r\n') #print proc_response #print 'returning terminal response [%s]' % proc_response self.__write('%s' % proc_response) #send only takes string if len(proc_response) < max and len(children) == 0: self.__write(self.__generate_promptstring()) time.sleep( 0.005 ) # Hack currently to fix unknown race condition in CYW library preventing fast output # sys.stderr.flush() # elif r is process.stderr: # error arrived from terminal # print 'reading from error!' # errmsg += process.stderr.read(1) # if len(errmsg) == 0: # logger.info('Received CTRL^D, terminating terminal...') # self.stop() # break # if errmsg.endswith('>>> '): # errmsg = errmsg[:-4] # if errmsg.endswith('\n'): # self.__write('%s' % errmsg) # errmsg = '' except Exception as e: self.__write('\r\nINTERNAL SERVER ERROR: %s\r\n' % e) #send only takes string logger.error(traceback.format_exc()) logger.info('Virtual Terminal has exited.') process.stdout.close() process.stderr.close() process.terminate() self.__evt_stopping.set() """ Spawns the Linux `sh` process, captures its pipes, and creates a thread for routing I/O """ def __spawn_shell_process(self): cwd = '/bin/bash --norc' pw_record = pwd.getpwnam(self.__manager.get_username()) user_name = pw_record.pw_name user_home_dir = pw_record.pw_dir user_uid = pw_record.pw_uid user_gid = pw_record.pw_gid env = os.environ.copy() env['HOME'] = user_home_dir env['LOGNAME'] = user_name env['PWD'] = cwd env['USER'] = user_name logger.debug('Spawning shell process for user %s' % pw_record.pw_name) process = Popen('/bin/sh', preexec_fn=self.__demote(user_uid, user_gid), cwd=user_home_dir, env=env, stdin=PIPE, stdout=PIPE, stderr=PIPE) # grab a file descriptor for the virtual terminal, use this to send data to your virtual terminal self.pin = process.stdin logger.debug('Sending prompt string for user %s' % pw_record.pw_name) self.__write(self.__generate_promptstring()) start_new_thread(self.__run, (process, self.pin)) """ Writes output to the client transport """ def __write(self, text): self.__manager.get_io().write(text) # def write(self, message): # self.__write(self.__generate_promptstring()) # self.pin.write(msg+'\r\n') """ Constructs a Linux-bash style command prompt """ def __generate_promptstring(self): workdir = os.getcwd() if workdir == os.getenv('HOME'): workdir = "~" return '%s@%s:%s# ' % (self.__manager.get_username(), platform.node(), workdir) """ Writes a welcome message to the client transport """ def __print_welcome(self): if os.path.isfile(WELCOME_MESSAGE_PATH): logger.info('Sending welcome message from file: %s' % WELCOME_MESSAGE_PATH) self.__write('\n') with open(WELCOME_MESSAGE_PATH, "r") as welcome_file: for line in welcome_file: self.__write(line.strip('\n') + '\r\n') self.__write('\r\n\r\n') """ Demote child process to target user and group. This ensures Linux fs permissions will be honored. """ def __demote(self, user_uid, user_gid): def result(): os.setgid(user_gid) os.setuid(user_uid) return result
class _WorkerThread(Thread): """WorkerThreads process incoming messages off of the work queue on a loop. By themselves, they don't do any sort of network IO. Parameters: broker(Broker) consumers(dict[str, _ConsumerThread]) work_queue(Queue) worker_timeout(int) """ def __init__(self, *, broker, consumers, work_queue, worker_timeout): super().__init__(daemon=True) self.logger = get_logger(__name__, "WorkerThread") self.running = False self.paused = False self.paused_event = Event() self.broker = broker self.consumers = consumers self.work_queue = work_queue self.timeout = worker_timeout / 1000 def run(self): self.logger.debug("Running worker thread...") self.running = True while self.running: if self.paused: self.logger.debug("Worker is paused. Sleeping for %.02f...", self.timeout) self.paused_event.set() time.sleep(self.timeout) continue try: _, message = self.work_queue.get(timeout=self.timeout) self.process_message(message) except Empty: continue self.broker.emit_before("worker_thread_shutdown", self) self.logger.debug("Worker thread stopped.") def process_message(self, message): """Process a message pulled off of the work queue then push it back to its associated consumer for post processing. Parameters: message(MessageProxy) """ try: self.logger.debug("Received message %s with id %r.", message, message.message_id) self.broker.emit_before("process_message", message) res = None if not message.failed: res = self.call_actor(message) self.broker.emit_after("process_message", message, result=res) except SkipMessage: self.logger.warning("Message %s was skipped.", message) self.broker.emit_after("skip_message", message) except MessageCanceled: self.logger.warning("Message %s has been canceled", message) self.broker.emit_after("message_canceled", message) except BaseException as e: if isinstance(e, RateLimitExceeded): self.logger.warning("Rate limit exceeded in message %s: %s.", message, e) else: self.logger.error( "Failed to process message %s with unhandled exception.", message, exc_info=True, extra={ "input": { "args": str(message.args), "kwargs": str(message.kwargs) } }, ) self.broker.emit_after("process_message", message, exception=e) finally: # NOTE: There is no race here as any message that was # processed must have come off of a consumer. Therefore, # there has to be a consumer for that message's queue so # this is safe. Probably. self.consumers[message.queue_name].post_process_message(message) self.work_queue.task_done() def call_actor(self, message): """Call an actor with the arguments stored in the message Handle the logging of the start/end of the actor with runtime measure Parameters: message(MessageProxy) Returns: Whatever the actor returns. """ actor = self.broker.get_actor(message.actor_name) message_id = message.message_id try: self.logger.info("Started Actor %s", message, extra={"message_id": message_id}) start = time.perf_counter() return actor(*message.args, **message.kwargs) finally: runtime = (time.perf_counter() - start) * 1000 extra = {"message_id": message_id, "runtime": runtime} self.logger.info("Finished Actor %s after %.02fms.", message, runtime, extra=extra) def pause(self): """Pause this worker. """ self.paused = True self.paused_event.clear() def resume(self): """Resume this worker. """ self.paused = False self.paused_event.clear() def stop(self): """Initiate the WorkerThread shutdown process. Code calling this method should then join on the thread and wait for it to finish shutting down. """ self.logger.debug("Stopping worker thread...") self.running = False
NUMBER_OF_FINGERS = 3 NUMBER_OF_JOINTS = 3 initial_angle_ = 0 listeMoteur1_ = [initial_angle_] listeMoteur2_ = [initial_angle_] listeMoteur3_ = [initial_angle_] instructionListe_ = [] command_ = 1 execute = "" # ====== Concurrency variable and thread ======= routine_event_cv_ = Event() routine_event_cv_.set() send_lock_ = RLock() routine_cv_waiting_for_unlock_ = False ready_to_send_bloc_ = False is_play_thread_alive_ = False stop_flag_lock_ = RLock() stop_flag_ = False threads_on_ = True # Record thread class Record: lock = RLock()
class Module(MgrModule): OPTIONS = [ { 'name': 'active' }, { 'name': 'begin_time' }, { 'name': 'crush_compat_max_iterations' }, { 'name': 'crush_compat_metrics' }, { 'name': 'crush_compat_step' }, { 'name': 'end_time' }, { 'name': 'min_score' }, { 'name': 'mode' }, { 'name': 'sleep_interval' }, { 'name': 'upmap_max_iterations' }, { 'name': 'upmap_max_deviation' }, ] COMMANDS = [ { "cmd": "balancer status", "desc": "Show balancer status", "perm": "r", }, { "cmd": "balancer mode name=mode,type=CephChoices,strings=none|crush-compat|upmap", "desc": "Set balancer mode", "perm": "rw", }, { "cmd": "balancer on", "desc": "Enable automatic balancing", "perm": "rw", }, { "cmd": "balancer off", "desc": "Disable automatic balancing", "perm": "rw", }, { "cmd": "balancer eval name=option,type=CephString,req=false", "desc": "Evaluate data distribution for the current cluster or specific pool or specific plan", "perm": "r", }, { "cmd": "balancer eval-verbose name=option,type=CephString,req=false", "desc": "Evaluate data distribution for the current cluster or specific pool or specific plan (verbosely)", "perm": "r", }, { "cmd": "balancer optimize name=plan,type=CephString name=pools,type=CephString,n=N,req=false", "desc": "Run optimizer to create a new plan", "perm": "rw", }, { "cmd": "balancer show name=plan,type=CephString", "desc": "Show details of an optimization plan", "perm": "r", }, { "cmd": "balancer rm name=plan,type=CephString", "desc": "Discard an optimization plan", "perm": "rw", }, { "cmd": "balancer reset", "desc": "Discard all optimization plans", "perm": "rw", }, { "cmd": "balancer dump name=plan,type=CephString", "desc": "Show an optimization plan", "perm": "r", }, { "cmd": "balancer ls", "desc": "List all plans", "perm": "r", }, { "cmd": "balancer execute name=plan,type=CephString", "desc": "Execute an optimization plan", "perm": "rw", }, ] active = False run = True plans = {} mode = '' def __init__(self, *args, **kwargs): super(Module, self).__init__(*args, **kwargs) self.event = Event() def handle_command(self, inbuf, command): self.log.warn("Handling command: '%s'" % str(command)) if command['prefix'] == 'balancer status': s = { 'plans': list(self.plans.keys()), 'active': self.active, 'mode': self.get_config('mode', default_mode), } return (0, json.dumps(s, indent=4), '') elif command['prefix'] == 'balancer mode': self.set_config('mode', command['mode']) return (0, '', '') elif command['prefix'] == 'balancer on': if not self.active: self.set_config('active', '1') self.active = True self.event.set() return (0, '', '') elif command['prefix'] == 'balancer off': if self.active: self.set_config('active', '') self.active = False self.event.set() return (0, '', '') elif command['prefix'] == 'balancer eval' or command[ 'prefix'] == 'balancer eval-verbose': verbose = command['prefix'] == 'balancer eval-verbose' pools = [] if 'option' in command: plan = self.plans.get(command['option']) if not plan: # not a plan, does it look like a pool? osdmap = self.get_osdmap() valid_pool_names = [ p['pool_name'] for p in osdmap.dump().get('pools', []) ] option = command['option'] if option not in valid_pool_names: return (-errno.EINVAL, '', 'option "%s" not a plan or a pool' % option) pools.append(option) ms = MappingState(osdmap, self.get("pg_dump"), 'pool "%s"' % option) else: pools = plan.pools ms = plan.final_state() else: ms = MappingState(self.get_osdmap(), self.get("pg_dump"), 'current cluster') return (0, self.evaluate(ms, pools, verbose=verbose), '') elif command['prefix'] == 'balancer optimize': pools = [] if 'pools' in command: pools = command['pools'] osdmap = self.get_osdmap() valid_pool_names = [ p['pool_name'] for p in osdmap.dump().get('pools', []) ] invalid_pool_names = [] for p in pools: if p not in valid_pool_names: invalid_pool_names.append(p) if len(invalid_pool_names): return (-errno.EINVAL, '', 'pools %s not found' % invalid_pool_names) plan = self.plan_create(command['plan'], osdmap, pools) r, detail = self.optimize(plan) # remove plan if we are currently unable to find an optimization # or distribution is already perfect if r: self.plan_rm(command['plan']) return (r, '', detail) elif command['prefix'] == 'balancer rm': self.plan_rm(command['plan']) return (0, '', '') elif command['prefix'] == 'balancer reset': self.plans = {} return (0, '', '') elif command['prefix'] == 'balancer ls': return (0, json.dumps([p for p in self.plans], indent=4), '') elif command['prefix'] == 'balancer dump': plan = self.plans.get(command['plan']) if not plan: return (-errno.ENOENT, '', 'plan %s not found' % command['plan']) return (0, plan.dump(), '') elif command['prefix'] == 'balancer show': plan = self.plans.get(command['plan']) if not plan: return (-errno.ENOENT, '', 'plan %s not found' % command['plan']) return (0, plan.show(), '') elif command['prefix'] == 'balancer execute': plan = self.plans.get(command['plan']) if not plan: return (-errno.ENOENT, '', 'plan %s not found' % command['plan']) r, detail = self.execute(plan) self.plan_rm(command['plan']) return (r, '', detail) else: return (-errno.EINVAL, '', "Command not found '{0}'".format(command['prefix'])) def shutdown(self): self.log.info('Stopping') self.run = False self.event.set() def time_in_interval(self, tod, begin, end): if begin <= end: return tod >= begin and tod < end else: return tod >= begin or tod < end def serve(self): self.log.info('Starting') while self.run: self.active = self.get_config('active', '') is not '' begin_time = self.get_config('begin_time') or '0000' end_time = self.get_config('end_time') or '2400' timeofday = time.strftime('%H%M', time.localtime()) self.log.debug('Waking up [%s, scheduled for %s-%s, now %s]', "active" if self.active else "inactive", begin_time, end_time, timeofday) sleep_interval = float( self.get_config('sleep_interval', default_sleep_interval)) if self.active and self.time_in_interval(timeofday, begin_time, end_time): self.log.debug('Running') name = 'auto_%s' % time.strftime(TIME_FORMAT, time.gmtime()) plan = self.plan_create(name, self.get_osdmap(), []) r, detail = self.optimize(plan) if r == 0: self.execute(plan) self.plan_rm(name) self.log.debug('Sleeping for %d', sleep_interval) self.event.wait(sleep_interval) self.event.clear() def plan_create(self, name, osdmap, pools): plan = Plan( name, MappingState(osdmap, self.get("pg_dump"), 'plan %s initial' % name), pools) self.plans[name] = plan return plan def plan_rm(self, name): if name in self.plans: del self.plans[name] def calc_eval(self, ms, pools): pe = Eval(ms) pool_rule = {} pool_info = {} for p in ms.osdmap_dump.get('pools', []): if len(pools) and p['pool_name'] not in pools: continue # skip dead or not-yet-ready pools too if p['pool'] not in ms.poolids: continue pe.pool_name[p['pool']] = p['pool_name'] pe.pool_id[p['pool_name']] = p['pool'] pool_rule[p['pool_name']] = p['crush_rule'] pe.pool_roots[p['pool_name']] = [] pool_info[p['pool_name']] = p if len(pool_info) == 0: return pe self.log.debug('pool_name %s' % pe.pool_name) self.log.debug('pool_id %s' % pe.pool_id) self.log.debug('pools %s' % pools) self.log.debug('pool_rule %s' % pool_rule) osd_weight = { a['osd']: a['weight'] for a in ms.osdmap_dump.get('osds', []) if a['weight'] > 0 } # get expected distributions by root actual_by_root = {} rootids = ms.crush.find_takes() roots = [] for rootid in rootids: ls = ms.osdmap.get_pools_by_take(rootid) want = [] # find out roots associating with pools we are passed in for candidate in ls: if candidate in pe.pool_name: want.append(candidate) if len(want) == 0: continue root = ms.crush.get_item_name(rootid) pe.root_pools[root] = [] for poolid in want: pe.pool_roots[pe.pool_name[poolid]].append(root) pe.root_pools[root].append(pe.pool_name[poolid]) pe.root_ids[root] = rootid roots.append(root) weight_map = ms.crush.get_take_weight_osd_map(rootid) adjusted_map = { osd: cw * osd_weight[osd] for osd, cw in six.iteritems(weight_map) if osd in osd_weight and cw > 0 } sum_w = sum(adjusted_map.values()) assert len(adjusted_map) == 0 or sum_w > 0 pe.target_by_root[root] = { osd: w / sum_w for osd, w in six.iteritems(adjusted_map) } actual_by_root[root] = { 'pgs': {}, 'objects': {}, 'bytes': {}, } for osd in pe.target_by_root[root]: actual_by_root[root]['pgs'][osd] = 0 actual_by_root[root]['objects'][osd] = 0 actual_by_root[root]['bytes'][osd] = 0 pe.total_by_root[root] = { 'pgs': 0, 'objects': 0, 'bytes': 0, } self.log.debug('pool_roots %s' % pe.pool_roots) self.log.debug('root_pools %s' % pe.root_pools) self.log.debug('target_by_root %s' % pe.target_by_root) # pool and root actual for pool, pi in six.iteritems(pool_info): poolid = pi['pool'] pm = ms.pg_up_by_poolid[poolid] pgs = 0 objects = 0 bytes = 0 pgs_by_osd = {} objects_by_osd = {} bytes_by_osd = {} for root in pe.pool_roots[pool]: for osd in pe.target_by_root[root]: pgs_by_osd[osd] = 0 objects_by_osd[osd] = 0 bytes_by_osd[osd] = 0 for pgid, up in six.iteritems(pm): for osd in [int(osd) for osd in up]: if osd == CRUSHMap.ITEM_NONE: continue pgs_by_osd[osd] += 1 objects_by_osd[osd] += ms.pg_stat[pgid]['num_objects'] bytes_by_osd[osd] += ms.pg_stat[pgid]['num_bytes'] # pick a root to associate this pg instance with. # note that this is imprecise if the roots have # overlapping children. # FIXME: divide bytes by k for EC pools. for root in pe.pool_roots[pool]: if osd in pe.target_by_root[root]: actual_by_root[root]['pgs'][osd] += 1 actual_by_root[root]['objects'][osd] += ms.pg_stat[ pgid]['num_objects'] actual_by_root[root]['bytes'][osd] += ms.pg_stat[ pgid]['num_bytes'] pgs += 1 objects += ms.pg_stat[pgid]['num_objects'] bytes += ms.pg_stat[pgid]['num_bytes'] pe.total_by_root[root]['pgs'] += 1 pe.total_by_root[root]['objects'] += ms.pg_stat[ pgid]['num_objects'] pe.total_by_root[root]['bytes'] += ms.pg_stat[ pgid]['num_bytes'] break pe.count_by_pool[pool] = { 'pgs': {k: v for k, v in six.iteritems(pgs_by_osd)}, 'objects': {k: v for k, v in six.iteritems(objects_by_osd)}, 'bytes': {k: v for k, v in six.iteritems(bytes_by_osd)}, } pe.actual_by_pool[pool] = { 'pgs': { k: float(v) / float(max(pgs, 1)) for k, v in six.iteritems(pgs_by_osd) }, 'objects': { k: float(v) / float(max(objects, 1)) for k, v in six.iteritems(objects_by_osd) }, 'bytes': { k: float(v) / float(max(bytes, 1)) for k, v in six.iteritems(bytes_by_osd) }, } pe.total_by_pool[pool] = { 'pgs': pgs, 'objects': objects, 'bytes': bytes, } for root in pe.total_by_root: pe.count_by_root[root] = { 'pgs': { k: float(v) for k, v in six.iteritems(actual_by_root[root]['pgs']) }, 'objects': { k: float(v) for k, v in six.iteritems(actual_by_root[root]['objects']) }, 'bytes': { k: float(v) for k, v in six.iteritems(actual_by_root[root]['bytes']) }, } pe.actual_by_root[root] = { 'pgs': { k: float(v) / float(max(pe.total_by_root[root]['pgs'], 1)) for k, v in six.iteritems(actual_by_root[root]['pgs']) }, 'objects': { k: float(v) / float(max(pe.total_by_root[root]['objects'], 1)) for k, v in six.iteritems(actual_by_root[root]['objects']) }, 'bytes': { k: float(v) / float(max(pe.total_by_root[root]['bytes'], 1)) for k, v in six.iteritems(actual_by_root[root]['bytes']) }, } self.log.debug('actual_by_pool %s' % pe.actual_by_pool) self.log.debug('actual_by_root %s' % pe.actual_by_root) # average and stddev and score pe.stats_by_root = { a: pe.calc_stats(b, pe.target_by_root[a], pe.total_by_root[a]) for a, b in six.iteritems(pe.count_by_root) } self.log.debug('stats_by_root %s' % pe.stats_by_root) # the scores are already normalized pe.score_by_root = { r: { 'pgs': pe.stats_by_root[r]['pgs']['score'], 'objects': pe.stats_by_root[r]['objects']['score'], 'bytes': pe.stats_by_root[r]['bytes']['score'], } for r in pe.total_by_root.keys() } self.log.debug('score_by_root %s' % pe.score_by_root) # get the list of score metrics, comma separated metrics = self.get_config('crush_compat_metrics', 'pgs,objects,bytes').split(',') # total score is just average of normalized stddevs pe.score = 0.0 for r, vs in six.iteritems(pe.score_by_root): for k, v in six.iteritems(vs): if k in metrics: pe.score += v pe.score /= len(metrics) * len(roots) return pe def evaluate(self, ms, pools, verbose=False): pe = self.calc_eval(ms, pools) return pe.show(verbose=verbose) def optimize(self, plan): self.log.info('Optimize plan %s' % plan.name) plan.mode = self.get_config('mode', default_mode) max_misplaced = float(self.get_option('target_max_misplaced_ratio')) self.log.info('Mode %s, max misplaced %f' % (plan.mode, max_misplaced)) info = self.get('pg_status') unknown = info.get('unknown_pgs_ratio', 0.0) degraded = info.get('degraded_ratio', 0.0) inactive = info.get('inactive_pgs_ratio', 0.0) misplaced = info.get('misplaced_ratio', 0.0) self.log.debug('unknown %f degraded %f inactive %f misplaced %g', unknown, degraded, inactive, misplaced) if unknown > 0.0: detail = 'Some PGs (%f) are unknown; try again later' % unknown self.log.info(detail) return -errno.EAGAIN, detail elif degraded > 0.0: detail = 'Some objects (%f) are degraded; try again later' % degraded self.log.info(detail) return -errno.EAGAIN, detail elif inactive > 0.0: detail = 'Some PGs (%f) are inactive; try again later' % inactive self.log.info(detail) return -errno.EAGAIN, detail elif misplaced >= max_misplaced: detail = 'Too many objects (%f > %f) are misplaced; ' \ 'try again later' % (misplaced, max_misplaced) self.log.info(detail) return -errno.EAGAIN, detail else: if plan.mode == 'upmap': return self.do_upmap(plan) elif plan.mode == 'crush-compat': return self.do_crush_compat(plan) elif plan.mode == 'none': detail = 'Please do "ceph balancer mode" to choose a valid mode first' self.log.info('Idle') return -errno.ENOEXEC, detail else: detail = 'Unrecognized mode %s' % plan.mode self.log.info(detail) return -errno.EINVAL, detail ## def do_upmap(self, plan): self.log.info('do_upmap') max_iterations = int(self.get_config('upmap_max_iterations', 10)) max_deviation = float(self.get_config('upmap_max_deviation', .01)) ms = plan.initial if len(plan.pools): pools = plan.pools else: # all pools = [ str(i['pool_name']) for i in ms.osdmap_dump.get('pools', []) ] if len(pools) == 0: detail = 'No pools available' self.log.info(detail) return -errno.ENOENT, detail # shuffle pool list so they all get equal (in)attention random.shuffle(pools) self.log.info('pools %s' % pools) inc = plan.inc total_did = 0 left = max_iterations for pool in pools: did = ms.osdmap.calc_pg_upmaps(inc, max_deviation, left, [pool]) total_did += did left -= did if left <= 0: break self.log.info('prepared %d/%d changes' % (total_did, max_iterations)) if total_did == 0: return -errno.EALREADY, 'Unable to find further optimization,' \ 'or distribution is already perfect' return 0, '' def do_crush_compat(self, plan): self.log.info('do_crush_compat') max_iterations = int(self.get_config('crush_compat_max_iterations', 25)) if max_iterations < 1: return -errno.EINVAL, '"crush_compat_max_iterations" must be >= 1' step = float(self.get_config('crush_compat_step', .5)) if step <= 0 or step >= 1.0: return -errno.EINVAL, '"crush_compat_step" must be in (0, 1)' max_misplaced = float(self.get_option('target_max_misplaced_ratio')) min_pg_per_osd = 2 ms = plan.initial osdmap = ms.osdmap crush = osdmap.get_crush() pe = self.calc_eval(ms, plan.pools) min_score_to_optimize = float(self.get_config('min_score', 0)) if pe.score <= min_score_to_optimize: if pe.score == 0: detail = 'Distribution is already perfect' else: detail = 'score %f <= min_score %f, will not optimize' \ % (pe.score, min_score_to_optimize) self.log.info(detail) return -errno.EALREADY, detail # get current osd reweights orig_osd_weight = { a['osd']: a['weight'] for a in ms.osdmap_dump.get('osds', []) } reweighted_osds = [ a for a, b in six.iteritems(orig_osd_weight) if b < 1.0 and b > 0.0 ] # get current compat weight-set weights orig_ws = self.get_compat_weight_set_weights(ms) if not orig_ws: return -errno.EAGAIN, 'compat weight-set not available' orig_ws = {a: b for a, b in six.iteritems(orig_ws) if a >= 0} # Make sure roots don't overlap their devices. If so, we # can't proceed. roots = pe.target_by_root.keys() self.log.debug('roots %s', roots) visited = {} overlap = {} root_ids = {} for root, wm in six.iteritems(pe.target_by_root): for osd in wm: if osd in visited: if osd not in overlap: overlap[osd] = [visited[osd]] overlap[osd].append(root) visited[osd] = root if len(overlap) > 0: detail = 'Some osds belong to multiple subtrees: %s' % \ overlap self.log.error(detail) return -errno.EOPNOTSUPP, detail # rebalance by pgs, objects, or bytes metrics = self.get_config('crush_compat_metrics', 'pgs,objects,bytes').split(',') key = metrics[0] # balancing using the first score metric if key not in ['pgs', 'bytes', 'objects']: self.log.warn( "Invalid crush_compat balancing key %s. Using 'pgs'." % key) key = 'pgs' # go best_ws = copy.deepcopy(orig_ws) best_ow = copy.deepcopy(orig_osd_weight) best_pe = pe left = max_iterations bad_steps = 0 next_ws = copy.deepcopy(best_ws) next_ow = copy.deepcopy(best_ow) while left > 0: # adjust self.log.debug('best_ws %s' % best_ws) random.shuffle(roots) for root in roots: pools = best_pe.root_pools[root] osds = len(best_pe.target_by_root[root]) min_pgs = osds * min_pg_per_osd if best_pe.total_by_root[root][key] < min_pgs: self.log.info( 'Skipping root %s (pools %s), total pgs %d ' '< minimum %d (%d per osd)', root, pools, best_pe.total_by_root[root][key], min_pgs, min_pg_per_osd) continue self.log.info('Balancing root %s (pools %s) by %s' % (root, pools, key)) target = best_pe.target_by_root[root] actual = best_pe.actual_by_root[root][key] queue = sorted(actual.keys(), key=lambda osd: -abs(target[osd] - actual[osd])) for osd in queue: if orig_osd_weight[osd] == 0: self.log.debug('skipping out osd.%d', osd) else: deviation = target[osd] - actual[osd] if deviation == 0: break self.log.debug('osd.%d deviation %f', osd, deviation) weight = best_ws[osd] ow = orig_osd_weight[osd] if actual[osd] > 0: calc_weight = target[osd] / actual[ osd] * weight * ow else: # not enough to go on here... keep orig weight calc_weight = weight / orig_osd_weight[osd] new_weight = weight * (1.0 - step) + calc_weight * step self.log.debug('Reweight osd.%d %f -> %f', osd, weight, new_weight) next_ws[osd] = new_weight if ow < 1.0: new_ow = min( 1.0, max(step + (1.0 - step) * ow, ow + .005)) self.log.debug('Reweight osd.%d reweight %f -> %f', osd, ow, new_ow) next_ow[osd] = new_ow # normalize weights under this root root_weight = crush.get_item_weight(pe.root_ids[root]) root_sum = sum(b for a, b in six.iteritems(next_ws) if a in target.keys()) if root_sum > 0 and root_weight > 0: factor = root_sum / root_weight self.log.debug( 'normalizing root %s %d, weight %f, ' 'ws sum %f, factor %f', root, pe.root_ids[root], root_weight, root_sum, factor) for osd in actual.keys(): next_ws[osd] = next_ws[osd] / factor # recalc plan.compat_ws = copy.deepcopy(next_ws) next_ms = plan.final_state() next_pe = self.calc_eval(next_ms, plan.pools) next_misplaced = next_ms.calc_misplaced_from(ms) self.log.debug('Step result score %f -> %f, misplacing %f', best_pe.score, next_pe.score, next_misplaced) if next_misplaced > max_misplaced: if best_pe.score < pe.score: self.log.debug('Step misplaced %f > max %f, stopping', next_misplaced, max_misplaced) break step /= 2.0 next_ws = copy.deepcopy(best_ws) next_ow = copy.deepcopy(best_ow) self.log.debug( 'Step misplaced %f > max %f, reducing step to %f', next_misplaced, max_misplaced, step) else: if next_pe.score > best_pe.score * 1.0001: bad_steps += 1 if bad_steps < 5 and random.randint(0, 100) < 70: self.log.debug('Score got worse, taking another step') else: step /= 2.0 next_ws = copy.deepcopy(best_ws) next_ow = copy.deepcopy(best_ow) self.log.debug( 'Score got worse, trying smaller step %f', step) else: bad_steps = 0 best_pe = next_pe best_ws = copy.deepcopy(next_ws) best_ow = copy.deepcopy(next_ow) if best_pe.score == 0: break left -= 1 # allow a small regression if we are phasing out osd weights fudge = 0 if next_ow != orig_osd_weight: fudge = .001 if best_pe.score < pe.score + fudge: self.log.info('Success, score %f -> %f', pe.score, best_pe.score) plan.compat_ws = best_ws for osd, w in six.iteritems(best_ow): if w != orig_osd_weight[osd]: self.log.debug('osd.%d reweight %f', osd, w) plan.osd_weights[osd] = w return 0, '' else: self.log.info('Failed to find further optimization, score %f', pe.score) plan.compat_ws = {} return -errno.EDOM, 'Unable to find further optimization, ' \ 'change balancer mode and retry might help' def get_compat_weight_set_weights(self, ms): if not CRUSHMap.have_default_choose_args(ms.crush_dump): # enable compat weight-set first self.log.debug('ceph osd crush weight-set create-compat') result = CommandResult('') self.send_command( result, 'mon', '', json.dumps({ 'prefix': 'osd crush weight-set create-compat', 'format': 'json', }), '') r, outb, outs = result.wait() if r != 0: self.log.error('Error creating compat weight-set') return result = CommandResult('') self.send_command( result, 'mon', '', json.dumps({ 'prefix': 'osd crush dump', 'format': 'json', }), '') r, outb, outs = result.wait() if r != 0: self.log.error('Error dumping crush map') return try: crushmap = json.loads(outb) except: raise RuntimeError('unable to parse crush map') else: crushmap = ms.crush_dump raw = CRUSHMap.get_default_choose_args(crushmap) weight_set = {} for b in raw: bucket = None for t in crushmap['buckets']: if t['id'] == b['bucket_id']: bucket = t break if not bucket: raise RuntimeError('could not find bucket %s' % b['bucket_id']) self.log.debug('bucket items %s' % bucket['items']) self.log.debug('weight set %s' % b['weight_set'][0]) if len(bucket['items']) != len(b['weight_set'][0]): raise RuntimeError( 'weight-set size does not match bucket items') for pos in range(len(bucket['items'])): weight_set[bucket['items'][pos] ['id']] = b['weight_set'][0][pos] self.log.debug('weight_set weights %s' % weight_set) return weight_set def do_crush(self): self.log.info('do_crush (not yet implemented)') def do_osd_weight(self): self.log.info('do_osd_weight (not yet implemented)') def execute(self, plan): self.log.info('Executing plan %s' % plan.name) commands = [] # compat weight-set if len(plan.compat_ws) and \ not CRUSHMap.have_default_choose_args(plan.initial.crush_dump): self.log.debug('ceph osd crush weight-set create-compat') result = CommandResult('') self.send_command( result, 'mon', '', json.dumps({ 'prefix': 'osd crush weight-set create-compat', 'format': 'json', }), '') r, outb, outs = result.wait() if r != 0: self.log.error('Error creating compat weight-set') return r, outs for osd, weight in six.iteritems(plan.compat_ws): self.log.info( 'ceph osd crush weight-set reweight-compat osd.%d %f', osd, weight) result = CommandResult('') self.send_command( result, 'mon', '', json.dumps({ 'prefix': 'osd crush weight-set reweight-compat', 'format': 'json', 'item': 'osd.%d' % osd, 'weight': [weight], }), '') commands.append(result) # new_weight reweightn = {} for osd, weight in six.iteritems(plan.osd_weights): reweightn[str(osd)] = str(int(weight * float(0x10000))) if len(reweightn): self.log.info('ceph osd reweightn %s', reweightn) result = CommandResult('') self.send_command( result, 'mon', '', json.dumps({ 'prefix': 'osd reweightn', 'format': 'json', 'weights': json.dumps(reweightn), }), '') commands.append(result) # upmap incdump = plan.inc.dump() for pgid in incdump.get('old_pg_upmap_items', []): self.log.info('ceph osd rm-pg-upmap-items %s', pgid) result = CommandResult('foo') self.send_command( result, 'mon', '', json.dumps({ 'prefix': 'osd rm-pg-upmap-items', 'format': 'json', 'pgid': pgid, }), 'foo') commands.append(result) for item in incdump.get('new_pg_upmap_items', []): self.log.info('ceph osd pg-upmap-items %s mappings %s', item['pgid'], item['mappings']) osdlist = [] for m in item['mappings']: osdlist += [m['from'], m['to']] result = CommandResult('foo') self.send_command( result, 'mon', '', json.dumps({ 'prefix': 'osd pg-upmap-items', 'format': 'json', 'pgid': item['pgid'], 'id': osdlist, }), 'foo') commands.append(result) # wait for commands self.log.debug('commands %s' % commands) for result in commands: r, outb, outs = result.wait() if r != 0: self.log.error('execute error: r = %d, detail = %s' % (r, outs)) return r, outs self.log.debug('done') return 0, ''
class SSHReaderThread(Thread): # pylint: disable=too-many-instance-attributes """ Thread that reads data from ssh session socket and forwards it to Queue. It is needed because socket buffer gets overflowed if data is sent faster than watchers can process it, so we have to have Queue as a buffer with 'endless' memory, and fast reader that reads data from the socket and forward it to the Queue. As part of this process it splits data into lines, because watchers expect it is organized in this way. """ def __init__(self, session: Session, channel: Channel, timeout: NullableTiming, timeout_read_data: NullableTiming): self.stdout = Queue() self.stderr = Queue() self.timeout_reached = False self._session = session self._channel = channel self._timeout = timeout self._timeout_read_data = timeout_read_data self.raised = None self._can_run = Event() self._can_run.set() super().__init__(daemon=True) def run(self): try: self._read_output(self._session, self._channel, self._timeout, self._timeout_read_data, self.stdout, self.stderr) except Exception as exc: # pylint: disable=broad-except self.raised = exc def _read_output( # pylint: disable=too-many-arguments,too-many-branches self, session: Session, channel: Channel, timeout: NullableTiming, timeout_read_data: NullableTiming, stdout_stream: Queue, stderr_stream: Queue): """Reads data from ssh session, split it into lines and forward lines into stderr ad stdout pipes It is required for it to be fast, that is why there is code duplications and non-pythonic code """ # pylint: disable=too-many-locals stdout_remainder = stderr_remainder = b'' if timeout is None: end_time = float_info.max else: end_time = perf_counter() + timeout eof_result = stdout_size = stderr_size = 1 while eof_result == LIBSSH2_ERROR_EAGAIN or stdout_size == LIBSSH2_ERROR_EAGAIN or \ stdout_size > 0 or stderr_size == LIBSSH2_ERROR_EAGAIN or stderr_size > 0: # pylint: disable=consider-using-in if not self._can_run.is_set(): break if perf_counter() > end_time: self.timeout_reached = True break with session.lock: if stdout_size == LIBSSH2_ERROR_EAGAIN and stderr_size == LIBSSH2_ERROR_EAGAIN: # pylint: disable=consider-using-in session.simple_select(timeout=timeout_read_data) stdout_size, stdout_chunk = channel.read() stderr_size, stderr_chunk = channel.read_stderr() eof_result = channel.eof() if stdout_chunk and stdout_stream is not None: data_splitted = stdout_chunk.split(LINESEP) if len(data_splitted) == 1: stdout_remainder = stdout_remainder + data_splitted.pop() else: if stdout_remainder: stdout_stream.put(stdout_remainder + data_splitted.pop(0)) stdout_remainder = data_splitted.pop() for chunk in data_splitted: stdout_stream.put(chunk) if stderr_chunk and stderr_stream is not None: data_splitted = stderr_chunk.split(LINESEP) if len(data_splitted) == 1: stderr_remainder = stderr_remainder + data_splitted.pop() else: if stderr_remainder: stderr_stream.put(stderr_remainder + data_splitted.pop(0)) stderr_remainder = data_splitted.pop() for chunk in data_splitted: stderr_stream.put(chunk) if stdout_remainder: stdout_stream.put(stdout_remainder) if stderr_remainder: stderr_stream.put(stderr_remainder) def stop(self, timeout: float = None): self._can_run.clear() self.join(timeout)
print("Searching for device...") d = MetaWear(sys.argv[1]) d.connect() print("Connected to " + d.address) s = State(d) print("Configuring device") libmetawear.mbl_mw_settings_set_connection_parameters(s.device.board, 7.5, 7.5, 0, 6000) sleep(1.0) try: s.setup_logger() print("Logging data for 15s") sleep(15.0) s.download_data() except RuntimeError as e: print(e) finally: print("Resetting device") e = Event() s.device.on_disconnect = lambda status: e.set() libmetawear.mbl_mw_debug_reset(s.device.board) e.wait()
class _ConsumerThread(Thread): def __init__(self, *, broker, queue_name, prefetch, work_queue, worker_timeout): super().__init__(daemon=True) self.logger = get_logger(__name__, "ConsumerThread(%s)" % queue_name) self.running = False self.paused = False self.paused_event = Event() self.consumer = None self.broker = broker self.prefetch = prefetch self.queue_name = queue_name self.work_queue = work_queue self.worker_timeout = worker_timeout self.delay_queue = PriorityQueue() # type: PriorityQueue def run(self): self.logger.debug("Running consumer thread...") self.running = True restart_consumer = CONSUMER_RESTART_DELAY_SECS > 0 while self.running: if self.paused: self.logger.debug( "Consumer is paused. Sleeping for %.02fms...", self.worker_timeout) self.paused_event.set() time.sleep(self.worker_timeout / 1000) continue try: self.consumer = self.broker.consume( queue_name=self.queue_name, prefetch=self.prefetch, timeout=self.worker_timeout) for message in self.consumer: if message is not None: self.handle_message(message) elif self.paused: break self.handle_delayed_messages() if not self.running: break except ConnectionError as e: self.logger.critical( "Consumer encountered a connection error: %s", e) self.delay_queue = PriorityQueue() if not restart_consumer: self.stop() except Exception: self.logger.critical( "Consumer encountered an unexpected error.", exc_info=True) # Avoid leaving any open file descriptors around when # an exception occurs. self.close() if not restart_consumer: self.stop() # While the consumer is running (i.e. hasn't been shut down), # try to restart it once every CONSUMER_RESTART_DELAY_SECS second. if self.running and restart_consumer: self.logger.info("Restarting consumer in %0.2f seconds.", CONSUMER_RESTART_DELAY_SECS) self.close() time.sleep(CONSUMER_RESTART_DELAY_SECS) # If it's no longer running, then shut it down gracefully. self.broker.emit_before("consumer_thread_shutdown", self) self.logger.debug("Consumer thread stopped.") def handle_delayed_messages(self): """Enqueue any delayed messages whose eta has passed. """ for eta, message in iter_queue(self.delay_queue): if eta > current_millis(): self.delay_queue.put((eta, message)) self.delay_queue.task_done() break queue_name = q_name(message.queue_name) new_message = message.copy(queue_name=queue_name) del new_message.options["eta"] self.broker.enqueue(new_message) self.post_process_message(message) self.delay_queue.task_done() def handle_message(self, message): """Handle a message received off of the underlying consumer. If the message has an eta, delay it. Otherwise, put it on the work queue. """ try: if "eta" in message.options: self.logger.debug("Pushing message %r onto delay queue.", message.message_id) self.broker.emit_before("delay_message", message) self.delay_queue.put((message.options.get("eta", 0), message)) else: actor = self.broker.get_actor(message.actor_name) self.logger.debug("Pushing message %r onto work queue.", message.message_id) self.work_queue.put((-actor.priority, message)) except ActorNotFound: self.logger.error( "Received message for undefined actor %r. Moving it to the DLQ.", message.actor_name, exc_info=True, ) message.fail() self.post_process_message(message) def post_process_message(self, message): """Called by worker threads whenever they're done processing individual messages, signaling that each message is ready to be acked or rejected. """ if message.failed: self.logger.debug("Rejecting message %r.", message.message_id) self.broker.emit_before("nack", message) self.consumer.nack(message) self.broker.emit_after("nack", message) else: self.logger.debug("Acknowledging message %r.", message.message_id) self.broker.emit_before("ack", message) self.consumer.ack(message) self.broker.emit_after("ack", message) def requeue_messages(self, messages): """Called on worker shutdown and whenever there is a connection error to move unacked messages back to their respective queues asap. """ self.consumer.requeue(messages) def pause(self): """Pause this consumer. """ self.paused = True self.paused_event.clear() def resume(self): """Resume this consumer. """ self.paused = False self.paused_event.clear() def stop(self): """Initiate the ConsumerThread shutdown sequence. Code calling this method should then join on the thread and wait for it to finish shutting down. """ self.logger.debug("Stopping consumer thread...") self.running = False def close(self): """Close this consumer thread and its underlying connection. """ try: if self.consumer: self.requeue_messages(m for _, m in iter_queue(self.delay_queue)) self.consumer.close() except ConnectionError: pass
class JobQueue(object): """This class allows you to periodically perform tasks with the bot. Attributes: _queue (:obj:`PriorityQueue`): The queue that holds the Jobs. bot (:class:`telegram.Bot`): The bot instance that should be passed to the jobs. DEPRECATED: Use :attr:`set_dispatcher` instead. """ def __init__(self, bot=None): self._queue = PriorityQueue() if bot: warnings.warn( "Passing bot to jobqueue is deprecated. Please use set_dispatcher " "instead!", TelegramDeprecationWarning, stacklevel=2) class MockDispatcher(object): def __init__(self): self.bot = bot self.use_context = False self._dispatcher = MockDispatcher() else: self._dispatcher = None self.logger = logging.getLogger(self.__class__.__name__) self.__start_lock = Lock() self.__next_peek_lock = Lock( ) # to protect self._next_peek & self.__tick self.__tick = Event() self.__thread = None self._next_peek = None self._running = False def set_dispatcher(self, dispatcher): """Set the dispatcher to be used by this JobQueue. Use this instead of passing a :class:`telegram.Bot` to the JobQueue, which is deprecated. Args: dispatcher (:class:`telegram.ext.Dispatcher`): The dispatcher. """ self._dispatcher = dispatcher def _put(self, job, time_spec=None, previous_t=None): """ Enqueues the job, scheduling its next run at the correct time. Args: job (telegram.ext.Job): job to enqueue time_spec (optional): Specification of the time for which the job should be scheduled. The precise semantics of this parameter depend on its type (see :func:`telegram.ext.JobQueue.run_repeating` for details). Defaults to now + ``job.interval``. previous_t (optional): Time at which the job last ran (``None`` if it hasn't run yet). """ # get time at which to run: if time_spec is None: time_spec = job.interval if time_spec is None: raise ValueError( "no time specification given for scheduling non-repeating job") next_t = to_float_timestamp(time_spec, reference_timestamp=previous_t) # enqueue: self.logger.debug('Putting job %s with t=%s', job.name, time_spec) self._queue.put((next_t, job)) # Wake up the loop if this job should be executed next self._set_next_peek(next_t) def run_once(self, callback, when, context=None, name=None): """Creates a new ``Job`` that runs once and adds it to the queue. Args: callback (:obj:`callable`): The callback function that should be executed by the new job. Callback signature for context based API: ``def callback(CallbackContext)`` ``context.job`` is the :class:`telegram.ext.Job` instance. It can be used to access its ``job.context`` or change it to a repeating job. when (:obj:`int` | :obj:`float` | :obj:`datetime.timedelta` | \ :obj:`datetime.datetime` | :obj:`datetime.time`): Time in or at which the job should run. This parameter will be interpreted depending on its type. * :obj:`int` or :obj:`float` will be interpreted as "seconds from now" in which the job should run. * :obj:`datetime.timedelta` will be interpreted as "time from now" in which the job should run. * :obj:`datetime.datetime` will be interpreted as a specific date and time at which the job should run. If the timezone (``datetime.tzinfo``) is ``None``, UTC will be assumed. * :obj:`datetime.time` will be interpreted as a specific time of day at which the job should run. This could be either today or, if the time has already passed, tomorrow. If the timezone (``time.tzinfo``) is ``None``, UTC will be assumed. context (:obj:`object`, optional): Additional data needed for the callback function. Can be accessed through ``job.context`` in the callback. Defaults to ``None``. name (:obj:`str`, optional): The name of the new job. Defaults to ``callback.__name__``. Returns: :class:`telegram.ext.Job`: The new ``Job`` instance that has been added to the job queue. """ job = Job(callback, repeat=False, context=context, name=name, job_queue=self) self._put(job, time_spec=when) return job def run_repeating(self, callback, interval, first=None, context=None, name=None): """Creates a new ``Job`` that runs at specified intervals and adds it to the queue. Args: callback (:obj:`callable`): The callback function that should be executed by the new job. Callback signature for context based API: ``def callback(CallbackContext)`` ``context.job`` is the :class:`telegram.ext.Job` instance. It can be used to access its ``job.context`` or change it to a repeating job. interval (:obj:`int` | :obj:`float` | :obj:`datetime.timedelta`): The interval in which the job will run. If it is an :obj:`int` or a :obj:`float`, it will be interpreted as seconds. first (:obj:`int` | :obj:`float` | :obj:`datetime.timedelta` | \ :obj:`datetime.datetime` | :obj:`datetime.time`, optional): Time in or at which the job should run. This parameter will be interpreted depending on its type. * :obj:`int` or :obj:`float` will be interpreted as "seconds from now" in which the job should run. * :obj:`datetime.timedelta` will be interpreted as "time from now" in which the job should run. * :obj:`datetime.datetime` will be interpreted as a specific date and time at which the job should run. If the timezone (``datetime.tzinfo``) is ``None``, UTC will be assumed. * :obj:`datetime.time` will be interpreted as a specific time of day at which the job should run. This could be either today or, if the time has already passed, tomorrow. If the timezone (``time.tzinfo``) is ``None``, UTC will be assumed. Defaults to ``interval`` context (:obj:`object`, optional): Additional data needed for the callback function. Can be accessed through ``job.context`` in the callback. Defaults to ``None``. name (:obj:`str`, optional): The name of the new job. Defaults to ``callback.__name__``. Returns: :class:`telegram.ext.Job`: The new ``Job`` instance that has been added to the job queue. Notes: `interval` is always respected "as-is". That means that if DST changes during that interval, the job might not run at the time one would expect. It is always recommended to pin servers to UTC time, then time related behaviour can always be expected. """ job = Job(callback, interval=interval, repeat=True, context=context, name=name, job_queue=self) self._put(job, time_spec=first) return job def run_daily(self, callback, time, days=Days.EVERY_DAY, context=None, name=None): """Creates a new ``Job`` that runs on a daily basis and adds it to the queue. Args: callback (:obj:`callable`): The callback function that should be executed by the new job. Callback signature for context based API: ``def callback(CallbackContext)`` ``context.job`` is the :class:`telegram.ext.Job` instance. It can be used to access its ``job.context`` or change it to a repeating job. time (:obj:`datetime.time`): Time of day at which the job should run. If the timezone (``time.tzinfo``) is ``None``, UTC will be assumed. days (Tuple[:obj:`int`], optional): Defines on which days of the week the job should run. Defaults to ``EVERY_DAY`` context (:obj:`object`, optional): Additional data needed for the callback function. Can be accessed through ``job.context`` in the callback. Defaults to ``None``. name (:obj:`str`, optional): The name of the new job. Defaults to ``callback.__name__``. Returns: :class:`telegram.ext.Job`: The new ``Job`` instance that has been added to the job queue. Notes: Daily is just an alias for "24 Hours". That means that if DST changes during that interval, the job might not run at the time one would expect. It is always recommended to pin servers to UTC time, then time related behaviour can always be expected. """ job = Job(callback, interval=datetime.timedelta(days=1), repeat=True, days=days, tzinfo=time.tzinfo, context=context, name=name, job_queue=self) self._put(job, time_spec=time) return job def _set_next_peek(self, t): # """ # Set next peek if not defined or `t` is before next peek. # In case the next peek was set, also trigger the `self.__tick` event. # """ with self.__next_peek_lock: if not self._next_peek or self._next_peek > t: self._next_peek = t self.__tick.set() def tick(self): """Run all jobs that are due and re-enqueue them with their interval.""" now = time.time() self.logger.debug('Ticking jobs with t=%f', now) while True: try: t, job = self._queue.get(False) except Empty: break self.logger.debug('Peeked at %s with t=%f', job.name, t) if t > now: # We can get here in two conditions: # 1. At the second or later pass of the while loop, after we've already # processed the job(s) we were supposed to at this time. # 2. At the first iteration of the loop only if `self.put()` had triggered # `self.__tick` because `self._next_peek` wasn't set self.logger.debug("Next task isn't due yet. Finished!") self._queue.put((t, job)) self._set_next_peek(t) break if job.removed: self.logger.debug('Removing job %s', job.name) continue if job.enabled: try: current_week_day = datetime.datetime.now( job.tzinfo).date().weekday() if current_week_day in job.days: self.logger.debug('Running job %s', job.name) job.run(self._dispatcher) except Exception: self.logger.exception( 'An uncaught error was raised while executing job %s', job.name) else: self.logger.debug('Skipping disabled job %s', job.name) if job.repeat and not job.removed: self._put(job, previous_t=t) else: self.logger.debug('Dropping non-repeating or removed job %s', job.name) def start(self): """Starts the job_queue thread.""" self.__start_lock.acquire() if not self._running: self._running = True self.__start_lock.release() self.__thread = Thread(target=self._main_loop, name="Bot:{}:job_queue".format( self._dispatcher.bot.id)) self.__thread.start() self.logger.debug('%s thread started', self.__class__.__name__) else: self.__start_lock.release() def _main_loop(self): """ Thread target of thread ``job_queue``. Runs in background and performs ticks on the job queue. """ while self._running: # self._next_peek may be (re)scheduled during self.tick() or self.put() with self.__next_peek_lock: tmout = self._next_peek - time.time( ) if self._next_peek else None self._next_peek = None self.__tick.clear() self.__tick.wait(tmout) # If we were woken up by self.stop(), just bail out if not self._running: break self.tick() self.logger.debug('%s thread stopped', self.__class__.__name__) def stop(self): """Stops the thread.""" with self.__start_lock: self._running = False self.__tick.set() if self.__thread is not None: self.__thread.join() def jobs(self): """Returns a tuple of all jobs that are currently in the ``JobQueue``.""" with self._queue.mutex: return tuple(job[1] for job in self._queue.queue if job) def get_jobs_by_name(self, name): """Returns a tuple of jobs with the given name that are currently in the ``JobQueue``""" with self._queue.mutex: return tuple(job[1] for job in self._queue.queue if job and job[1].name == name)
class TimeoutExecutor(CanCustomizeBind, Executor): """An executor which delegates to another executor while applying a timeout to each returned future. For any futures returned by this executor, if the future hasn't completed approximately within `timeout` seconds of its creation, an attempt will be made to cancel the future. Note that only a single attempt is made to cancel any future, and there is no guarantee that this will succeed. .. versionadded:: 1.7.0 """ def __init__(self, delegate, timeout, logger=None): """ Parameters: delegate (~concurrent.futures.Executor): an executor to which callables are submitted timeout (float): timeout (in seconds) after which :meth:`concurrent.futures.Future.cancel()` will be invoked on any generated future which has not completed logger (~logging.Logger): a logger used for messages from this executor """ self._log = logger if logger else LOG self._delegate = delegate self._timeout = timeout self._shutdown = False self._jobs = [] self._jobs_lock = Lock() self._jobs_write = Event() event = self._jobs_write self_ref = weakref.ref(self, lambda _: event.set()) self._job_thread = Thread(name="TimeoutExecutor", target=self._job_loop, args=(self_ref, )) self._job_thread.daemon = True self._job_thread.start() def submit(self, *args, **kwargs): # pylint: disable=arguments-differ return self.submit_timeout(self._timeout, *args, **kwargs) def submit_timeout(self, timeout, fn, *args, **kwargs): """Like :code:`submit(fn, *args, **kwargs)`, but uses the specified timeout rather than this executor's default. .. versionadded:: 1.19.0 """ delegate_future = self._delegate.submit(fn, *args, **kwargs) future = MapFuture(delegate_future) future.add_done_callback(self._on_future_done) job = Job(future, delegate_future, monotonic() + timeout) with self._jobs_lock: self._jobs.append(job) self._jobs_write.set() return future def shutdown(self, wait=True): self._log.debug("shutdown") self._shutdown = True self._jobs_write.set() self._delegate.shutdown(wait) if wait: self._job_thread.join(MAX_TIMEOUT) def _partition_jobs(self): pending = [] overdue = [] now = monotonic() for job in self._jobs: if job.future.done(): self._log.debug("Discarding job for completed future: %s", job) elif job.deadline < now: overdue.append(job) else: pending.append(job) return (pending, overdue) def _on_future_done(self, future): self._log.debug("Waking thread for %s", future) self._jobs_write.set() def _do_cancel(self, job): self._log.debug("Attempting cancel: %s", job) cancel_result = job.future.cancel() self._log.debug("Cancel of %s resulted in %s", job, cancel_result) @classmethod @executor_loop def _job_loop(cls, executor_ref): while True: (event, wait_time) = cls._job_loop_iter(executor_ref()) if not event: break event.wait(wait_time) event.clear() @classmethod def _job_loop_iter(cls, executor): if not executor: LOG.debug("Executor was collected") return (None, None) if executor._shutdown: executor._log.debug("Executor was shut down") return (None, None) executor._log.debug("job loop") with executor._jobs_lock: (pending, overdue) = executor._partition_jobs() executor._jobs = pending executor._log.debug("jobs: %s overdue, %s pending", len(overdue), len(pending)) for job in overdue: executor._do_cancel(job) wait_time = None if pending: earliest = min([job.deadline for job in pending]) wait_time = max(earliest - monotonic(), 0) executor._log.debug("Wait until %s", wait_time) return (executor._jobs_write, wait_time)
class Job(object): """This class encapsulates a Job. Attributes: callback (:obj:`callable`): The callback function that should be executed by the new job. context (:obj:`object`): Optional. Additional data needed for the callback function. name (:obj:`str`): Optional. The name of the new job. Args: callback (:obj:`callable`): The callback function that should be executed by the new job. Callback signature for context based API: ``def callback(CallbackContext)`` a ``context.job`` is the :class:`telegram.ext.Job` instance. It can be used to access its ``job.context`` or change it to a repeating job. interval (:obj:`int` | :obj:`float` | :obj:`datetime.timedelta`, optional): The time interval between executions of the job. If it is an :obj:`int` or a :obj:`float`, it will be interpreted as seconds. If you don't set this value, you must set :attr:`repeat` to ``False`` and specify :attr:`time_spec` when you put the job into the job queue. repeat (:obj:`bool`, optional): If this job should be periodically execute its callback function (``True``) or only once (``False``). Defaults to ``True``. context (:obj:`object`, optional): Additional data needed for the callback function. Can be accessed through ``job.context`` in the callback. Defaults to ``None``. name (:obj:`str`, optional): The name of the new job. Defaults to ``callback.__name__``. days (Tuple[:obj:`int`], optional): Defines on which days of the week the job should run. Defaults to ``Days.EVERY_DAY`` job_queue (:class:`telegram.ext.JobQueue`, optional): The ``JobQueue`` this job belongs to. Only optional for backward compatibility with ``JobQueue.put()``. tzinfo (:obj:`datetime.tzinfo`, optional): timezone associated to this job. Used when checking the day of the week to determine whether a job should run (only relevant when ``days is not Days.EVERY_DAY``). Defaults to UTC. """ def __init__(self, callback, interval=None, repeat=True, context=None, days=Days.EVERY_DAY, name=None, job_queue=None, tzinfo=None): self.callback = callback self.context = context self.name = name or callback.__name__ self._repeat = None self._interval = None self.interval = interval self.repeat = repeat self._days = None self.days = days self.tzinfo = tzinfo or _UTC self._job_queue = weakref.proxy( job_queue) if job_queue is not None else None self._remove = Event() self._enabled = Event() self._enabled.set() def run(self, dispatcher): """Executes the callback function.""" if dispatcher.use_context: self.callback(CallbackContext.from_job(self, dispatcher)) else: self.callback(dispatcher.bot, self) def schedule_removal(self): """ Schedules this job for removal from the ``JobQueue``. It will be removed without executing its callback function again. """ self._remove.set() @property def removed(self): """:obj:`bool`: Whether this job is due to be removed.""" return self._remove.is_set() @property def enabled(self): """:obj:`bool`: Whether this job is enabled.""" return self._enabled.is_set() @enabled.setter def enabled(self, status): if status: self._enabled.set() else: self._enabled.clear() @property def interval(self): """ :obj:`int` | :obj:`float` | :obj:`datetime.timedelta`: Optional. The interval in which the job will run. """ return self._interval @interval.setter def interval(self, interval): if interval is None and self.repeat: raise ValueError( "The 'interval' can not be 'None' when 'repeat' is set to 'True'" ) if not (interval is None or isinstance(interval, (Number, datetime.timedelta))): raise ValueError( "The 'interval' must be of type 'datetime.timedelta'," " 'int' or 'float'") self._interval = interval @property def interval_seconds(self): """:obj:`int`: The interval for this job in seconds.""" interval = self.interval if isinstance(interval, datetime.timedelta): return interval.total_seconds() else: return interval @property def repeat(self): """:obj:`bool`: Optional. If this job should periodically execute its callback function.""" return self._repeat @repeat.setter def repeat(self, repeat): if self.interval is None and repeat: raise ValueError( "'repeat' can not be set to 'True' when no 'interval' is set") self._repeat = repeat @property def days(self): """Tuple[:obj:`int`]: Optional. Defines on which days of the week the job should run.""" return self._days @days.setter def days(self, days): if not isinstance(days, tuple): raise ValueError("The 'days' argument should be of type 'tuple'") if not all(isinstance(day, int) for day in days): raise ValueError( "The elements of the 'days' argument should be of type 'int'") if not all(0 <= day <= 6 for day in days): raise ValueError( "The elements of the 'days' argument should be from 0 up to and " "including 6") self._days = days @property def job_queue(self): """:class:`telegram.ext.JobQueue`: Optional. The ``JobQueue`` this job belongs to.""" return self._job_queue @job_queue.setter def job_queue(self, job_queue): # Property setter for backward compatibility with JobQueue.put() if not self._job_queue: self._job_queue = weakref.proxy(job_queue) else: raise RuntimeError( "The 'job_queue' attribute can only be set once.") def __lt__(self, other): return False
class BookingApi: ENDPOINTS = {"countries": "list", "hotels": "list"} def __init__(self, login, password, version): major_minor = version.split(".") assert len(major_minor) == 2 assert int(major_minor[0]) >= 2 assert 0 <= int(major_minor[1]) <= 4 self._event = Event() self._event.set() self._timeout = 5 * 60 # in seconds self._login = login self._password = password self._base_url = f"https://distribution-xml.booking.com/{version}/json" self._set_endpoints() @sleep_and_retry @limits(calls=LIMIT_REQUESTS_PER_MINUTE, period=60) def call_endpoint(self, endpoint, **params): self._event.wait() try: attempts = ATTEMPTS_COUNT while attempts: attempts -= 1 response = None try: response = requests.get(f"{self._base_url}/{endpoint}", auth=(self._login, self._password), params=params, timeout=self._timeout) except requests.exceptions.ReadTimeout: logging.exception("Timeout error.") continue if response.status_code == 200: data = response.json() return data["result"] else: self._handle_errors(response) raise AttemptsSpentError(f"{ATTEMPTS_COUNT} attempts were spent.") except Exception as e: if not self._event.is_set(): self._event.set() raise e def _handle_errors(self, response): error_message = "" data = response.json() try: error_message = ",".join(x["message"] for x in data["errors"]) except KeyError: error_message = data if response.status_code == 429: self._event.clear() wait_seconds = randint(*MINMAX_LIMIT_WAIT_AFTER_429_ERROR_SECONDS) logging.warning( f"Http error {response.status_code}: {error_message}. " f"It waits {wait_seconds} seconds and tries again.") sleep(wait_seconds) self._event.set() else: raise HTTPError( f"Http error with code {response.status_code}: {error_message}." ) def _set_endpoints(self): for endpoint in BookingApi.ENDPOINTS: setattr(self, endpoint, partial(self.call_endpoint, endpoint))