def test_exception_and_forwarded_and_processed_queues_behave_correctly_when_dummy_handler_and_bad_event(self): events = Queue() exceptions_q = Queue() forward_q = Queue() processed_event_q = Queue() looper = EventLoop(events, DummyEventHandler(), exceptions_q=exceptions_q, forward_q=forward_q, processed_event_q=processed_event_q) price_thread = Thread(target=looper.start, args=[]) price_thread.start() ev = None events.put(ev) sleep(2*looper.heartbeat) looper.stop() price_thread.join(timeout=2*looper.heartbeat) out_event = exceptions_q.get_nowait() self.assertIsNone(out_event.orig_event) self.assertIsNone(ev, forward_q.get_nowait()) try: processed_event_q.get_nowait() self.fail('got processed event when exception happened') except Empty: pass
def test_max_connections_blocks(self): """Getting a connection should block for until available.""" import time from copy import deepcopy from threading import Thread # We use a queue for cross thread communication within the unit test. try: # Python 3 from queue import Queue except ImportError: from Queue import Queue q = Queue() q.put_nowait('Not yet got') pool = self.get_pool(max_connections=2, timeout=5) c1 = pool.get_connection('_') c2 = pool.get_connection('_') target = lambda: q.put_nowait(pool.get_connection('_')) Thread(target=target).start() # Blocks while non available. time.sleep(0.05) c3 = q.get_nowait() self.assertEquals(c3, 'Not yet got') # Then got when available. pool.release(c1) time.sleep(0.05) c3 = q.get_nowait() self.assertEquals(c1, c3)
class ThreadedClient(threading.Thread): def __init__(self, host, port=None, start=False, timeout=3.0): super(ThreadedClient, self).__init__() self.host = host self.port = port self.timeout = timeout self._q = Queue() if start: self.start() def run(self, *args, **kw): self.client = Client((self.host, self.port)) while True: msg = self._q.get() if msg is None: break addr, msg = msg log.debug("Sending OSC msg %s, %r", addr, msg) self.client.send(addr, *msg) self.client.close() def send(self, addr, *args, **kw): self._q.put((addr, args), timeout=kw.get('timeout', self.timeout)) def close(self, **kw): timeout = kw.get('timeout', self.timeout) log.debug("Emptying send queue...") while True: try: self._q.get_nowait() except QueueEmpty: break if self.is_alive(): log.debug("Signalling OSC client thread to exit...") self._q.put(None, timeout=timeout) log.debug("Joining OSC client thread...") self.join(timeout) if self.is_alive(): log.warning("OSC client thread still alive after join().") def __enter__(self): return self def __exit__(self, *args): self.close()
def downloads(urls, outputs=[], concurrency=cpu_count()): # 用于线程同步的队列 exit_queue = Queue(1) job_queue = Queue() result_queue = Queue() # 创建下载任务,并加入到任务队列 outputs = [None for _ in urls] if not outputs else outputs for url, output in zip(urls, outputs): job_queue.put(Param(url, output)) job_size = job_queue.qsize() works = [] # 创建工作线程并启动 concurrency = job_size if concurrency > job_size else concurrency for _ in range(concurrency): t = Worker(job_queue, result_queue, exit_queue) works.append(t) t.start() # 检测任务是否完成,主要有两种情况 # 1.所有任务都执行了 # 2.用户主动按ctrl+c结束任务,这里会等待已经运行的任务继续运行 alive = True try: while alive: for work in works: if work.isAlive(): alive = True break else: alive = False if result_queue.qsize() == job_size and exit_queue.qsize() == 0: exit_queue.put(1) except KeyboardInterrupt: logger.warning("ctrl + c is precessed!wait running task to complate..") exit_queue.put(1) for work in works: if work.isAlive(): work.join() # 结果收集并返回 results = [] while job_queue.qsize() > 0: param = job_queue.get_nowait() results.append(Result(False, "task not excute", param.url)) while result_queue.qsize() > 0: result = result_queue.get_nowait() results.append(result) return results
def test_forwarded_and_processed_event_should_be_same_as_input_event_when_dummy_handler(self): events = Queue() forward_q = Queue() processed_event_q = Queue() looper = EventLoop(events, DummyEventHandler(), forward_q=forward_q, processed_event_q=processed_event_q) price_thread = Thread(target=looper.start, args=[]) price_thread.start() ev = 'dummy event' events.put(ev) sleep(2*looper.heartbeat) looper.stop() price_thread.join(timeout=2*looper.heartbeat) self.assertEqual(ev, forward_q.get_nowait()) self.assertEqual(ev, processed_event_q.get_nowait())
class PeekableIterator(object): def __init__(self, stream): self._stream = iter(stream) self._peeked = Queue() def __iter__(self): return self def __next__(self): try: return self._peeked.get_nowait() except Empty: return next(self._stream) def peek(self, n=1): peeked = tuple(islice(self, None, n)) put = self._peeked.put_nowait for item in peeked: put(item) return peeked def lookahead_iter(self): while True: yield from self.peek(1) try: next(self) except StopIteration: break
def run(self): ON_POSIX = 'posix' in sys.builtin_module_names def enqueue_output(out, queue): for line in iter(out.readline, b''): queue.put(line) out.close() commands = self.make_gatt_commands() proc = subprocess.Popen([commands],stdout=subprocess.PIPE, stdin=subprocess.PIPE, close_fds=ON_POSIX, shell=True) q = Queue() t = Thread(target=enqueue_output, args=(proc.stdout, q)) t.daemon = True # thread dies with the program t.start() while True: try: line = q.get_nowait().decode() # print(str(datetime.datetime.now()), line.decode()) self._parse_notification(line) except Empty: # print('no output yet') pass time.sleep(1)
class ConnectionPool(object): def __init__(self, max_connections=5): self.q = Queue() self.max_connections = max_connections self._created_connections = Counter() self.connection_class = Connection self.connection_kwargs = {} def configure(self, max_connections=5, **connection_kwargs): self.max_connections = max_connections self.connection_kwargs = connection_kwargs def get(self): try: return self.q.get_nowait() except Empty: if self._created_connections.current() < self.max_connections: conn = self.connection_class(**self.connection_kwargs).conn self._created_connections.incr() return conn raise def put(self, connection): self.q.put(connection) self._created_connections.decr() def created(self): return self._created_connections.current()
def download_cover(self, log, result_queue, abort, # {{{ title=None, authors=None, identifiers={}, timeout=30, get_best_cover=False): cached_url = self.get_cached_cover_url(identifiers) if cached_url is None: log.info('No cached cover found, running identify') rq = Queue() self.identify(log, rq, abort, title=title, authors=authors, identifiers=identifiers) if abort.is_set(): return results = [] while True: try: results.append(rq.get_nowait()) except Empty: break results.sort(key=self.identify_results_keygen( title=title, authors=authors, identifiers=identifiers)) for mi in results: cached_url = self.get_cached_cover_url(mi.identifiers) if cached_url is not None: break if cached_url is None: log.info('No cover found') return if abort.is_set(): return br = self.browser log('Downloading cover from:', cached_url) try: cdata = br.open_novisit(cached_url, timeout=timeout).read() result_queue.put((self, cdata)) except: log.exception('Failed to download cover from:', cached_url)
def findFirstTri(vert, lastMFace, mirrorMesh, searchData) : # # Itterates through all faces in the mirror mesh and tests for intersection, first intersecting adjacent face connected to the initial search face is returned # faceQueue = Queue() #Tag keep track on what face we have tested / in queue. False for not tested taggedFaces = [] #Start testing from the initial face! lastMFace.tag = True faceQueue.put_nowait(lastMFace) taggedFaces.append(lastMFace) while not faceQueue.empty() : face = faceQueue.get_nowait() mDat = MirrorMesh.triIntersection(vert.co, face) if mDat is not None and mDat._intersected : searchData[vert.index].setMirror(mDat) break #we found an intersecting tri #Queue connected faces MirrorMesh.queueConnectedFaces(face, mirrorMesh, faceQueue, taggedFaces) for f in taggedFaces : f.tag = False
class Kernel(Thread): def __init__(self): Thread.__init__(self) self.programsQueue = Queue() self.isFirstLoad = True self.shouldShutDown= False def initializeKernel(self, clock, programloader, scheduler): self.programLoader = programloader self.scheduler = scheduler self.clock = clock def load(self, program): # Sets a program that the program loader will load to the memory self.programsQueue.put_nowait(program) def run(self): Thread.run(self) while not self.shouldShutDown: if not self.programsQueue.qsize() == 0: program = self.programsQueue.get_nowait() self.isFirstLoad = len(self.programLoader.pcbTable.pcbs) == 0 self.programLoader.load(program) if self.isFirstLoad: self.scheduler.setNextPcbToCpu()
class CarCommandHandler(WebSocketHandler): need_background = True def connect_handler(self, request): self.car_id = int(request.match_info['car_id']) self.car = core.models.Car.objects.get(id=self.car_id) print('websocket connection started for car ID', self.car_id) self.command_queue = Queue() async def process_msg(self, msg_text: str): # print('Got', msg_text, now()) self.command_queue.put(msg_text) async def background(self, ws: web.WebSocketResponse): commander = car_connector.Commander(self.car) while not ws.closed: try: command = self.command_queue.get_nowait() commander.send_command(command) ws.send_str(command) # print('Response:', response, now()) self.command_queue.task_done() except Empty: await asyncio.sleep(0.01) print('Close background for car ID', self.car_id)
class TestStreamTradingRandom(unittest.TestCase): def setUp(self): self.events = Queue() self.execution_q = Queue() self.strategy = StrategyOrderManager(BuyOrSellAt5thTickStrategy(), 100) self.executor = DummyExecutor() self.algo_trader = AlgoTrader(None, self.strategy, self.executor) self.trader = EventLoop(self.events, self.algo_trader) self.trader.started = True def test_order_generation_in_separate_queue(self): self.trader.processed_event_q = self.execution_q for i in range(1, 7): tick = TickEvent("EUR_GBP", get_time(), round(0.87 + (i / 100), 2), 0.88 + (i / 100)) self.events.put(tick) self.trader.pull_process() order = self.execution_q.get_nowait() self.assertEquals(tick.instrument, order.instrument) self.assertEquals(EVENT_TYPES_ORDER, order.TYPE) self.assertEquals(self.strategy.units, order.units) def test_order_generation_in_same_queue(self): self.trader.processed_event_q = self.events for i in range(1, 6): tick = TickEvent("EUR_GBP", get_time(), round(0.87 + (i / 100), 2), 0.88 + (i / 100)) self.events.put(tick) self.trader.pull_process() # order should be generated at the 6th tick order = self.executor.get_last_event() self.assertEquals(tick.instrument, order.instrument) self.assertEquals(EVENT_TYPES_ORDER, order.TYPE) self.assertEquals(self.strategy.units, order.units)
class Metric(object): """ This class stores generic time-series data in a queue. Values are stored as (timestamp, value) tuples """ def __init__(self): self.metric = Queue() def push(self, value, timestamp=None): if timestamp is None: timestamp = int(time.time()) elif not isinstance(timestamp, int): raise ValueError( "Timestamp should be an integer, but it is '%s'" % type(timestamp)) self.metric.put((timestamp, value)) def next(self): try: return self.metric.get_nowait() except Empty: raise StopIteration def get(self): # TODO: decide what we should return here return None def __iter__(self): return self
def download_cover( # {{{ self, log, result_queue, abort, title=None, authors=None, identifiers={}, timeout=30, get_best_cover=False ): cached_url = self.get_cached_cover_url(identifiers) if cached_url is None: log.info('No cached cover found, running identify') rq = Queue() self.identify( log, rq, abort, title=title, authors=authors, identifiers=identifiers ) if abort.is_set(): return results = [] while True: try: results.append(rq.get_nowait()) except Empty: break results.sort( key=self.identify_results_keygen( title=title, authors=authors, identifiers=identifiers ) ) for mi in results: cached_url = self.get_cached_cover_url(mi.identifiers) if cached_url is not None: break if cached_url is None: log.info('No cover found') return br = self.browser for candidate in (0, 1): if abort.is_set(): return url = cached_url + '&zoom={}'.format(candidate) log('Downloading cover from:', cached_url) try: cdata = br.open_novisit(url, timeout=timeout).read() if cdata: if hashlib.md5(cdata).hexdigest() in self.DUMMY_IMAGE_MD5: log.warning('Google returned a dummy image, ignoring') else: result_queue.put((self, cdata)) break except Exception: log.exception('Failed to download cover from:', cached_url)
class ElasticDataSink: def __init__(self, name, conn, model_identifier, workers=5, bound=10000): self.name = name self.conn = conn self.model_identifier = model_identifier self.queue = Queue(maxsize=bound) self.pool = ThreadPoolExecutor(max_workers=workers) def start(self): self.model_identifier.model_class.init(using=self.conn) def __sink_item(self): try: item = self.queue.get_nowait() save_status = item.save(using=self.conn) if not save_status: logger.error("Error saving the item to the sink") else: logger.info("item saved to the sink") except Empty as e: logger.warn("sink queue is empty") logger.warn(e) def sink_item(self, item): assert isinstance(item, self.model_identifier.model_class), \ " item must be instance of " + self.model_identifier.model_class try: self.queue.put(item, timeout=10) self.pool.submit(self.__sink_item) except Full as e: logger.error("sink queue is full") logger.error(e)
def breadthFirstExplore(maze,x,y): '''explore a maze in a breadth first manner''' qLen=len(maze[0][:])*len(maze) pointQueue=Queue(qLen) pointQueue.put_nowait((x,y,qLen)) goal=None maze[x][y]=qLen while not pointQueue.empty(): i=pointQueue.get_nowait() x=i[0] y=i[1] k=i[2] val=getMazeValue(maze, x, y) if val==2: goal=(x,y) return goal else: maze[x][y]=k #explore the neighborhood a=getMazeValue(maze, x+1, y) b=getMazeValue(maze, x, y+1) c=getMazeValue(maze, x-1, y) d=getMazeValue(maze, x, y-1) if a==0 or a ==2: pointQueue.put((x+1,y,k-1)) if b==0 or b==2: pointQueue.put((x,y+1,k-1)) if c==0 or c==2: pointQueue.put((x-1,y,k-1)) if d==0 or d==2: pointQueue.put((x,y-1,k-1)) return goal
class TestStreamTrading(unittest.TestCase): def setUp(self): self.events = Queue() self.execution_q = Queue() self.strategy = StrategyOrderManager(DummyBuyStrategy(), 100) self.executor = DummyExecutor() self.algo_trader = AlgoTrader(None, self.strategy, self.executor) self.trader = EventLoop(self.events, self.algo_trader) self.trader.started = True def test_order_generation_in_separate_queue(self): self.trader.processed_event_q = self.execution_q tick = TickEvent('EUR_GBP', get_time(), 0.87, 0.88) self.events.put(tick) self.trader.pull_process() order = self.execution_q.get_nowait() self.assertEquals(tick.instrument, order.instrument) self.assertEquals(EVENT_TYPES_ORDER, order.TYPE) self.assertEquals(self.strategy.units, order.units) def test_order_generation_in_same_queue(self): self.trader.processed_event_q = self.events tick = TickEvent('EUR_GBP', get_time(), 0.87, 0.88) self.events.put(tick) self.trader.pull_process() order = self.executor.get_last_event() self.assertEquals(tick.instrument, order.instrument) self.assertEquals(EVENT_TYPES_ORDER, order.TYPE) self.assertEquals(self.strategy.units, order.units)
class LogReader: """ Tails a log file and allows lines to be retreived in a non-blocking way. If windows: needs cygwin (or just tail.exe) in path needs set CYGWIN=nodosfilewarning environment variable """ def __init__(self, filePath): self.proc = subprocess.Popen( ["tail", "--sleep-interval=0.25", "-F", filePath], stdout=subprocess.PIPE, universal_newlines=True ) self.queue = Queue() self.thread = Thread( target=self.enqueue_output, args=( self.proc.stdout, self.queue ) ) self.thread.daemon = True self.thread.start() def enqueue_output( self, pipe, queue ): for line in iter( pipe.readline, b'' ): queue.put( line ) pipe.close() def get_line( self ): line = None try: line = str( self.queue.get_nowait() ) except Empty: pass return line def kill( self ): self.proc.terminate()
class InMemoryBufferedLogSubscriber(LogSubscriber): @property def enabled(self): return self._enabled @enabled.setter def enabled(self, value): self._enabled = value @property def is_buffer_full(self): return self._log_entry_queue.qsize() >= self._max_buffer_size def __init__(self, max_buffer_size): self._log_entry_queue = Queue() self._max_buffer_size = max_buffer_size self._enabled = True def log(self, log_entry): if not self._enabled or self.is_buffer_full: return self._log_entry_queue.put(log_entry) def flush(self): items = [] for i in range(self._max_buffer_size): try: items.append(self._log_entry_queue.get_nowait()) except Empty: break return items def close(self): pass # Intentionally blank
class ThreadHandler(): """ Class for running calculations in a seperate thread """ def __init__(self): self._resultsQueue = Queue() self._currentThreadID = 0 self._currentCalculationType = None self._currentResult = None def startCalculation(self, calculationType, args): """ Creates a WorkerThread to carry out func(*args). Will cause any previous still running calculations to be cancelled """ if calculationType == Model.EXP: function = exponentialModelAnalysis elif calculationType == Model.POW: function = powerLawModelAnalysis elif calculationType == Model.WEI: function = weibullModelAnalysis self._currentThreadID += 1 self._currentCalculationType = calculationType newThread = WorkerThread(function, args, self._calculationFinished, self._resultsQueue, self._currentThreadID) newThread.setDaemon(True) newThread.start() def _calculationFinished(self): """ Called internally by the WorkerThreads. If the calculation was cancelled the result is ignored, otherwise if it ended in an error the result type is changed to error, and the result of the calculation is stored in _currentResult ready for retrieval """ threadID, results = self._resultsQueue.get_nowait() if threadID == None: self._currentResult = ("Error",results) elif threadID == self._currentThreadID: self._currentResult = (self._currentCalculationType,results) def cancelLastCalculation(self): """ Tells the ThreadHandler to cancel the last calculation At the moment this involves leaving the calculation to run and ignoring the result when it is returned """ self._currentThreadID += 1 def getCurrentCalculationResult(self): """ Returns the result of the last calculation if finished otherwise returns None """ if self._currentResult is not None: result = self._currentResult self._currentResult = None return result else: return None
def manage(worker, n_threads = 10, catalogs = download.catalogs): 'Manage a bunch of worker threads, and generate their results.' # Download in random order args = [] for catalog in catalogs: for dataset in download.datasets(catalog): args.append((catalog, dataset)) random.shuffle(args) read_queue = Queue() for a in args: read_queue.put(a) write_queue = Queue() threads = [] for i in range(n_threads): threads.append(Thread(target = worker, args = (read_queue,write_queue))) threads[-1].start() while not (read_queue.empty() and write_queue.empty() and set(t.is_alive() for t in threads) == {False}): try: x = write_queue.get_nowait() except Empty: pass else: yield x
class FakeBot: def __init__(self, *args, **kwargs): self.next_message_id = next_message_id() self.next_update_id = next_message_id() self.user = telegram.User(1234567890, 'Unittest') self.updates = Queue() def getUpdates(self, last_update_id, *args, **kwargs): updates = [] try: while not self.updates.empty(): updates.append(self.updates.get_nowait()) except Empty: pass return updates def sendMessage(self, chat_id, message, *args, **kwargs): chat = telegram.Chat(chat_id, telegram.Chat.SUPERGROUP) message = telegram.Message(next(self.next_message_id), self.user, datetime.datetime.now(), chat) return message def sendSticker(self, chat_id, *args, **kwargs): pass def sendLocation(self, chat_id, *args, **kwargs): pass def add_update(self, chat_id, text): chat = telegram.Chat(chat_id, telegram.Chat.SUPERGROUP) user = telegram.User(1234, 'test') message = telegram.Message(next(self.next_message_id), user, datetime.datetime.now(), chat, text=text) update = telegram.Update(next(self.next_update_id), message=message) self.updates.put_nowait(update)
def test_should_allow_to_read_string_journals(self): filename = os.path.join(OUTPUT_DIR, 'journal_read_ut.txt') try: os.remove(filename) except OSError: pass print('writing..') journaler = FileJournaler(full_path=filename) journaler.start() event = 'this is a test event #1' journaler.log_event(get_time(), event) sleep(0.2) journaler.close() print('reading..') eq = Queue() reader = FileJournalerReader(eq, full_path=filename) reader.read_events() try: message = eq.get_nowait() self.assertEqual(event, message) except Empty: pass
class TurtleCanvas(tk.Canvas): def __init__(self, master, *args, **kwargs): tk.Canvas.__init__(self, master, *args, **kwargs) self.pil_image = None self._q = Queue() self._cancel_current = Event() self.update_this() def begin_new(self): self._q = Queue() self._q.put((None, None)) def queue_line(self, line, color): self._q.put((line, color)) def update_this(self): try: for i in range(4000): # draw 2000 lines at a cycle line, color = self._q.get_nowait() if line is None: self.delete('all') continue self.create_line(*line, fill=color) except Empty: pass self.update_idletasks() self.after(5, self.update_this)
def test_sigpipe(self): r, w = os.pipe() outstream = os.fdopen(w, 'wb') task = self.create_task(self.context(console_outstream=outstream)) raised = Queue(maxsize=1) def execute(): try: task.execute() except IOError as e: raised.put(e) execution = threading.Thread(target=execute, name='ConsoleTaskTestBase_sigpipe') execution.setDaemon(True) execution.start() try: data = os.read(r, 5) self.assertEqual(b'jake\n', data) os.close(r) finally: task.stop() execution.join() with self.assertRaises(Empty): e = raised.get_nowait() # Instead of taking the generic assertRaises raises message, provide a more detailed failure # message that shows exactly what untrapped error was on the queue. self.fail('task raised {0}'.format(e))
class StreamHandler(object): def __init__(self, stream, name=None): self.stream = stream self.name = name self.alive = True self.out = Queue() if self.name is None: def add(): while self.alive: line = self.stream.readline() if line: self.out.put(line) else: self.out.put('\n') self.alive = False break else: def add(): with open('/tmp/%s.log' % self.name, 'w') as log: while self.alive: line = self.stream.readline() if line: print(line, file=log) self.out.put(line) else: self.out.put('\n') print('Finished %s' % self.name, file=log) self.alive = False break self.thd = Thread(target=add) self.thd.daemon = True self.thd.start() def readline(self, timeout=None): try: if self.alive: return self.out.get(block=True, timeout=timeout) return self.out.get_nowait() except Empty: return None def kill(self): self.alive = False def clear(self): while self.out.qsize() > 0: self.out.get_nowait()
def run_job_in_thread(jobs): ''' Manage paralell launch jobs - dictionary of products to run {prod:[env,mode]} ''' func_name=run_job_in_thread.__name__ try: job_queue = Queue() res_queue = Queue() #stdout and stderr of subprocess stdstr_queue = Queue() start=time.time() #print "DEBUG function %s job_queue=" %(func_name), job_queue,res_queue #Determine Number of threads to use, but max out at 25 if len(jobs) < 25: num_threads = len(jobs) else: num_threads = 25 #Start thread pool for i in range(num_threads): #for cmd in cmds: # for job,params in jobs.iteritems(): #print "DEBUG func_name num_threads is " ,job,params #target is a function to launch #in my case getResponseparams1 status=getResponseparams1 (prod,env,oper) # Creates same workers which will take parameteres to launch from the job_queue worker = Thread(target=getResponseparams, args=(job_queue,res_queue,stdstr_queue)) worker.setDaemon(True) worker.start() logger.info('Main Thread Waiting') for job,params in jobs.items(): logger.info("DEBUG %s job %s,params %s: %s " % (func_name ,job,params[0],params[1])) #job - product,params[0] - environmnet ,params[1] = operation job_queue.put([job,params[0],params[1]]) job_queue.join() #wait for all jobs'll come to finish end=time.time() logger.info( "Dispatch Completed in %s seconds" % (end-start)) items_to_send=[] #get result from queue while res_queue.empty() != True: #items_to_send.append([prod,env,status]) items_to_send.append(res_queue.get_nowait()) logger.info( "DEBUG %s %s" % (func_name,items_to_send) ) return items_to_send,stdstr_queue except Exception as err: logger.error( " %s: %s" % (func_name, err)) finally: pass
class InsnPool: def __init__(self, proc, max_threads = None): self.numThreads = 0 if max_threads is None: max_threads = multiprocessing.cpu_count() * 5 self.max_threads = max_threads self.queue = Queue() self.proc = proc # A lock used to wake up the polling thread if asynchronous self.lock = threading.Semaphore() def query(self, insn): self.queue.put(insn) def signal(self): self.numThreads -= 1 if self.has_cycled(): # Wake up the polling thread if batch is done processing self.lock.release() def has_cycled(self): return self.numThreads == 0 def has_finished(self): return self.has_cycled() and self.queue.empty() def poll_all(self, blocking = True, callback = None): if not blocking and callback is None: raise ValueError("If called in a non blocking way you must provide a callback function.") def poll_all_impl(): while not self.has_finished(): # Wait for all threads to process self.poll() self.lock.acquire() # Pauses the current thread and waits till batch is processed if callback: callback() if blocking: poll_all_impl() else: thread = threading.Thread(daemon = True, target = poll_all_impl) thread.start() def poll(self): # Batch processing, we only start a new batch when the old # one has finished. It is only possible to jump to a location once. if not self.has_cycled(): return locations = set() while self.numThreads < self.max_threads and not self.queue.empty(): insn = self.queue.get_nowait() if insn.pc in locations: continue else: locations.add(insn.pc) insn.start() self.numThreads += 1
class QueueCommandManager(CommandManager): """ A command manager that takes a queue of functions that act on the tracer and apply them one at a time with each call to next_command(). """ def __init__(self): super(QueueCommandManager, self).__init__() self.queue = Queue() self.sent = [] def enqueue(self, fn): """ Enqueues a new message to be consumed. """ self.queue.put(fn) def user_wait(self, duration): """ Simulates waiting on the user to feed us input for duration seconds. """ self.enqueue(lambda t: sleep(duration + int(PY3))) def clear(self): """ Clears the internal list of functions. """ self.queue = Queue() def user_next_command(self, tracer): """ Removes one message from the internal queue and apply it to the debugger. """ try: self.queue.get_nowait()(tracer) except Empty: return def send(self, msg): # Collect the output so that we can make assertions about it. self.sent.append(json.loads(msg)) user_stop = clear def start(self, tracer, auth_msg=''): pass
class VideoStream: default_fps = 30. def __init__(self, stream_source, interval=0.5): ''' Parameters: stream_source: RTSP, camera index, or video file name self.interval: how long to wait before next frame is served (sec) ''' if stream_source == "": raise ValueError("stream cannot be empty") if interval <= 0 or interval >= 24 * 3600: raise ValueError( "pulse interval should be positive, shorter than a day") self.keep_listeing_for_frames = True self.frame_queue = Queue(100) self.cam = stream_source self.interval = interval self.frame_grabber = None self.is_rtsp = self.cam.lower().startswith('rtsp') self.fps = None self.delay_frames = None self.delay_time = None self.video_capture = None def stop(self): self.keep_listeing_for_frames = False if self.frame_grabber is None: return try: if self.frame_grabber.is_alive(): self.frame_grabber.join(1) self.frame_grabber = None logging.info("Stopped grabbing frames") except: logging.critical("Error while stopping thread") def reset(self, stream_source, interval): ''' Any change to stream source or interval will re-set streaming ''' if stream_source == "": raise ValueError("stream cannot be empty") if interval <= 0 or interval >= 24 * 3600: raise ValueError( "pulse interval should be positive, shorter than a day") self.stop() self.cam = stream_source self.interval = interval self.start() def start(self): if self.frame_grabber is not None: self.stop() self.keep_listeing_for_frames = True self.frame_grabber = threading.Thread(target=self.stream_video) self.frame_grabber.daemon = True self.frame_grabber.start() logging.info(f"Started listening for {self.cam}") def get_frame_with_id(self): ''' Retrieves the frame together with its frame id ''' try: frame_and_id = self.frame_queue.get_nowait() except Empty: frame_and_id = (-1, None) return frame_and_id def setup_stream(self): self.video_capture = cv2.VideoCapture(self.cam) # retrieve camera properties. # self.fps may not always be available # TODO: Need to support frame counting for RTSP! self.fps = self.video_capture.get(cv2.CAP_PROP_FPS) if self.fps is not None and self.fps > 0: self.delay_frames = int(round(self.interval * self.fps)) logging.info(f"Retrieved FPS: {self.fps}") else: self.delay_time = self.interval def stream_video(self): repeat = 3 wait = 0.1 frame = None cur_frame = 0 # this is used for frame delays if the video is on an infinite loop continuous_frame = 0 # will create a new video capture and determine streaming speed self.setup_stream() while self.keep_listeing_for_frames: start_time = time.time() for _ in range(repeat): try: res, frame = self.video_capture.read() if not res: self.video_capture = cv2.VideoCapture(self.cam) res, frame = self.video_capture.read() cur_frame = 0 continuous_frame = 0 break except: # try to re-capture the stream logging.info( "Could not capture video. Recapturing and retrying...") time.sleep(wait) if frame is None: logging.info("Failed to capture frame, sending blank image") continue # if we don't know how many frames we should be skipping # we defer to if self.delay_frames is None and not self.is_rtsp: cur_delay = self.delay_time - time.time() + start_time if cur_delay > 0: time.sleep(cur_delay) # we are reading from a file, simulate 30 self.fps streaming # delay appropriately before enqueueing cur_frame += 1 continuous_frame += 1 if self.delay_frames is not None and (continuous_frame - 1) % self.delay_frames != 0: continue self.frame_queue.put((cur_frame, frame)) self.video_capture.release() self.video_capture = None
class Run_and_log_command(QWidget): """ run a command and send a signal when it complete, or it has failed. use a Qt timer to check the process realtime streaming of the terminal output has so proved to be fruitless """ finished_signal = pyqtSignal(bool) display_signal = pyqtSignal(str) def __init__(self): super(Run_and_log_command, self).__init__() self.polling_interval = 10. self.initUI() def initUI(self): """ Just setup a qlabel showing the shell command and another showing the status of the process """ # Make a grid layout #layout = QGridLayout() hbox = QHBoxLayout() # add the layout to the central widget self.setLayout(hbox) # show the command being executed self.command_label0 = QLabel(self) self.command_label0.setText('<b>Command:</b>') self.command_label = QLabel(self) #self.command_label.setMaximumSize(50, 250) # show the status of the command self.status_label0 = QLabel(self) self.status_label0.setText('<b>Status:</b>') self.status_label = QLabel(self) # add to layout hbox.addWidget(self.status_label0) hbox.addWidget(self.status_label) hbox.addWidget(self.command_label0) hbox.addWidget(self.command_label) hbox.addStretch(1) def run_cmd(self, cmd): from subprocess import PIPE, Popen import shlex self.command_label.setText(cmd) self.status_label.setText('running the command') self.p = Popen(shlex.split(cmd), stdout=PIPE, stderr=PIPE) self.q = Queue() t = Thread(target=enqueue_output, args=(self.p.stdout, self.q)) # make sure the thread dies with the program t.daemon = True t.start() # start a Qt timer to update the status QTimer.singleShot(self.polling_interval, self.update_status) def update_status(self): status = self.p.poll() if status is None: self.status_label.setText('Running') # non blocking readline try: line = self.q.get_nowait() sys.stdout.buffer.write(line) sys.stdout.flush() # emmit a signal on 'display:' line = line.decode("utf-8") if 'display:' in line: self.display_signal.emit(line.split('display:')[1].strip()) except Empty: pass # start a Qt timer to update the status QTimer.singleShot(self.polling_interval, self.update_status) elif status is 0: self.status_label.setText('Finished') # non blocking readline try: line = self.q.get_nowait() sys.stdout.buffer.write(line) sys.stdout.flush() # emmit a signal on 'display:' line = line.decode("utf-8") if 'display:' in line: self.display_signal.emit(line.split('display:')[1].strip()) except Empty: pass # emmit a signal when complete self.finished_signal.emit(True) else: self.status_label.setText(str(status)) # get the output and error msg for line in iter(self.p.stderr.readline, b''): sys.stdout.buffer.write(line) sys.stdout.flush() # emmit a signal when complete self.finished_signal.emit(False)
class RobotState: """ Class responsible for inner-thread communication. All jobs coming from the robot are stored in this class to be retrieved by the simulator for updating/rendering of the simulated robot. """ def __init__(self): cfg = get_config().get_data() large_sim_type = get_config().is_large_sim_type() self.address_motor_center = cfg['alloc_settings']['motor']['center'] if large_sim_type else '' self.address_motor_left = cfg['alloc_settings']['motor']['left'] self.address_motor_right = cfg['alloc_settings']['motor']['right'] self.center_motor_queue = Queue() self.left_motor_queue = Queue() self.right_motor_queue = Queue() self.sound_queue = Queue() self.left_brick_left_led_color = 1 self.left_brick_right_led_color = 1 self.right_brick_left_led_color = 1 self.right_brick_right_led_color = 1 self.should_reset = False self.values = {} self.locks = {} self.motor_lock = threading.Lock() def put_center_motor_job(self, job: float): """ Add a new move job to the queue for the center motor. :param job: to add. """ self.center_motor_queue.put_nowait(job) def put_left_motor_job(self, job: float): """ Add a new move job to the queue for the left motor. :param job: to add. """ self.left_motor_queue.put_nowait(job) def put_right_motor_job(self, job: float): """ Add a new move job to the queue for the right motor. :param job: to add. """ self.right_motor_queue.put_nowait(job) def next_motor_jobs(self) -> Tuple[float, float, float]: """ Get the next move jobs for the left and right motor from the queues. :return: a floating point numbers representing the job move distances. """ self.motor_lock.acquire() try: center = self.center_motor_queue.get_nowait() except Empty: center = None try: left = self.left_motor_queue.get_nowait() except Empty: left = None try: right = self.right_motor_queue.get_nowait() except Empty: right = None self.motor_lock.release() return center, left, right def clear_motor_jobs(self, side: str): self.motor_lock.acquire() if side == 'center': self.center_motor_queue = Queue() elif side == 'left': self.left_motor_queue = Queue() else: self.right_motor_queue = Queue() self.motor_lock.release() def put_sound_job(self, job: str): """ Add a new sound job to the queue to be displayed. :param job: to add. """ self.sound_queue.put_nowait(job) def next_sound_job(self) -> str: """ Get the next sound job from the queue. :return: a str representing the sound as text to be displayed. """ try: return self.sound_queue.get_nowait() except Empty: return None def set_led_color(self, brick_name, led_id, color): if brick_name == 'left_brick:': if led_id == 'led0': self.left_brick_left_led_color = color else: self.left_brick_right_led_color = color else: if led_id == 'led0': self.right_brick_left_led_color = color else: self.right_brick_right_led_color = color def reset(self): """ Reset the data of this State :return: """ self.clear_motor_jobs('center') self.clear_motor_jobs('left') self.clear_motor_jobs('right') self.values.clear() self.should_reset = False def get_motor_side(self, address: str) -> str: """ Get the location of the motor on the actual robot based on its address. :param address: of the motor :return 'center', 'left' or 'right' """ if self.address_motor_center == address: return 'center' if self.address_motor_left == address: return 'left' if self.address_motor_right == address: return 'right' def load_sensor(self, sensor): """ Load the given sensor adding its default value to this state. Also create a lock for the given sensor. :param sensor: to load. """ self.values[sensor.address] = sensor.get_default_value() self.locks[sensor.address] = threading.Lock() def release_locks(self): """ Release all the locked sensor locks. This re-allows for reading the sensor values. """ for lock in self.locks.values(): if lock.locked(): lock.release() def get_value(self, address: str) -> Any: """ Get the value of a sensor by its address. Blocks if the lock of the requested sensor is not available. :param address: of the sensor to get the value from. :return: the value of the sensor. """ self.locks[address].acquire() return self.values[address]
def _forest2json_compressed(forest, compressed_forest, columns, name_feature, get_date, milestones=None, dates_are_dates=True, is_mixed=False): e_size_scaling, font_scaling, size_scaling, transform_e_size, transform_size = \ get_size_transformations(compressed_forest) sort_key = lambda n: ( getattr(n, UNRESOLVED, 0), get_column_value_str(n, name_feature, format_list=True) if name_feature else '', *(get_column_value_str( n, column, format_list=True) for column in columns), -getattr( n, NUM_TIPS_INSIDE), -len(getattr(n, ROOTS)), n.name) i = 0 node2id = {} todo = Queue() for compressed_tree in compressed_forest: todo.put_nowait(compressed_tree) node2id[compressed_tree] = i i += 1 while not todo.empty(): n = todo.get_nowait() for c in sorted(n.children, key=sort_key): node2id[c] = i i += 1 todo.put_nowait(c) n2state = {} # Set the cytoscape features for compressed_tree in compressed_forest: for n in compressed_tree.traverse(): state = get_column_value_str( n, name_feature, format_list=False, list_value='') if name_feature else '' n2state[n] = state root_names = [_.name for _ in getattr(n, ROOTS)] root_dates = [ get_formatted_date(_, dates_are_dates) for _ in getattr(n, ROOTS) ] set_cyto_features_compressed(n, size_scaling, e_size_scaling, font_scaling, transform_size, transform_e_size, state, root_names, root_dates, is_mixed=is_mixed) # Calculate node coordinates min_size = 2 * min( min(getattr(_, NODE_SIZE) for _ in compressed_tree.traverse()) for compressed_tree in compressed_forest) n2width = {} for compressed_tree in compressed_forest: for n in compressed_tree.traverse('postorder'): n2width[n] = max( getattr(n, NODE_SIZE), sum(n2width[c] for c in n.children) + min_size * (len(n.children) - 1)) n2x, n2y = {}, { compressed_tree: 0 for compressed_tree in compressed_forest } n2offset = {} tree_offset = 0 for compressed_tree in compressed_forest: n2offset[compressed_tree] = tree_offset tree_offset += n2width[compressed_tree] + 2 * min_size for n in compressed_tree.traverse('preorder'): n2x[n] = n2offset[n] + n2width[n] / 2 offset = n2offset[n] if not n.is_leaf(): for c in sorted(n.children, key=lambda c: node2id[c]): n2offset[c] = offset offset += n2width[c] + min_size n2y[c] = n2y[n] + getattr(n, NODE_SIZE) / 2 + getattr( c, NODE_SIZE) / 2 + min_size for n in compressed_tree.traverse('postorder'): if not n.is_leaf(): n2x[n] = np.mean([n2x[c] for c in n.children]) def filter_by_date(items, date): return [_ for _ in items if get_date(_) <= date] # Set the cytoscape feature for different timeline points for tree, compressed_tree in zip(forest, compressed_forest): if len(milestones) > 1: nodes = list(compressed_tree.traverse()) for i in range(len(milestones) - 1, -1, -1): milestone = milestones[i] nodes_i = [] # remove too recent nodes from the original tree for n in tree.traverse('postorder'): if n.is_root(): continue if get_date(n) > milestone: n.up.remove_child(n) suffix = '_{}'.format(i) for n in nodes: state = n2state[n] tips_inside, tips_below, internal_nodes_inside, roots = getattr(n, TIPS_INSIDE, []), \ getattr(n, TIPS_BELOW, []), \ getattr(n, INTERNAL_NODES_INSIDE, []), \ getattr(n, ROOTS, []) tips_inside_i, tips_below_i, internal_nodes_inside_i, roots_i = [], [], [], [] for ti, tb, ini, root in zip(tips_inside, tips_below, internal_nodes_inside, roots): if get_date(root) <= milestone: roots_i.append(root) ti = filter_by_date(ti, milestone) tb = [ _ for _ in tb if getattr(_, DATE) <= milestone ] ini = filter_by_date(ini, milestone) tips_inside_i.append( ti + [_ for _ in ini if _.is_leaf()]) tips_below_i.append(tb) internal_nodes_inside_i.append( [_ for _ in ini if not _.is_leaf()]) n.add_features( **{ TIPS_INSIDE: tips_inside_i, TIPS_BELOW: tips_below_i, INTERNAL_NODES_INSIDE: internal_nodes_inside_i, ROOTS: roots_i }) if roots_i: n.add_feature(MILESTONE, i) root_names = [ getattr(_, BRANCH_NAME) if getattr(_, DATE) > milestone else _.name for _ in roots_i ] root_dates = [getattr(_, DATE) for _ in roots_i] if dates_are_dates: try: root_dates = [ numeric2datetime(_).strftime("%d %b %Y") for _ in root_dates ] except: pass set_cyto_features_compressed(n, size_scaling, e_size_scaling, font_scaling, transform_size, transform_e_size, state, root_names=root_names, suffix=suffix, root_dates=root_dates, is_mixed=is_mixed) nodes_i.append(n) nodes = nodes_i # Save the structure clazzes = set() nodes, edges = [], [] one_column = columns[0] if len(columns) == 1 else None for n, n_id in node2id.items(): if one_column: values = getattr(n, one_column, set()) clazz = tuple(sorted(values)) else: clazz = tuple('{}_{}'.format( column, get_column_value_str( n, column, format_list=False, list_value='')) for column in columns) if clazz: clazzes.add(clazz) nodes.append( get_node(n, n_id, tooltip=get_tooltip(n, columns), clazz=clazz, x=n2x[n], y=n2y[n])) for child in sorted(n.children, key=lambda _: node2id[_]): edge_attributes = { feature: getattr(child, feature) for feature in child.features if feature.startswith('edge_') or feature == MILESTONE or feature == IS_POLYTOMY } source_name = n_id edges.append( get_edge(source_name, node2id[child], **edge_attributes)) json_dict = {NODES: nodes, EDGES: edges} return json_dict, sorted(clazzes)
def recursive_update(pkgs, verbose): if not pkgs: logger.warn('no package specified') return set() # TODO: get distro from environment distro = get_rosdistro('kinetic') repositories = [ r for r in distro.repositories.values() if r.source_repository and r.source_repository.patched_packages ] pkgs_queue = Queue() pkgs_done = set() for pkg in pkgs: pkgs_queue.put_nowait(pkg) # mapping from manifest to found location pkgs_manifests = {} try: while True: pkg = pkgs_queue.get_nowait() if pkg in pkgs_done: continue repo = [ repo for repo in repositories if pkg in repo.source_repository.patched_packages ] if len(repo) == 0: logger.debug("Package '%s' can't be found in a repository", pkg) continue assert len( repo) == 1, "Package '%s' is in multiple repositories" % pkg repo = repo[0] logger.info('installing: %s', pkg) if not len([m for m in pkgs_manifests if m.name == pkg]): update_folder(target_path, {repo.name: repo}, verbose) # which packages did we download? updated_packages = find_packages_allowing_duplicates( os.path.join(target_path, repo.name)) # first check for expected packages that were not found found_names = set(package.name for package in updated_packages.values()) for package in repo.source_repository.patched_packages: if package not in found_names: logger.warning( "Package '%s' not found in the repo: '%s'", package, repo.name) # then check for found packages that were not in the yaml for subfolder, package in updated_packages.items(): if package.name in repo.source_repository.patched_packages: logger.info("found '%s'" % os.path.join(repo.name, subfolder)) pkgs_manifests[package] = os.path.join( repo.name, subfolder) else: logger.debug( "Found package '%s' in an unexpected repo: '%s'", package.name, repo.name) manifest = [m for m in pkgs_manifests if m.name == pkg] if not len(manifest): logger.error("Required package '%s' not found", pkg) exit(1) assert len( manifest) == 1, "Package '%s' was found multiple times" % pkg manifest = manifest[0] deps = manifest.buildtool_depends + manifest.build_depends + manifest.run_depends + manifest.test_depends for dep in deps: if dep not in pkgs_done: logger.debug("queing: '%s'", dep.name) pkgs_queue.put_nowait(dep.name) pkgs_done.add(pkg) except Empty: pass if not pkgs_done: logger.error('no repository updated, package could not be found') # create the symlinks in the src folder for pkg in pkgs_done: manifest = [m for m in pkgs_manifests if m.name == pkg] assert len(manifest) == 1 manifest = manifest[0] folder = pkgs_manifests[manifest] source = os.path.join(target_path, folder) link_name = os.path.join(link_dir, manifest.name) source = os.path.relpath(source, link_dir) symlink_force(source, link_name) return pkgs_done
class AbstractBaseThread(QThread): started = pyqtSignal() stopped = pyqtSignal() sender_needs_restart = pyqtSignal() def __init__(self, freq, sample_rate, bandwidth, gain, if_gain, baseband_gain, receiving: bool, ip='127.0.0.1', parent=None): super().__init__(parent) self.ip = ip self.gr_port = 1337 self._sample_rate = sample_rate self._freq = freq self._gain = gain self._if_gain = if_gain self._baseband_gain = baseband_gain self._bandwidth = bandwidth self._freq_correction = 1 self._direct_sampling_mode = 0 self._antenna_index = 0 self._channel_index = 0 self._receiving = receiving # False for Sender-Thread self.device = "USRP" self.current_index = 0 self.is_in_spectrum_mode = False self.context = None self.socket = None gnuradio_path_file = os.path.join(tempfile.gettempdir(), "gnuradio_path.txt") if constants.SETTINGS.value("use_gnuradio_install_dir", False, bool): gnuradio_dir = constants.SETTINGS.value("gnuradio_install_dir", "") with open(gnuradio_path_file, "w") as f: f.write(gnuradio_dir) if os.path.isfile(os.path.join(gnuradio_dir, "gr-python27", "pythonw.exe")): self.python2_interpreter = os.path.join(gnuradio_dir, "gr-python27", "pythonw.exe") else: self.python2_interpreter = os.path.join(gnuradio_dir, "gr-python27", "python.exe") else: try: os.remove(gnuradio_path_file) except OSError: pass self.python2_interpreter = constants.SETTINGS.value("python2_exe", "") self.queue = Queue() self.data = None # Placeholder for SenderThread self.current_iteration = 0 # Counts number of Sendings in SenderThread self.tb_process = None @property def sample_rate(self): return self._sample_rate @sample_rate.setter def sample_rate(self, value): self._sample_rate = value if self.tb_process: try: self.tb_process.stdin.write(b'SR:' + bytes(str(value), "utf8") + b'\n') self.tb_process.stdin.flush() except BrokenPipeError: pass @property def freq(self): return self._freq @freq.setter def freq(self, value): self._freq = value if self.tb_process: try: self.tb_process.stdin.write(b'F:' + bytes(str(value), "utf8") + b'\n') self.tb_process.stdin.flush() except BrokenPipeError: pass @property def gain(self): return self._gain @gain.setter def gain(self, value): self._gain = value if self.tb_process: try: self.tb_process.stdin.write(b'G:' + bytes(str(value), "utf8") + b'\n') self.tb_process.stdin.flush() except BrokenPipeError: pass @property def if_gain(self): return self._if_gain @if_gain.setter def if_gain(self, value): self._if_gain = value if self.tb_process: try: self.tb_process.stdin.write(b'IFG:' + bytes(str(value), "utf8") + b'\n') self.tb_process.stdin.flush() except BrokenPipeError: pass @property def baseband_gain(self): return self._baseband_gain @baseband_gain.setter def baseband_gain(self, value): self._baseband_gain = value if self.tb_process: try: self.tb_process.stdin.write(b'BBG:' + bytes(str(value), "utf8") + b'\n') self.tb_process.stdin.flush() except BrokenPipeError: pass @property def bandwidth(self): return self._bandwidth @bandwidth.setter def bandwidth(self, value): self._bandwidth = value if self.tb_process: try: self.tb_process.stdin.write(b'BW:' + bytes(str(value), "utf8") + b'\n') self.tb_process.stdin.flush() except BrokenPipeError: pass @property def freq_correction(self): return self._freq_correction @freq_correction.setter def freq_correction(self, value): self._freq_correction = value if self.tb_process: try: self.tb_process.stdin.write(b'FC:' + bytes(str(value), "utf8") + b'\n') self.tb_process.stdin.flush() except BrokenPipeError: pass @property def channel_index(self): return self._channel_index @channel_index.setter def channel_index(self, value): self._channel_index = value @property def antenna_index(self): return self._antenna_index @antenna_index.setter def antenna_index(self, value): self._antenna_index = value @property def direct_sampling_mode(self): return self._direct_sampling_mode @direct_sampling_mode.setter def direct_sampling_mode(self, value): self._direct_sampling_mode = value if self.tb_process: try: self.tb_process.stdin.write(b'DSM:' + bytes(str(value), "utf8") + b'\n') self.tb_process.stdin.flush() except BrokenPipeError: pass def initialize_process(self): self.started.emit() if not hasattr(sys, 'frozen'): rp = os.path.realpath(os.path.join(os.path.dirname(__file__), "scripts")) else: rp = os.path.realpath(os.path.dirname(sys.executable)) suffix = "_recv.py" if self._receiving else "_send.py" filename = self.device.lower().split(" ")[0] + suffix if not self.python2_interpreter: self.stop("FATAL: Could not find python 2 interpreter. Make sure you have a running gnuradio installation.") return options = [self.python2_interpreter, os.path.join(rp, filename), "--samplerate", str(self.sample_rate), "--freq", str(self.freq), "--gain", str(self.gain), "--bandwidth", str(self.bandwidth), "--port", str(self.gr_port)] if self.device.upper() == "HACKRF": options.extend(["--if-gain", str(self.if_gain), "--baseband-gain", str(self.baseband_gain)]) if self.device.upper() == "RTL-SDR": options.extend(["--freq-correction", str(self.freq_correction), "--direct-sampling", str(self.direct_sampling_mode)]) logger.info("Starting Gnuradio") logger.debug(" ".join(options)) self.tb_process = Popen(options, stdout=PIPE, stderr=PIPE, stdin=PIPE, bufsize=1) logger.info("Started Gnuradio") t = Thread(target=self.enqueue_output, args=(self.tb_process.stderr, self.queue)) t.daemon = True # thread dies with the program t.start() def init_recv_socket(self): logger.info("Initalizing receive socket") self.context = zmq.Context() self.socket = self.context.socket(zmq.PULL) self.socket.setsockopt(zmq.RCVTIMEO, 90) logger.info("Initalized receive socket") while not self.isInterruptionRequested(): try: time.sleep(0.1) logger.info("Trying to get a connection to gnuradio...") self.socket.connect("tcp://{0}:{1}".format(self.ip, self.gr_port)) logger.info("Got connection") break except (ConnectionRefusedError, ConnectionResetError): continue except Exception as e: logger.error("Unexpected error", str(e)) def run(self): pass def read_errors(self, initial_errors=None): result = [] if initial_errors is None else initial_errors while True: try: result.append(self.queue.get_nowait()) except Empty: break result = b"".join(result) try: return result.decode("utf-8") except UnicodeDecodeError: return "Could not decode device message" def enqueue_output(self, out, queue): for line in iter(out.readline, b''): queue.put(line) out.close() def stop(self, msg: str): if msg and not msg.startswith("FIN"): self.requestInterruption() time.sleep(0.1) try: logger.info("Kill grc process") self.tb_process.kill() logger.info("Term grc process") self.tb_process.terminate() self.tb_process = None except AttributeError: pass logger.info(msg) self.stopped.emit()
class printcore(): def __init__(self, port=None, baud=None, dtr=None): """Initializes a printcore instance. Pass the port and baud rate to connect immediately""" self.baud = None self.dtr = None self.port = None self.analyzer = gcoder.GCode() # Serial instance connected to the printer, should be None when # disconnected self.printer = None # clear to send, enabled after responses # FIXME: should probably be changed to a sliding window approach self.clear = 0 # The printer has responded to the initial command and is active self.online = False # is a print currently running, true if printing, false if paused self.printing = False self.mainqueue = None self.priqueue = Queue(0) self.queueindex = 0 self.lineno = 0 self.resendfrom = -1 self.paused = False self.sentlines = {} self.log = deque(maxlen=10000) self.sent = [] self.writefailures = 0 self.tempcb = None # impl (wholeline) self.recvcb = None # impl (wholeline) self.sendcb = None # impl (wholeline) self.preprintsendcb = None # impl (wholeline) self.printsendcb = None # impl (wholeline) self.layerchangecb = None # impl (wholeline) self.errorcb = None # impl (wholeline) self.startcb = None # impl () self.endcb = None # impl () self.onlinecb = None # impl () self.loud = False # emit sent and received lines to terminal self.tcp_streaming_mode = False self.greetings = ['start', 'Grbl '] self.wait = 0 # default wait period for send(), send_now() self.read_thread = None self.stop_read_thread = False self.send_thread = None self.stop_send_thread = False self.print_thread = None self.event_handler = PRINTCORE_HANDLER # Not all platforms need to do this parity workaround, and some drivers # don't support it. Limit it to platforms that actually require it # here to avoid doing redundant work elsewhere and potentially breaking # things. self.needs_parity_workaround = platform.system( ) == "linux" and os.path.exists("/etc/debian") for handler in self.event_handler: try: handler.on_init() except: logging.error(traceback.format_exc()) if port is not None and baud is not None: self.connect(port, baud) self.xy_feedrate = None self.z_feedrate = None def addEventHandler(self, handler): ''' Adds an event handler. @param handler: The handler to be added. ''' self.event_handler.append(handler) def logError(self, error): for handler in self.event_handler: try: handler.on_error(error) except: logging.error(traceback.format_exc()) if self.errorcb: try: self.errorcb(error) except: logging.error(traceback.format_exc()) else: logging.error(error) @locked def disconnect(self): """Disconnects from printer and pauses the print """ if self.printer: if self.read_thread: self.stop_read_thread = True if threading.current_thread() != self.read_thread: self.read_thread.join() self.read_thread = None if self.print_thread: self.printing = False self.print_thread.join() self._stop_sender() try: self.printer.close() except socket.error: pass except OSError: pass for handler in self.event_handler: try: handler.on_disconnect() except: logging.error(traceback.format_exc()) self.printer = None self.online = False self.printing = False @locked def connect(self, port=None, baud=None, dtr=None): """Set port and baudrate if given, then connect to printer """ if self.printer: self.disconnect() if port is not None: self.port = port if baud is not None: self.baud = baud if dtr is not None: self.dtr = dtr if self.port is not None and self.baud is not None: # Connect to socket if "port" is an IP, device if not host_regexp = re.compile( "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$" ) is_serial = True if ":" in self.port: bits = self.port.split(":") if len(bits) == 2: hostname = bits[0] try: port_number = int(bits[1]) if host_regexp.match( hostname) and 1 <= port_number <= 65535: is_serial = False except: pass self.writefailures = 0 if not is_serial: self.printer_tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.printer_tcp.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) self.timeout = 0.25 self.printer_tcp.settimeout(1.0) try: self.printer_tcp.connect((hostname, port_number)) self.printer_tcp.settimeout(self.timeout) self.printer = self.printer_tcp.makefile() except socket.error as e: if (e.strerror is None): e.strerror = "" self.logError( _("Could not connect to %s:%s:") % (hostname, port_number) + "\n" + _("Socket error %s:") % e.errno + "\n" + e.strerror) self.printer = None self.printer_tcp = None return else: disable_hup(self.port) self.printer_tcp = None try: if self.needs_parity_workaround: self.printer = Serial(port=self.port, baudrate=self.baud, timeout=0.25, parity=PARITY_ODD) self.printer.close() self.printer.parity = PARITY_NONE else: self.printer = Serial(baudrate=self.baud, timeout=0.25, parity=PARITY_NONE) self.printer.setPort(self.port) try: #this appears not to work on many platforms, so we're going to call it but not care if it fails self.printer.setDTR(dtr) except: #self.logError(_("Could not set DTR on this platform")) #not sure whether to output an error message pass self.printer.open() except SerialException as e: self.logError( _("Could not connect to %s at baudrate %s:") % (self.port, self.baud) + "\n" + _("Serial error: %s") % e) self.printer = None return except IOError as e: self.logError( _("Could not connect to %s at baudrate %s:") % (self.port, self.baud) + "\n" + _("IO error: %s") % e) self.printer = None return for handler in self.event_handler: try: handler.on_connect() except: logging.error(traceback.format_exc()) self.stop_read_thread = False self.read_thread = threading.Thread(target=self._listen) self.read_thread.start() self._start_sender() def reset(self): """Reset the printer """ if self.printer and not self.printer_tcp: self.printer.setDTR(1) time.sleep(0.2) self.printer.setDTR(0) def _readline(self): try: try: try: line = self.printer.readline().decode('ascii') except UnicodeDecodeError: self.logError( _("Got rubbish reply from %s at baudrate %s:") % (self.port, self.baud) + "\n" + _("Maybe a bad baudrate?")) return None if self.printer_tcp and not line: raise OSError(-1, "Read EOF from socket") except socket.timeout: return "" if len(line) > 1: self.log.append(line) for handler in self.event_handler: try: handler.on_recv(line) except: logging.error(traceback.format_exc()) if self.recvcb: try: self.recvcb(line) except: self.logError(traceback.format_exc()) if self.loud: logging.info("RECV: %s" % line.rstrip()) return line except SelectError as e: if 'Bad file descriptor' in e.args[1]: self.logError( _("Can't read from printer (disconnected?) (SelectError {0}): {1}" ).format(e.errno, decode_utf8(e.strerror))) return None else: self.logError( _("SelectError ({0}): {1}").format(e.errno, decode_utf8( e.strerror))) raise except SerialException as e: self.logError( _("Can't read from printer (disconnected?) (SerialException): {0}" ).format(decode_utf8(str(e)))) return None except socket.error as e: self.logError( _("Can't read from printer (disconnected?) (Socket error {0}): {1}" ).format(e.errno, decode_utf8(e.strerror))) return None except OSError as e: if e.errno == errno.EAGAIN: # Not a real error, no data was available return "" self.logError( _("Can't read from printer (disconnected?) (OS Error {0}): {1}" ).format(e.errno, e.strerror)) return None def _listen_can_continue(self): if self.printer_tcp: return not self.stop_read_thread and self.printer return (not self.stop_read_thread and self.printer and self.printer.isOpen()) def _listen_until_online(self): while not self.online and self._listen_can_continue(): self._send("M105") if self.writefailures >= 4: logging.error( _("Aborting connection attempt after 4 failed writes.")) return empty_lines = 0 while self._listen_can_continue(): line = self._readline() if line is None: break # connection problem # workaround cases where M105 was sent before printer Serial # was online an empty line means read timeout was reached, # meaning no data was received thus we count those empty lines, # and once we have seen 15 in a row, we just break and send a # new M105 # 15 was chosen based on the fact that it gives enough time for # Gen7 bootloader to time out, and that the non received M105 # issues should be quite rare so we can wait for a long time # before resending if not line: empty_lines += 1 if empty_lines == 15: break else: empty_lines = 0 if line.startswith(tuple(self.greetings)) \ or line.startswith('ok') or "T:" in line: self.online = True for handler in self.event_handler: try: handler.on_online() except: logging.error(traceback.format_exc()) if self.onlinecb: try: self.onlinecb() except: self.logError(traceback.format_exc()) return def _listen(self): """This function acts on messages from the firmware """ self.clear = True if not self.printing: self._listen_until_online() while self._listen_can_continue(): line = self._readline() if line is None: break if line.startswith('DEBUG_'): continue if line.startswith(tuple(self.greetings)) or line.startswith('ok'): self.clear = True if line.startswith('ok') and "T:" in line: for handler in self.event_handler: try: handler.on_temp(line) except: logging.error(traceback.format_exc()) if line.startswith('ok') and "T:" in line and self.tempcb: # callback for temp, status, whatever try: self.tempcb(line) except: self.logError(traceback.format_exc()) elif line.startswith('Error'): self.logError(line) # Teststrings for resend parsing # Firmware exp. result # line="rs N2 Expected checksum 67" # Teacup 2 if line.lower().startswith("resend") or line.startswith("rs"): for haystack in ["N:", "N", ":"]: line = line.replace(haystack, " ") linewords = line.split() while len(linewords) != 0: try: toresend = int(linewords.pop(0)) self.resendfrom = toresend break except: pass self.clear = True self.clear = True def _start_sender(self): self.stop_send_thread = False self.send_thread = threading.Thread(target=self._sender) self.send_thread.start() def _stop_sender(self): if self.send_thread: self.stop_send_thread = True self.send_thread.join() self.send_thread = None def _sender(self): while not self.stop_send_thread: try: command = self.priqueue.get(True, 0.1) except QueueEmpty: continue while self.printer and self.printing and not self.clear: time.sleep(0.001) self._send(command) while self.printer and self.printing and not self.clear: time.sleep(0.001) def _checksum(self, command): return reduce(lambda x, y: x ^ y, map(ord, command)) def startprint(self, gcode, startindex=0): """Start a print, gcode is an array of gcode commands. returns True on success, False if already printing. The print queue will be replaced with the contents of the data array, the next line will be set to 0 and the firmware notified. Printing will then start in a parallel thread. """ if self.printing or not self.online or not self.printer: return False self.queueindex = startindex self.mainqueue = gcode self.printing = True self.lineno = 0 self.resendfrom = -1 self._send("M110", -1, True) if not gcode or not gcode.lines: return True self.clear = False resuming = (startindex != 0) self.print_thread = threading.Thread(target=self._print, kwargs={"resuming": resuming}) self.print_thread.start() return True def cancelprint(self): self.pause() self.paused = False self.mainqueue = None self.clear = True # run a simple script if it exists, no multithreading def runSmallScript(self, filename): if filename is None: return f = None try: with open(filename) as f: for i in f: l = i.replace("\n", "") l = l[:l.find(";")] # remove comments self.send_now(l) except: pass def pause(self): """Pauses the print, saving the current position. """ if not self.printing: return False self.paused = True self.printing = False # try joining the print thread: enclose it in try/except because we # might be calling it from the thread itself try: self.print_thread.join() except RuntimeError as e: if e.message == "cannot join current thread": pass else: self.logError(traceback.format_exc()) except: self.logError(traceback.format_exc()) self.print_thread = None # saves the status self.pauseX = self.analyzer.abs_x self.pauseY = self.analyzer.abs_y self.pauseZ = self.analyzer.abs_z self.pauseE = self.analyzer.abs_e self.pauseF = self.analyzer.current_f self.pauseRelative = self.analyzer.relative def resume(self): """Resumes a paused print. """ if not self.paused: return False if self.paused: # restores the status self.send_now("G90") # go to absolute coordinates xyFeedString = "" zFeedString = "" if self.xy_feedrate is not None: xyFeedString = " F" + str(self.xy_feedrate) if self.z_feedrate is not None: zFeedString = " F" + str(self.z_feedrate) self.send_now("G1 X%s Y%s%s" % (self.pauseX, self.pauseY, xyFeedString)) self.send_now("G1 Z" + str(self.pauseZ) + zFeedString) self.send_now("G92 E" + str(self.pauseE)) # go back to relative if needed if self.pauseRelative: self.send_now("G91") # reset old feed rate self.send_now("G1 F" + str(self.pauseF)) self.paused = False self.printing = True self.print_thread = threading.Thread(target=self._print, kwargs={"resuming": True}) self.print_thread.start() def send(self, command, wait=0): """Adds a command to the checksummed main command queue if printing, or sends the command immediately if not printing""" if self.online: if self.printing: self.mainqueue.append(command) else: self.priqueue.put_nowait(command) else: self.logError(_("Not connected to printer.")) def send_now(self, command, wait=0): """Sends a command to the printer ahead of the command queue, without a checksum""" if self.online: self.priqueue.put_nowait(command) else: self.logError(_("Not connected to printer.")) def _print(self, resuming=False): self._stop_sender() try: for handler in self.event_handler: try: handler.on_start(resuming) except: logging.error(traceback.format_exc()) if self.startcb: # callback for printing started try: self.startcb(resuming) except: self.logError( _("Print start callback failed with:") + "\n" + traceback.format_exc()) while self.printing and self.printer and self.online: self._sendnext() self.sentlines = {} self.log.clear() self.sent = [] for handler in self.event_handler: try: handler.on_end() except: logging.error(traceback.format_exc()) if self.endcb: # callback for printing done try: self.endcb() except: self.logError( _("Print end callback failed with:") + "\n" + traceback.format_exc()) except: self.logError( _("Print thread died due to the following error:") + "\n" + traceback.format_exc()) finally: self.print_thread = None self._start_sender() def process_host_command(self, command): """only ;@pause command is implemented as a host command in printcore, but hosts are free to reimplement this method""" command = command.lstrip() if command.startswith(";@pause"): self.pause() def _sendnext(self): if not self.printer: return while self.printer and self.printing and not self.clear: time.sleep(0.001) # Only wait for oks when using serial connections or when not using tcp # in streaming mode if not self.printer_tcp or not self.tcp_streaming_mode: self.clear = False if not (self.printing and self.printer and self.online): self.clear = True return if self.resendfrom < self.lineno and self.resendfrom > -1: self._send(self.sentlines[self.resendfrom], self.resendfrom, False) self.resendfrom += 1 return self.resendfrom = -1 if not self.priqueue.empty(): self._send(self.priqueue.get_nowait()) self.priqueue.task_done() return if self.printing and self.mainqueue.has_index(self.queueindex): (layer, line) = self.mainqueue.idxs(self.queueindex) gline = self.mainqueue.all_layers[layer][line] if self.queueindex > 0: (prev_layer, prev_line) = self.mainqueue.idxs(self.queueindex - 1) if prev_layer != layer: for handler in self.event_handler: try: handler.on_layerchange(layer) except: logging.error(traceback.format_exc()) if self.layerchangecb and self.queueindex > 0: (prev_layer, prev_line) = self.mainqueue.idxs(self.queueindex - 1) if prev_layer != layer: try: self.layerchangecb(layer) except: self.logError(traceback.format_exc()) for handler in self.event_handler: try: handler.on_preprintsend(gline, self.queueindex, self.mainqueue) except: logging.error(traceback.format_exc()) if self.preprintsendcb: if self.mainqueue.has_index(self.queueindex + 1): (next_layer, next_line) = self.mainqueue.idxs(self.queueindex + 1) next_gline = self.mainqueue.all_layers[next_layer][ next_line] else: next_gline = None gline = self.preprintsendcb(gline, next_gline) if gline is None: self.queueindex += 1 self.clear = True return tline = gline.raw if tline.lstrip().startswith(";@"): # check for host command self.process_host_command(tline) self.queueindex += 1 self.clear = True return # Strip comments tline = gcoder.gcode_strip_comment_exp.sub("", tline).strip() if tline: self._send(tline, self.lineno, True) self.lineno += 1 for handler in self.event_handler: try: handler.on_printsend(gline) except: logging.error(traceback.format_exc()) if self.printsendcb: try: self.printsendcb(gline) except: self.logError(traceback.format_exc()) else: self.clear = True self.queueindex += 1 else: self.printing = False self.clear = True if not self.paused: self.queueindex = 0 self.lineno = 0 self._send("M110", -1, True) def _send(self, command, lineno=0, calcchecksum=False): # Only add checksums if over serial (tcp does the flow control itself) if calcchecksum and not self.printer_tcp: prefix = "N" + str(lineno) + " " + command command = prefix + "*" + str(self._checksum(prefix)) if "M110" not in command: self.sentlines[lineno] = command if self.printer: self.sent.append(command) # run the command through the analyzer gline = None try: gline = self.analyzer.append(command, store=False) except: logging.warning( _("Could not analyze command %s:") % command + "\n" + traceback.format_exc()) if self.loud: logging.info("SENT: %s" % command) for handler in self.event_handler: try: handler.on_send(command, gline) except: logging.error(traceback.format_exc()) if self.sendcb: try: self.sendcb(command, gline) except: self.logError(traceback.format_exc()) try: self.printer.write((command + "\n").encode('ascii')) if self.printer_tcp: try: self.printer.flush() except socket.timeout: pass self.writefailures = 0 except socket.error as e: if e.errno is None: self.logError( _("Can't write to printer (disconnected ?):") + "\n" + traceback.format_exc()) else: self.logError( _("Can't write to printer (disconnected?) (Socket error {0}): {1}" ).format(e.errno, decode_utf8(e.strerror))) self.writefailures += 1 except SerialException as e: self.logError( _("Can't write to printer (disconnected?) (SerialException): {0}" ).format(decode_utf8(str(e)))) self.writefailures += 1 except RuntimeError as e: self.logError( _("Socket connection broken, disconnected. ({0}): {1}"). format(e.errno, decode_utf8(e.strerror))) self.writefailures += 1
class RunPanel(mforms.Table): def __init__(self, editor, log_callback): mforms.Table.__init__(self) self.set_managed() self.set_release_on_add() self.set_row_count(2) self.set_column_count(1) self.set_padding(-1) self.label = mforms.newLabel("Running script...") self.add(self.label, 0, 1, 0, 1, mforms.HFillFlag) self.progress = mforms.newProgressBar() self.add(self.progress, 0, 1, 1, 2, mforms.HFillFlag) self.progress.set_size(400, -1) self.log_callback = log_callback self.editor = editor self.importer = MySQLScriptImporter(editor.connection) self.importer.report_progress = self.report_progress self.importer.report_output = self.report_output self._worker_queue = Queue() self._worker = None self._progress_status = None self._progress_value = 0 self._update_timer = None def __del__(self): if self._update_timer: mforms.Utilities.cancel_timeout(self._update_timer) @property def is_busy(self): return self._worker != None def report_progress(self, status, current, total): self._progress_status = status if status: self._worker_queue.put(status) if total > 0: self._progress_value = float(current) / total def report_output(self, message): log_info("%s\n" % message) self._worker_queue.put(message) def update_ui(self): try: while True: data = self._worker_queue.get_nowait() if data is None: self._worker.join() self._worker = None self._update_timer = None self.progress.show(False) self.log_callback(None) return False if isinstance(data, Exception): self.log_callback(str(data) + "\n") if isinstance(data, grt.DBError) and data.args[1] == 1044: mforms.Utilities.show_error( "Run SQL Script", "The current MySQL account does not have enough privileges to execute the script.", "OK", "", "") elif isinstance(data, grt.DBLoginError): username = self.editor.connection.parameterValues[ "userName"] host = self.editor.connection.hostIdentifier mforms.Utilities.forget_password(host, username) mforms.Utilities.show_error( "Run SQL Script", "Error executing SQL script.\n" + str(data), "OK", "", "") else: mforms.Utilities.show_error( "Run SQL Script", "Error executing SQL script.\n" + str(data), "OK", "", "") else: self.log_callback(data + "\n") if self._progress_status is not None: self.label.set_text(self._progress_status) self.progress.set_value(self._progress_value) except Empty: pass if self._progress_status is not None: self.label.set_text(self._progress_status) self.progress.set_value(self._progress_value) return True def work(self, file, schema, charset): try: log_info("Executing %s...\n" % file) self._progress_status = "Executing %s..." % file self._progress_value = 0 self.importer.import_script(file, default_schema=schema, default_charset=charset) log_info("Run script finished\n") except grt.DBLoginError as e: log_error("MySQL login error running script: %s\n" % e) self._worker_queue.put(e) except grt.DBError as e: log_error("MySQL error running script: %s\n" % e) self._worker_queue.put(e) except Exception as e: import traceback log_error("Unexpected exception running script: %s\n%s\n" % (e, traceback.format_exc())) self._worker_queue.put(e) self._worker_queue.put(None) def start(self, what, default_db, default_charset): parameterValues = self.editor.connection.parameterValues username = parameterValues["userName"] host = self.editor.connection.hostIdentifier ok, pwd = mforms.Utilities.find_cached_password(host, username) if not ok: accepted, pwd = mforms.Utilities.find_or_ask_for_password( "Run SQL Script", host, username, False) if not accepted: return self.importer.set_password(pwd) self._worker = Thread(target=self.work, args=(what, default_db, default_charset)) self._worker.start() self._update_timer = mforms.Utilities.add_timeout(0.2, self.update_ui)
def q_learning( env, state_converter, max_steps=200, n_step=1, alpha=0.1, gamma=0.95, episode_count=2000, render=False, figure_path=None, ): gamma_n = gamma**n_step def update_q_value(q_value, prev_state, action, state, return_value): prev_index = state_converter.state_to_index(prev_state) delta = return_value - q_value[prev_index][action] if state is not None: index = state_converter.state_to_index(state) delta += gamma_n * np.max(q_value[index]) q_value[prev_index][action] += alpha * delta q_value = state_converter.get_initial_q_value() queue = Queue() time_steps = [] for i in range(episode_count): if not queue.empty(): queue.get_nowait() for _ in range(n_step): queue.put((None, None, 0)) queue.get() G = 0 prev_state = env.reset() for t in range(max_steps): if render: env.render() action = get_action(q_value, state_converter, prev_state) state, reward, is_terminated, _ = env.step(action) reward = - \ 100 if is_terminated and t < int(max_steps * 0.975) else 1 queue.put((prev_state, action, reward)) target_state, target_action, target_reward = queue.get() G += reward * gamma_n G /= gamma if t + 1 >= n_step and target_state is not None: update_q_value(q_value, target_state, target_action, state, G) G -= target_reward prev_state = state if is_terminated: while not queue.empty(): target_state, target_action, target_reward = queue.get() G /= gamma update_q_value(q_value, target_state, target_action, None, G) G -= target_reward print("Episode {} finished after {} timesteps".format( i, t + 1)) time_steps.append(t + 1) break if (i + 1) % 20 == 0: plot(time_steps, 'Time Step', figure_path)
class CommandPort(threading.Thread): """ Command port runs on a separate thread """ def __init__(self, queue_size=0, timeout=.1, port=5000, buffersize=4096, max_connections=5, return_result=False, result_as_json=False, redirect_output=False, share_environ=True): super(CommandPort, self).__init__() self.do_run = True self.return_result = return_result self.result_as_json = result_as_json self.redirect_output = redirect_output self.buffersize = buffersize self.max_connections = max_connections self.share_environ = share_environ self.commands_queue = Queue(queue_size) self.output_queue = Queue(queue_size) self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.socket.settimeout(timeout) self.socket.bind(('localhost', port)) def run(self): # ---- Run while main thread of Blender is Alive # This feels a little bit hacky, but that was the most reliable way I've found # There's no method or callback that Blender calls on exit that could be used to close the port # So I'm detecting if a main thread of blender did finish working. # If it did, then I'm breaking the loop and closing the port. threads = threading.enumerate() while any([t.name == "MainThread" and t.isAlive() for t in threads]): if not self.do_run: # ---- Break also if user requested closing the port. print("do_run is False") break self.socket.listen(self.max_connections) try: connection, address = self.socket.accept() data = connection.recv(self.buffersize) size = sys.getsizeof(data) if size >= self.buffersize: print("The length of input is probably too long: {}".format(size)) if size >= 0: command = data.decode() self.commands_queue.put(command) if self.redirect_output or self.return_result: while True: try: output = self.output_queue.get_nowait() except Empty: continue else: if isinstance(output, ResultContainer): if self.result_as_json: result = json.dumps(output.value) else: result = str(output.value) connection.sendall(result.encode()) break elif output and output != "\n": connection.sendall(output.encode()) else: connection.sendall('OK'.encode()) connection.shutdown(socket.SHUT_RDWR) connection.close() except socket.timeout: pass self.socket.close() print("Closing the socket") return
class ResourcePool(object): def __init__(self, cls, *args, capacity=0, idle_timeout=None, autowrap=False, close_on_exc=False, **kwargs): """Create a new pool object. @param cls(object): The object class to be manage. @param args(tuple): The positional parameters of cls. @param kwargs(dict): The key parameters of cls. @param capacity(int): The maximum capacity of the pool. If 0, the capacity is infinite. @param idle_timeout(int): The idle timeout. The unit is second. If None or 0, never time out. @param autowrap(bool): If True, it will wrap the obj in ResourcePoolSession automatically, which will release the obj into the pool when the session is closed or deleted. @param close_on_exc(bool): If True and autowrap is True, in with context, the session will close the obj firstly, then new an new one into the pool. """ capacity = capacity if capacity >= 0 else 0 self._cls = cls self._args = args self._kwargs = kwargs self._closed = False self._lock = Lock() self._capacity = capacity self._timeout = idle_timeout self._pools = Queue(capacity) self._autowrap = autowrap self._close_on_exc = close_on_exc while capacity > 0: self.put(None) capacity -= 1 def __del__(self): self.close() def _get_now(self): return int(time.time()) def _close_obj(self, obj): if obj: try: obj.close() except Exception: pass def close(self): """Close the pool and release all the objects. When closed, it will raise an RuntimeError if putting an object into it. """ with self._lock: if self._closed: return self._closed = True while True: try: self._close_obj(self._pools.get_nowait()[0]) self._pools.task_done() except Empty: return def get(self, timeout=None): """Get an object from the pool. When the pool is closed, it will raise a RuntimeError if calling this method. """ with self._lock: if self._closed: raise RuntimeError("The pool has been closed.") _get = lambda obj: _ResourcePoolSession(self, obj, self._close_on_exc) \ if obj and self._autowrap else obj if not self._capacity: try: obj = self._pools.get_nowait() self._pools.task_done() except Empty: obj = (self._cls(*self._args, **self._kwargs), self._get_now()) else: obj = self._pools.get(timeout=timeout) self._pools.task_done() if obj and obj[0]: if self._timeout and self._get_now() - obj[1] > self._timeout: return _get(self.get(timeout=timeout)) return _get(obj[0]) return _get(self._cls(*self._args, **self._kwargs)) def put(self, obj): """Put an object into the pool. When the pool is closed, it will close the object, not put it into the pool, if calling this method. """ with self._lock: if self._closed: self._close_obj(obj) return if isinstance(obj, _ResourcePoolSession): obj.release_to_pool() else: self._pools.put_nowait((obj, self._get_now())) def put_with_close(self, obj): self._close_obj(obj) self.put(None) def _put_from_session(self, obj): self._pools.put_nowait((obj, self._get_now()))
def uniform_cost_search(problem_subclass): num_nodes_expanded = 0 # Define a node of the form: (path_cost, state) node = Node(state=problem_subclass.initial, path_cost=0) frontier = Queue() # Add the initial node to the frontier: frontier.put_nowait(node) print("Just Added Node (State: %s, Action: %s, PC: %d) to Frontier." % (node.state, node.action, node.path_cost)) print("Frontier Now: %s" % frontier.queue) # Initialize the explored set: explored = set() while True: if frontier.empty(): # Failure, no solution. print( "CRITICAL: Frontier now empty. No solution possible. Search Failure." ) return None, num_nodes_expanded node = frontier.get_nowait() print( "Just Removed Node (State: %s, Action: %s, PC: %d) from Frontier for expansion." % (node.state, node.action, node.path_cost)) # print("Frontier Now Length %d: %s" % (len(frontier.queue), frontier.queue)) if problem_subclass.goal_test(node.state): print("CRITICAL: Reached goal state! Returning solution...") solution_string = get_solution_from_node(goal_node=node) return solution_string, num_nodes_expanded explored.add(node) print( "Just added Node (State: %s, Action: %s, PC: %d) to Explored (if not already in set)." % (node.state, node.action, node.path_cost)) # print("Explored now: %s" % explored) for action in problem_subclass.actions(node.state): resultant_state = problem_subclass.result(state=node.state, action=action) path_cost = problem_subclass.path_cost(c=1, state1=node.state, action=action, state2=resultant_state) child_node = Node(state=resultant_state, path_cost=path_cost + node.path_cost, problem=problem_subclass, node=node, action=action) num_nodes_expanded += 1 # child_node.problem.print_world(child_node.state) print( "Generated new child_node (State: %s, Action Performed: %s, PC: %d) for consideration." % (child_node.state, child_node.action, child_node.path_cost)) print("Agent moved FROM: %s TO: %s with action %s" % (node.state['agent_loc'], child_node.state['agent_loc'], action)) if child_node not in explored: # The child node is not explored. if child_node not in frontier.queue: # The child node is not explored, and is not in the frontier. # Add the child node to the frontier for expansion. frontier.put_nowait(child_node) print( "Just Added Node (State: %s, Action: %s, PC: %d) to Frontier." % (child_node.state, child_node.action, child_node.path_cost)) # print("Frontier Now: %s" % frontier.queue) else: # The child node is not explored, and is in the frontier. # Does the child in the frontier have a higher path cost: for index, frontier_node in enumerate(frontier.queue): if frontier_node == child_node: if frontier_node.path_cost > child_node.path_cost: # The frontier's copy of the node has a higher path-cost. # Replace the frontier's copy with the new copy with lower path cost. frontier.queue[index] = child_node else: # The child node is explored. print( "The generated child node was already in Explored. Generating a new one..." )
class IntcodeComputer: default_instruction_classes: List[Type[Instruction]] = [ AddInstruction, MultiplyInstruction, HaltInstruction, ] def __init__(self, initial: Union[str, Iterable[Union[int, str]]], instruction_classes: Iterable[Type[Instruction]] = None, stdin: Iterable[int] = None): if hasattr(initial, 'split'): initial = initial.split(',') self.memory: MutableMapping[int, int] = defaultdict( int, enumerate(int(val) for val in initial)) self.iptr = 0 self.rbptr = 0 self._stdin = Queue() for item in stdin or []: self.put_stdin(item) self._stdout = Queue() self.jumped: Optional[bool] = None self.instructions: MutableMapping[Opcode, Instruction] = {} self._alive = False self._started = False for instruction in instruction_classes or self.default_instruction_classes: self.register_instr(instruction) @property def is_terminated(self) -> bool: return self._started and not self._alive def get_stdin(self) -> int: return self._stdin.get() def get_stdout(self) -> int: return self._stdout.get() def get_stdout_nowait(self) -> int: return self._stdout.get_nowait() def put_stdout(self, value: int) -> None: self._stdout.put(value) def put_stdin(self, value: int) -> None: self._stdin.put(value) @property def stdout(self) -> Iterable[int]: with suppress(Empty): while True: yield self._stdout.get_nowait() def register_instr(self, instr_cls: Type[Instruction]) -> None: self.instructions[instr_cls.opcode] = instr_cls(self) def kill(self) -> None: self._alive = False def run(self) -> None: with suppress(Halt): while not self.is_terminated: self.tick() def tick(self) -> None: if not self._started: self._started = True self._alive = True if not self._alive: raise InvalidState("Computer has already halted!") # LOG.debug('iptr=%s, rbptr=%s, MEM=%s', self.iptr, self.rbptr, str(self)) opcode = Opcode(self.memory[self.iptr] % 100) # LOG.debug('Executing %s', opcode.name) instr = self.instructions[opcode] self.jumped = False instr.execute() if not self.jumped: self.iptr += 1 + instr.parameter_count def jump(self, newptr: int) -> None: self.jumped = True self.iptr = newptr def get_parameter(self, idx: int) -> int: parameter_mode = self.get_parameter_mode(idx) if parameter_mode == ParameterMode.POSITION: return self.memory[self.memory[self.iptr + idx]] elif parameter_mode == ParameterMode.IMMEDIATE: return self.memory[self.iptr + idx] elif parameter_mode == ParameterMode.RELATIVE: return self.memory[self.memory[self.iptr + idx] + self.rbptr] else: raise ParameterError("Invalid parameter mode: %s", parameter_mode) def get_parameter_mode(self, idx: int) -> ParameterMode: digit = (self.memory[self.iptr] // 10**(idx + 1)) % 10 return ParameterMode(digit) def set_parameter(self, idx: int, value: int) -> None: parameter_mode = self.get_parameter_mode(idx) if parameter_mode == ParameterMode.POSITION: self.memory[self.memory[self.iptr + idx]] = value elif parameter_mode == ParameterMode.RELATIVE: self.memory[self.memory[self.iptr + idx] + self.rbptr] = value else: raise ParameterError("Invalid parameter mode: %s", parameter_mode) def __str__(self) -> str: ret = [] last_address = 0 for address, value in self.memory.items(): item = '' if address > last_address + 1: item += f'...{hex(address)}::' item += str(value) ret.append(item) last_address = address return ','.join(ret)
class ThreadedSelectReactor(posixbase.PosixReactorBase): """A threaded select() based reactor - runs on all POSIX platforms and on Win32. """ def __init__(self): threadable.init(1) self.reads = {} self.writes = {} self.toThreadQueue = Queue() self.toMainThread = Queue() self.workerThread = None self.mainWaker = None posixbase.PosixReactorBase.__init__(self) self.addSystemEventTrigger("after", "shutdown", self._mainLoopShutdown) def wakeUp(self): # we want to wake up from any thread self.waker.wakeUp() def callLater(self, *args, **kw): tple = posixbase.PosixReactorBase.callLater(self, *args, **kw) self.wakeUp() return tple def _sendToMain(self, msg, *args): self.toMainThread.put((msg, args)) if self.mainWaker is not None: self.mainWaker() def _sendToThread(self, fn, *args): self.toThreadQueue.put((fn, args)) def _preenDescriptorsInThread(self): log.msg("Malformed file descriptor found. Preening lists.") readers = self.reads.keys() writers = self.writes.keys() self.reads.clear() self.writes.clear() for selDict, selList in ((self.reads, readers), (self.writes, writers)): for selectable in selList: try: select.select([selectable], [selectable], [selectable], 0) except BaseException: log.msg("bad descriptor %s" % selectable) else: selDict[selectable] = 1 def _workerInThread(self): try: while 1: fn, args = self.toThreadQueue.get() fn(*args) except SystemExit: pass # Exception indicates this thread should exit except BaseException: f = failure.Failure() self._sendToMain("Failure", f) def _doSelectInThread(self, timeout): """Run one iteration of the I/O monitor loop. This will run all selectables who had input or output readiness waiting for them. """ reads = self.reads writes = self.writes while 1: try: r, w, ignored = _select(reads.keys(), writes.keys(), [], timeout) break except ValueError: # Possibly a file descriptor has gone negative? log.err() self._preenDescriptorsInThread() except TypeError: # Something *totally* invalid (object w/o fileno, non-integral # result) was passed log.err() self._preenDescriptorsInThread() except OSError as se: # select(2) encountered an error if se.args[0] in (0, 2): # windows does this if it got an empty list if (not reads) and (not writes): return else: raise elif se.args[0] == EINTR: return elif se.args[0] == EBADF: self._preenDescriptorsInThread() else: # OK, I really don't know what's going on. Blow up. raise self._sendToMain("Notify", r, w) def _process_Notify(self, r, w): reads = self.reads writes = self.writes _drdw = self._doReadOrWrite _logrun = log.callWithLogger for selectables, method, dct in ((r, "doRead", reads), (w, "doWrite", writes)): for selectable in selectables: # if this was disconnected in another thread, kill it. if selectable not in dct: continue # This for pausing input when we're not ready for more. _logrun(selectable, _drdw, selectable, method, dct) def _process_Failure(self, f): f.raiseException() _doIterationInThread = _doSelectInThread def ensureWorkerThread(self): if self.workerThread is None or not self.workerThread.isAlive(): self.workerThread = Thread(target=self._workerInThread) self.workerThread.start() def doThreadIteration(self, timeout): self._sendToThread(self._doIterationInThread, timeout) self.ensureWorkerThread() msg, args = self.toMainThread.get() getattr(self, "_process_" + msg)(*args) doIteration = doThreadIteration def _interleave(self): while self.running: self.runUntilCurrent() t2 = self.timeout() t = self.running and t2 self._sendToThread(self._doIterationInThread, t) yield None msg, args = self.toMainThread.get_nowait() getattr(self, "_process_" + msg)(*args) def interleave(self, waker, *args, **kw): """ interleave(waker) interleaves this reactor with the current application by moving the blocking parts of the reactor (select() in this case) to a separate thread. This is typically useful for integration with GUI applications which have their own event loop already running. See the module docstring for more information. """ self.startRunning(*args, **kw) loop = self._interleave() def mainWaker(waker=waker, loop=loop): waker(partial(next, loop)) self.mainWaker = mainWaker next(loop) self.ensureWorkerThread() def _mainLoopShutdown(self): self.mainWaker = None if self.workerThread is not None: self._sendToThread(raiseException, SystemExit) self.wakeUp() try: while 1: msg, args = self.toMainThread.get_nowait() except Empty: pass self.workerThread.join() self.workerThread = None try: while 1: fn, args = self.toThreadQueue.get_nowait() if fn is self._doIterationInThread: log.msg("Iteration is still in the thread queue!") elif fn is raiseException and args[0] is SystemExit: pass else: fn(*args) except Empty: pass def _doReadOrWrite(self, selectable, method, dict): try: why = getattr(selectable, method)() handfn = getattr(selectable, "fileno", None) if not handfn: why = _NO_FILENO elif handfn() == -1: why = _NO_FILEDESC except BaseException: why = sys.exc_info()[1] log.err() if why: self._disconnectSelectable(selectable, why, method == "doRead") def addReader(self, reader): """Add a FileDescriptor for notification of data available to read.""" self._sendToThread(self.reads.__setitem__, reader, 1) self.wakeUp() def addWriter(self, writer): """Add a FileDescriptor for notification of data available to write.""" self._sendToThread(self.writes.__setitem__, writer, 1) self.wakeUp() def removeReader(self, reader): """Remove a Selectable for notification of data available to read.""" self._sendToThread(dictRemove, self.reads, reader) def removeWriter(self, writer): """Remove a Selectable for notification of data available to write.""" self._sendToThread(dictRemove, self.writes, writer) def removeAll(self): return self._removeAll(self.reads, self.writes) def getReaders(self): return list(self.reads.keys()) def getWriters(self): return list(self.writes.keys()) def stop(self): """ Extend the base stop implementation to also wake up the select thread so that C{runUntilCurrent} notices the reactor should stop. """ posixbase.PosixReactorBase.stop(self) self.wakeUp() def run(self, installSignalHandlers=True): self.startRunning(installSignalHandlers=installSignalHandlers) self.mainLoop() def mainLoop(self): q = Queue() self.interleave(q.put) while self.running: try: q.get()() except StopIteration: break
class Boss(Thread): daemon = True def __init__(self, glfw_window, opts, args): Thread.__init__(self, name='ChildMonitor') startup_session = create_session(opts, args) self.cursor_blink_zero_time = monotonic() self.cursor_blinking = True self.glfw_window_title = None self.action_queue = Queue() self.pending_resize = False self.resize_gl_viewport = False self.shutting_down = False self.screen_update_delay = opts.repaint_delay / 1000.0 self.signal_fd = handle_unix_signals() self.read_wakeup_fd, self.write_wakeup_fd = os.pipe2(os.O_NONBLOCK | os.O_CLOEXEC) self.read_dispatch_map = {self.signal_fd: self.signal_received, self.read_wakeup_fd: self.on_wakeup} self.all_writers = [] self.timers = Timers() self.ui_timers = Timers() self.pending_ui_thread_calls = Queue() self.write_dispatch_map = {} set_boss(self) cell_size.width, cell_size.height = set_font_family(opts.font_family, opts.font_size) self.opts, self.args = opts, args self.glfw_window = glfw_window glfw_window.framebuffer_size_callback = self.on_window_resize glfw_window.char_mods_callback = self.on_text_input glfw_window.key_callback = self.on_key glfw_window.mouse_button_callback = self.on_mouse_button glfw_window.scroll_callback = self.on_mouse_scroll glfw_window.cursor_pos_callback = self.on_mouse_move glfw_window.window_focus_callback = self.on_focus self.tab_manager = TabManager(opts, args, startup_session) self.sprites = Sprites() self.cell_program = ShaderProgram(*cell_shader) self.cursor_program = ShaderProgram(*cursor_shader) self.borders_program = BordersProgram() glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA) self.sprites.do_layout(cell_size.width, cell_size.height) self.glfw_window.set_click_cursor(False) self.show_mouse_cursor() self.start_cursor_blink() def signal_received(self): try: data = os.read(self.signal_fd, io.DEFAULT_BUFFER_SIZE) except BlockingIOError: return if data: signals = struct.unpack('%uB' % len(data), data) if signal.SIGINT in signals or signal.SIGTERM in signals: if not self.shutting_down: self.glfw_window.set_should_close(True) glfw_post_empty_event() @property def current_tab_bar_height(self): return self.tab_manager.tab_bar_height def __iter__(self): return iter(self.tab_manager) def iterwindows(self): for t in self: yield from t def queue_action(self, func, *args): self.action_queue.put((func, args)) wakeup() def on_wakeup(self): if not self.shutting_down: drain_read(self.read_wakeup_fd) while True: try: func, args = self.action_queue.get_nowait() except Empty: break try: func(*args) except Exception: import traceback traceback.print_exc() def add_child_fd(self, child_fd, read_ready, write_ready): self.read_dispatch_map[child_fd] = read_ready self.write_dispatch_map[child_fd] = write_ready def remove_child_fd(self, child_fd): self.read_dispatch_map.pop(child_fd, None) self.write_dispatch_map.pop(child_fd, None) try: self.all_writers.remove(child_fd) except Exception: pass def queue_ui_action(self, func, *args): self.pending_ui_thread_calls.put((func, args)) glfw_post_empty_event() def close_window(self, window=None): ' Can be called in either thread, will first kill the child, then remove the window from the gui ' if window is None: window = self.active_window if current_thread() is main_thread: self.queue_action(self.close_window, window) else: self.remove_child_fd(window.child_fd) window.destroy() self.queue_ui_action(self.gui_close_window, window) def close_tab(self, tab=None): ' Can be called in either thread, will first kill all children, then remove the tab from the gui ' if tab is None: tab = self.active_tab if current_thread() is main_thread: self.queue_action(self.close_tab, tab) else: for window in tab: self.remove_child_fd(window.child_fd) window.destroy() self.queue_ui_action(self.gui_close_window, window) def run(self): if self.args.profile: import cProfile import pstats pr = cProfile.Profile() pr.enable() self.loop() if self.args.profile: pr.disable() pr.create_stats() s = pstats.Stats(pr) s.dump_stats(self.args.profile) def loop(self): while not self.shutting_down: all_readers = list(self.read_dispatch_map) all_writers = [w.child_fd for w in self.iterwindows() if w.write_buf] readers, writers, _ = select.select(all_readers, all_writers, [], self.timers.timeout()) for r in readers: self.read_dispatch_map[r]() for w in writers: self.write_dispatch_map[w]() self.timers() for w in self.iterwindows(): if w.screen.is_dirty(): self.timers.add_if_missing(self.screen_update_delay, w.update_screen) @callback def on_window_resize(self, window, w, h): # debounce resize events self.pending_resize = True yield self.timers.add(0.02, self.apply_pending_resize, w, h) def apply_pending_resize(self, w, h): viewport_size.width, viewport_size.height = w, h self.tab_manager.resize() self.resize_gl_viewport = True self.pending_resize = False glfw_post_empty_event() def tabbar_visibility_changed(self): self.tab_manager.resize(only_tabs=True) glfw_post_empty_event() @property def active_tab(self): return self.tab_manager.active_tab def is_tab_visible(self, tab): return self.active_tab is tab @property def active_window(self): t = self.active_tab if t is not None: return t.active_window @callback def on_text_input(self, window, codepoint, mods): data = interpret_text_event(codepoint, mods) if data: w = self.active_window if w is not None: yield w w.write_to_child(data) @callback def on_key(self, window, key, scancode, action, mods): is_key_pressed[key] = action == GLFW_PRESS self.start_cursor_blink() self.cursor_blink_zero_time = monotonic() if action == GLFW_PRESS or action == GLFW_REPEAT: func = get_shortcut(self.opts.keymap, mods, key) if func is not None: f = getattr(self, func, None) if f is not None: passthrough = f() if not passthrough: return tab = self.active_tab if tab is None: return window = self.active_window if window is not None: yield window if func is not None: f = getattr(tab, func, getattr(window, func, None)) if f is not None: passthrough = f() if not passthrough: return if window.screen.auto_repeat_enabled or action == GLFW_PRESS: if window.char_grid.scrolled_by and key not in MODIFIER_KEYS: window.scroll_end() data = interpret_key_event(key, scancode, mods) if data: window.write_to_child(data) @callback def on_focus(self, window, focused): w = self.active_window if w is not None: yield w w.focus_changed(focused) def display_scrollback(self, data): if self.opts.scrollback_in_new_tab: self.queue_ui_action(self.display_scrollback_in_new_tab, data) else: tab = self.active_tab if tab is not None: tab.new_special_window(SpecialWindow(self.opts.scrollback_pager, data, _('History'))) def window_for_pos(self, x, y): tab = self.active_tab if tab is not None: for w in tab: if w.is_visible_in_layout and w.contains(x, y): return w def in_tab_bar(self, y): th = self.current_tab_bar_height return th > 0 and y >= viewport_size.height - th @callback def on_mouse_button(self, window, button, action, mods): mouse_button_pressed[button] = action == GLFW_PRESS self.show_mouse_cursor() x, y = mouse_cursor_pos w = self.window_for_pos(x, y) if w is None: if self.in_tab_bar(y): if button == GLFW_MOUSE_BUTTON_1 and action == GLFW_PRESS: self.tab_manager.activate_tab_at(x) return focus_moved = False old_focus = self.active_window tab = self.active_tab yield if button == GLFW_MOUSE_BUTTON_1 and w is not old_focus: tab.set_active_window(w) focus_moved = True if focus_moved: if old_focus is not None and not old_focus.destroyed: old_focus.focus_changed(False) w.focus_changed(True) w.on_mouse_button(button, action, mods) @callback def on_mouse_move(self, window, xpos, ypos): mouse_cursor_pos[:2] = xpos, ypos self.show_mouse_cursor() w = self.window_for_pos(xpos, ypos) if w is not None: yield w w.on_mouse_move(xpos, ypos) else: self.change_mouse_cursor(self.in_tab_bar(ypos)) @callback def on_mouse_scroll(self, window, x, y): self.show_mouse_cursor() w = self.window_for_pos(*mouse_cursor_pos) if w is not None: yield w w.on_mouse_scroll(x, y) # GUI thread API {{{ def show_mouse_cursor(self): self.glfw_window.set_input_mode(GLFW_CURSOR, GLFW_CURSOR_NORMAL) if self.opts.mouse_hide_wait > 0: self.ui_timers.add(self.opts.mouse_hide_wait, self.hide_mouse_cursor) def hide_mouse_cursor(self): self.glfw_window.set_input_mode(GLFW_CURSOR, GLFW_CURSOR_HIDDEN) def change_mouse_cursor(self, click=False): self.glfw_window.set_click_cursor(click) def start_cursor_blink(self): self.cursor_blinking = True if self.opts.cursor_stop_blinking_after > 0: self.ui_timers.add(self.opts.cursor_stop_blinking_after, self.stop_cursor_blinking) def stop_cursor_blinking(self): self.cursor_blinking = False def render(self): if self.pending_resize: return if self.resize_gl_viewport: glViewport(0, 0, viewport_size.width, viewport_size.height) self.resize_gl_viewport = False tab = self.active_tab if tab is not None: if tab.title != self.glfw_window_title: self.glfw_window_title = tab.title self.glfw_window.set_title(self.glfw_window_title) with self.sprites: self.sprites.render_dirty_cells() tab.render() render_data = {window: window.char_grid.prepare_for_render(self.sprites) for window in tab.visible_windows() if not window.needs_layout} with self.cell_program: self.tab_manager.render(self.cell_program, self.sprites) for window, rd in render_data.items(): if rd is not None: window.char_grid.render_cells(rd, self.cell_program, self.sprites) active = self.active_window rd = render_data.get(active) if rd is not None: draw_cursor = True if self.cursor_blinking and self.opts.cursor_blink_interval > 0: now = monotonic() - self.cursor_blink_zero_time t = int(now * 1000) d = int(self.opts.cursor_blink_interval * 1000) n = t // d draw_cursor = n % 2 == 0 self.ui_timers.add_if_missing(((n + 1) * d / 1000) - now, glfw_post_empty_event) if draw_cursor: with self.cursor_program: active.char_grid.render_cursor(rd, self.cursor_program) def gui_close_window(self, window): if window.char_grid.buffer_id is not None: self.sprites.destroy_sprite_map(window.char_grid.buffer_id) window.char_grid.buffer_id = None for tab in self.tab_manager: if window in tab: break else: return tab.remove_window(window) if len(tab) == 0: self.tab_manager.remove(tab) tab.destroy() if len(self.tab_manager) == 0: if not self.shutting_down: self.glfw_window.set_should_close(True) glfw_post_empty_event() def destroy(self): # Must be called in the main thread as it manipulates signal handlers signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) self.shutting_down = True wakeup() self.join() for t in self.tab_manager: t.destroy() del self.tab_manager self.sprites.destroy() del self.sprites del self.glfw_window def paste_from_clipboard(self): text = self.glfw_window.get_clipboard_string() if text: w = self.active_window if w is not None: self.queue_action(w.paste, text) def next_tab(self): self.queue_action(self.tab_manager.next_tab) def previous_tab(self): self.queue_action(self.tab_manager.next_tab, -1) def new_tab(self): self.tab_manager.new_tab() def move_tab_forward(self): self.queue_action(self.tab_manager.move_tab, 1) def move_tab_backward(self): self.queue_action(self.tab_manager.move_tab, -1) def display_scrollback_in_new_tab(self, data): self.tab_manager.new_tab(special_window=SpecialWindow(self.opts.scrollback_pager, data, _('History')))
class MainWindow(QMainWindow): def __init__(self): super(QMainWindow, self).__init__() self.img_num = 10 self.scale = 5 self.fq = Queue(maxsize=self.img_num) self.fq_lock = Lock() self.UI_resized = False self.img_wnds = [None] * self.img_num self.init_UI() def put_fq(self, frame): self.img_w = int(frame.shape[1] / self.scale) self.img_h = int(frame.shape[0] / self.scale) show = cv2.resize(frame, (self.img_w, self.img_h), interpolation=cv2.INTER_AREA) show = cv2.cvtColor(show, cv2.COLOR_BGR2RGB) show = QtGui.QImage(show.data, show.shape[1], show.shape[0], QtGui.QImage.Format_RGB888) with self.fq_lock: if self.fq.full(): self.fq.get_nowait() self.fq.put_nowait(show) #resize windows if not self.UI_resized: self.UI_resized = True self.scroll.show() self.imgarea.resize(self.img_w, self.img_h * self.img_num) #######设置scrollarea幕布的尺寸 self.setFixedWidth(frame.shape[1] + self.img_w + 50) self.setFixedHeight(frame.shape[0]) frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) frame = QtGui.QImage(frame.data, frame.shape[1], frame.shape[0], QtGui.QImage.Format_RGB888) self.video_wnd.setPixmap(QtGui.QPixmap.fromImage(frame)) self.show_fq() def init_UI(self): self.imgarea = QWidget() vbox = QVBoxLayout() for img_cnt in range(self.img_num): self.img_wnds[img_cnt] = QLabel() vbox.addWidget(self.img_wnds[img_cnt]) self.imgarea.setLayout(vbox) self.scroll = QScrollArea() self.scroll.setWidget(self.imgarea) self.scroll.hide() self.video_wnd = QLabel() hbox = QHBoxLayout() hbox.addWidget(self.video_wnd) hbox.addWidget(self.scroll) w = QWidget() w.setLayout(hbox) self.setCentralWidget(w) menu_height = 0 self.resize(641, 481 + menu_height) self.video_wnd.setGeometry( QtCore.QRect(0, menu_height, 641, 481 + menu_height)) self.move(100, 100) def show_fq(self): with self.fq_lock: for frame_idx in range(self.fq.qsize()): self.img_wnds[frame_idx].setPixmap( QtGui.QPixmap.fromImage(self.fq.queue[frame_idx]))
class ItemBuffer(threading.Thread, Singleton): dedup = None def __init__(self, table_folder): if not hasattr(self, "_table_item"): super(ItemBuffer, self).__init__() self._thread_stop = False self._is_adding_to_db = False self._table_folder = table_folder self._items_queue = Queue(maxsize=MAX_ITEM_COUNT) self._db = RedisDB() self._table_item = setting.TAB_ITEM self._table_request = setting.TAB_REQUSETS.format(table_folder=table_folder) self._item_tables = { # 'xxx_item': {'tab_item': 'xxx:xxx_item'} # 记录item名与redis中item名对应关系 } self._item_update_keys = { # 'xxx:xxx_item': ['id', 'name'...] # 记录redis中item名与需要更新的key对应关系 } self._export_data = ExportData() if setting.ADD_ITEM_TO_MYSQL else None self.db_tip() if setting.ITEM_FILTER_ENABLE and not self.__class__.dedup: self.__class__.dedup = Dedup(to_md5=False) def db_tip(self): msg = "\n" if setting.ADD_ITEM_TO_MYSQL: msg += "item 自动入mysql\n" if setting.ADD_ITEM_TO_REDIS: msg += "item 自动入redis\n" if msg == "\n": log.warning("*** 请注意检查item是否入库 !!!") else: log.info(msg) def run(self): while not self._thread_stop: self.flush() tools.delay_time(0.5) self.close() def stop(self): self._thread_stop = True def put_item(self, item): self._items_queue.put(item) def flush(self): try: items = [] update_items = [] requests = [] callbacks = [] items_fingerprints = [] data_count = 0 while not self._items_queue.empty(): data = self._items_queue.get_nowait() data_count += 1 # data 分类 if callable(data): callbacks.append(data) elif isinstance(data, UpdateItem): update_items.append(data) elif isinstance(data, Item): items.append(data) if setting.ITEM_FILTER_ENABLE: items_fingerprints.append(data.fingerprint) else: # request-redis requests.append(data) if data_count >= UPLOAD_BATCH_MAX_SIZE: self.__add_item_to_db( items, update_items, requests, callbacks, items_fingerprints ) items = [] update_items = [] requests = [] callbacks = [] items_fingerprints = [] data_count = 0 if data_count: self.__add_item_to_db( items, update_items, requests, callbacks, items_fingerprints ) except Exception as e: log.exception(e) def get_items_count(self): return self._items_queue.qsize() def is_adding_to_db(self): return self._is_adding_to_db def __dedup_items(self, items, items_fingerprints): """ 去重 @param items: @param items_fingerprints: @return: 返回去重后的items, items_fingerprints """ if not items: return items, items_fingerprints is_exists = self.__class__.dedup.get(items_fingerprints) is_exists = is_exists if isinstance(is_exists, list) else [is_exists] dedup_items = [] dedup_items_fingerprints = [] items_count = dedup_items_count = dup_items_count = 0 while is_exists: item = items.pop(0) items_fingerprint = items_fingerprints.pop(0) is_exist = is_exists.pop(0) items_count += 1 if not is_exist: dedup_items.append(item) dedup_items_fingerprints.append(items_fingerprint) dedup_items_count += 1 else: dup_items_count += 1 log.info( "待入库数据 {} 条, 重复 {} 条,实际待入库数据 {} 条".format( items_count, dup_items_count, dedup_items_count ) ) return dedup_items, dedup_items_fingerprints def __pick_items(self, items, is_update_item=False): """ 将每个表之间的数据分开 拆分后 原items为空 @param items: @param is_update_item: @return: """ datas_dict = { # 'xxx:xxx_item': [{}, {}] redis 中的item名与对应的数据 } while items: item = items.pop(0) # 取item下划线格式的名 # 下划线类的名先从dict中取,没有则现取,然后存入dict。加快下次取的速度 item_name = item.item_name item_table = self._item_tables.get(item_name) if not item_table: item_name_underline = item.name_underline tab_item = self._table_item.format( table_folder=self._table_folder, item_name=item_name_underline ) item_table = {} item_table["tab_item"] = tab_item self._item_tables[item_name] = item_table else: tab_item = item_table.get("tab_item") # 入库前的回调 item.per_to_db() if tab_item not in datas_dict: datas_dict[tab_item] = [] datas_dict[tab_item].append(item.to_dict) if is_update_item and tab_item not in self._item_update_keys: self._item_update_keys[tab_item] = item.update_key return datas_dict def __export_to_db(self, tab_item, datas, is_update=False, update_keys=()): export_success = False # 打点 校验 to_table = tools.get_info(tab_item, ":s_(.*?)_item", fetch_one=True) item_name = to_table + "_item" self.check_datas(table=to_table, datas=datas) if setting.ADD_ITEM_TO_MYSQL: # 任务表需要入mysql if isinstance(setting.ADD_ITEM_TO_MYSQL, (list, tuple)): for item in setting.ADD_ITEM_TO_MYSQL: if item in item_name: export_success = ( self._export_data.export_items(tab_item, datas) if not is_update else self._export_data.update_items( tab_item, datas, update_keys=update_keys ) ) else: export_success = ( self._export_data.export_items(tab_item, datas) if not is_update else self._export_data.update_items( tab_item, datas, update_keys=update_keys ) ) if setting.ADD_ITEM_TO_REDIS: if isinstance(setting.ADD_ITEM_TO_REDIS, (list, tuple)): for item in setting.ADD_ITEM_TO_REDIS: if item in item_name: self._db.sadd(tab_item, datas) export_success = True log.info("共导出 %s 条数据 到redis %s" % (len(datas), tab_item)) break else: self._db.sadd(tab_item, datas) export_success = True log.info("共导出 %s 条数据 到redis %s" % (len(datas), tab_item)) return export_success def __add_item_to_db( self, items, update_items, requests, callbacks, items_fingerprints ): export_success = False self._is_adding_to_db = True # 去重 if setting.ITEM_FILTER_ENABLE: items, items_fingerprints = self.__dedup_items(items, items_fingerprints) # 分捡 items_dict = self.__pick_items(items) update_items_dict = self.__pick_items(update_items, is_update_item=True) # item批量入库 while items_dict: tab_item, datas = items_dict.popitem() log.debug( """ -------------- item 批量入库 -------------- 表名: %s datas: %s """ % (tab_item, tools.dumps_json(datas, indent=16)) ) export_success = self.__export_to_db(tab_item, datas) # 执行批量update while update_items_dict: tab_item, datas = update_items_dict.popitem() log.debug( """ -------------- item 批量更新 -------------- 表名: %s datas: %s """ % (tab_item, tools.dumps_json(datas, indent=16)) ) update_keys = self._item_update_keys.get(tab_item) export_success = self.__export_to_db( tab_item, datas, is_update=True, update_keys=update_keys ) # 执行回调 while callbacks: try: callback = callbacks.pop(0) callback() except Exception as e: log.exception(e) # 删除做过的request if requests: self._db.zrem(self._table_request, requests) # 去重入库 if export_success and setting.ITEM_FILTER_ENABLE: if items_fingerprints: self.__class__.dedup.add(items_fingerprints, skip_check=True) self._is_adding_to_db = False def check_datas(self, table, datas): """ 打点 记录总条数及每个key情况 @param table: 表名 @param datas: 数据 列表 @return: """ pass def close(self): pass
class Process(object): def __init__(self, commands, context, cwd): startupinfo = None if os.name == 'nt': startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW self._proc = subprocess.Popen(commands, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, startupinfo=startupinfo, cwd=cwd) self._eof = False self._context = context self._queue_out = Queue() self._thread = Thread(target=self.enqueue_output) self._thread.start() def eof(self): return self._eof def kill(self): if not self._proc: return if self._proc.poll() is None: # _proc is running self._proc.kill() self._proc.wait() self._proc = None self._queue_out = None self._thread.join(1.0) self._thread = None def enqueue_output(self): for line in self._proc.stdout: if not self._queue_out: return self._queue_out.put( line.decode(self._context['encoding'], errors='replace').strip('\r\n')) def communicate(self, timeout): if not self._proc: return ([], []) start = time() outs = [] if self._queue_out.empty(): sleep(0.1) while not self._queue_out.empty() and time() < start + timeout: outs.append(self._queue_out.get_nowait()) if self._thread.is_alive() or not self._queue_out.empty(): return (outs, []) _, errs = self._proc.communicate(timeout=timeout) errs = errs.decode(self._context['encoding'], errors='replace').splitlines() self._eof = True self._proc = None self._thread = None self._queue = None return (outs, errs)
class TesterManager(object): """Manage evaluate data.""" def __init__(self, config_info, broker_master, s3_result_path=None): self.config_info = config_info self.s3_path = s3_result_path self.processed_model_count = 0 self.record_station_buf = dict() self.eval_rewards = deque([], maxlen=50) self.recv_broker = broker_master.register("eval_result") self.send_broker = broker_master.recv_local_q self.eval_info_queue = Queue() eval_info = self.config_info.get("benchmark", {}).get("eval", dict()) self.max_instance = eval_info.get("evaluator_num", 2) self.eval_interval = eval_info.get( "gap", XBConf.default_train_interval_per_eval) self.max_step_per_episode = eval_info.get("max_step_per_episode", 18000) self.used_node = dict() self.avail_node = list(((i, "test{}".format(tid)) for i in range(broker_master.node_num) for tid in range(self.max_instance))) self.last_eval_index = -9999 def check_finish_stat(self, target_model_count): """Check finish status.""" return self.processed_model_count >= target_model_count def if_eval(self, train_count): if train_count - self.last_eval_index < self.eval_interval: return False return True def to_eval(self, weights, train_count, actual_step, elapsed_time, train_reward, train_loss): # update current evaluate index self.last_eval_index = train_count train_info = { "train_index": train_count, "sample_step": actual_step, "elapsed_sec": elapsed_time, # time.time() - abs_start, "train_reward": train_reward, "loss": np.nan, } if type(train_loss) in (float, np.float64, np.float32, np.float16, np.float): train_info.update({"loss": train_loss}) self.record_station_buf.update({train_count: train_info}) self.put_test_model({train_count: weights}) # return self.get_avail_node() def fetch_eval_result(self): """Fetch eval results with no wait.""" ret = list() while True: try: item = self.eval_info_queue.get_nowait() ret.append(item) except Empty as err: break return ret def _parse_eval_result_and_archive(self, eval_result): _model_index = eval_result[-1][ "train_count"] # model receive is a list # fixme: only a model been test in an agent. _agent_id = list(eval_result[0].keys())[0] # find the eval model info, and update the eval reward if _model_index in self.record_station_buf: eval_info_dict = self.record_station_buf.pop(_model_index) else: eval_info_dict = dict() print("-->", eval_result) return # test.api will use print replace write db record eval_info_dict.update({ "agent_id": _agent_id, "eval_reward": np.nanmean(eval_result[0][_agent_id]["reward"]), "eval_name": _model_index, }) # custom evaluate for _key in ("custom_criteria", "battle_won"): if _key not in eval_result[0][_agent_id].keys(): continue eval_info_dict.update( {_key: np.nanmean(eval_result[0][_agent_id][_key])}) self.eval_info_queue.put(eval_info_dict) def recv_result(self): """Recieve test result.""" while True: recv_data = self.recv_broker.recv() result_data = get_msg_data(recv_data) self.processed_model_count += 1 logging.debug("result_data: \n{}".format(result_data)) _meta = result_data[1] self.used_node[(_meta["broker_id"], _meta["test_id"])] -= 1 self._parse_eval_result_and_archive(result_data) def put_test_model(self, model_weights): """Send test model.""" key = self.get_avail_node() ctr_info = {"cmd": "eval", "broker_id": key[0], "test_id": key[1]} eval_cmd = message(model_weights, **ctr_info) self.send_broker.send(eval_cmd) logging.debug("put evaluate model: {}".format(type(model_weights))) self.used_node[key] += 1 def send_create_evaluator_msg(self, broker_id, test_id): """Create evaluator.""" config = deepcopy(self.config_info) config.update({"test_id": test_id}) create_cmd = message(config, cmd="create_evaluator", broker_id=broker_id) self.send_broker.send(create_cmd) def get_avail_node(self): """Get available test node.""" if self.used_node: min_key = min(self.used_node, key=self.used_node.get) if self.used_node.get(min_key) < 1 or not self.avail_node: return min_key # create new evaluator new_key = self.avail_node.pop(0) self.send_create_evaluator_msg(*new_key) # (broker_id, test_id) self.used_node.update({new_key: 0}) logging.info("create evaluator: {}".format(new_key)) return new_key def start(self): t = threading.Thread(target=self.recv_result) t.start()
class Unit(Entity): """ A unit is an entity that can have its own entitys. (e.g. a Gunner that owns its fired bullets) """ def __init__(self, player_number: int, sprite: UnitSprite = None, max_entities: int = -1, nb_lives: int = 1, surviving_entities: bool = False, speed: int = 150): """ Instantiates a unit in the game Args: sprite: The sprite to draw on the board max_entities: The maximum number of entities for this unit (-1 = infinite) nb_lives: The number of lives this unit has before it dies surviving_entities: If true, the entities of this unit won't die with this unit """ super().__init__(sprite, nb_lives=nb_lives, speed=speed) self.survivingentitys = surviving_entities self.lastAction = None self.currentAction = None self.playerNumber = player_number self._entitiesSpriteGroup = pygame.sprite.Group() self._entitiesQueue = Queue() self._entitiesList = [] self._maxentitys = max_entities def setLastAction(self, last_action): """ Sets the last action of this unit Args: last_action: The last action performed by this unit """ self.lastAction = last_action def setCurrentAction(self, current_action): """ Sets the current action of this unit Args: current_action: The action this unit is currently performing """ self.currentAction = current_action def draw(self, surface: pygame.Surface) -> None: """ Draws the unit and its entities Args: surface: The surface the unit and its entity will be drawn on """ super().draw(surface) self._entitiesSpriteGroup.draw(surface) def kill(self) -> None: """ Kills the unit and all its entity if "surviving entities" was set to False at the creation of this unit """ super().kill() if not self.isAlive(): if not self.survivingentitys: while len(self._entitiesSpriteGroup) != 0: self.removeOldestentity() def addentity(self, entity: Entity) -> None: """ Adds an entity linked to this unit Args: entity: the entity to add to this unit """ if 0 <= self._maxentitys <= len(self._entitiesList): self.removeOldestentity() if self._entitiesQueue is not None: self._entitiesQueue.put(entity) self._entitiesList.append(entity) if entity.sprite is not None: self._entitiesSpriteGroup.add(entity.sprite) def removeOldestentity(self) -> None: """ Removes the oldest entity belonging to this unit- """ try: oldest_entity = self._entitiesList.pop(0) oldest_entity.kill() if self._entitiesQueue is not None: queue_first = self._entitiesQueue.get_nowait() # type: Entity assert queue_first is oldest_entity if oldest_entity.sprite is not None: self._entitiesSpriteGroup.remove(oldest_entity.sprite) except Empty: pass def removeentity(self, entity: Entity) -> None: """ Removes the given entity from the belongings of this unit Args: entity: The entity to remove """ temp_queue = Queue() try: while True: # Will stop when the Empty exception comes out from the Queue current = self._entitiesQueue.get_nowait() if current is not entity: temp_queue.put(current) except Empty: pass self._entitiesList.remove(entity) self._entitiesQueue = temp_queue entity.kill() if entity.sprite is not None: self._entitiesSpriteGroup.remove(entity.sprite) def hasentity(self, entity: Entity) -> bool: """ Checks if the given entity belongs to this unit Args: entity: The entity to check Returns: True if the given entity belongs to this unit """ return entity in self._entitiesList or ( self._entitiesSpriteGroup is not None and self._entitiesSpriteGroup.has(entity.sprite)) def getentitys(self) -> List[Entity]: """ Returns: A list containing all the entities of this unit """ return self._entitiesList def getentitysSpriteGroup(self) -> pygame.sprite.Group: """ Gets the "SpriteGroup" of this unit, including all of its entity. Can be used for pygame's collision check """ return self._entitiesSpriteGroup def isColliding(self, other_sprite_group) -> bool: """ Returns True if this unit or any of its entity collide with the other unit or any of its entity. (Uses the pygame collision check) Args: other_sprite_group: The other Sprite group with which we want to check if there is a collision Returns: True if the two group are colliding, False otherwise """ for sprite in other_sprite_group.sprites: # type: pygame.sprite.Sprite if pygame.sprite.collide_rect(self.sprite, sprite): return True return False def __deepcopy__(self, memo={}): cls = self.__class__ result = cls.__new__(cls) memo[id(self)] = result for k, v in self.__dict__.items(): if k != "_drawable" and k != "_entitiesQueue": value = deepcopy(v, memo) else: value = None setattr(result, k, value) return result
def interpreter(state, env): global dimension_process, game_state, t, q, prev_step player1 = state[0] player2 = state[1] ### 1.1: Initialize dimensions in the background within the orchestrator if we haven't already ### if dimension_process is None: # dimension_process = Popen(["ts-node", "-P", path.abspath(path.join(dir_path, "dimensions/tsconfig.json")), path.abspath(path.join(dir_path, "dimensions/run.ts"))], stdin=PIPE, stdout=PIPE) try: dimension_process = Popen([ "node", path.abspath(path.join(dir_path, "dimensions/main.js")) ], stdin=PIPE, stdout=PIPE, stderr=PIPE) except FileNotFoundError: import warnings warnings.warn("Node not installed") return state # following 4 lines from https://stackoverflow.com/questions/375427/a-non-blocking-read-on-a-subprocess-pipe-in-python q = Queue() t = Thread(target=enqueue_output, args=(dimension_process.stdout, q)) t.daemon = True # thread dies with the program t.start() atexit.register(cleanup_dimensions) # filter out actions such as debug annotations so they aren't saved filter_actions(state, env) ### 1.2: Initialize a blank state game if new episode is starting ### if env.done: # TODO: allow resetting to a specific state # print("Initialize game", "steps", len(env.steps), "prev_step", prev_step) # last_state = None # if prev_step >= len(env.steps): # last_state = env.steps[-1] # prev_step = len(env.steps) # print("prev_step now", prev_step) if "seed" in env.configuration: seed = env.configuration["seed"] else: seed = math.floor(random.random() * 1e9) env.configuration["seed"] = seed if "loglevel" in env.configuration: loglevel = env.configuration["loglevel"] else: loglevel = 0 # warnings, 1: errors, 0: none env.configuration["loglevel"] = loglevel if "annotations" in env.configuration: annotations = env.configuration["annotations"] else: annotations = False # warnings, 1: errors, 0: none env.configuration["annotations"] = annotations if "width" in env.configuration: width = env.configuration["width"] else: width = -1 # -1 for randomly selected env.configuration["width"] = width if "height" in env.configuration: height = env.configuration["height"] else: height = -1 # -1 for randomly selected env.configuration["height"] = height initiate = { "type": "start", "agent_names": [], # unsure if this is provided? "config": env.configuration } # if last_state is not None: # initiate["state"] = last_state dimension_process.stdin.write((json.dumps(initiate) + "\n").encode()) dimension_process.stdin.flush() agent1res = get_message(dimension_process) agent2res = get_message(dimension_process) match_obs_meta = get_message(dimension_process) player1.observation.player = 0 player2.observation.player = 1 player1.observation.updates = agent1res # player2.observation.updates = agent2res # duplicated and not added player1.observation.globalCityIDCount = match_obs_meta[ "globalCityIDCount"] player1.observation.globalUnitIDCount = match_obs_meta[ "globalUnitIDCount"] player1.observation.width = match_obs_meta["width"] player1.observation.height = match_obs_meta["height"] game_state = Game() game_state._initialize(agent1res) return state # print("prev_step", prev_step, "stored steps", len(env.steps)) # prev_step += 1 ### 2. : Pass in actions (json representation along with id of who made that action), agent information (id, status) to dimensions via stdin dimension_process.stdin.write((json.dumps(state) + "\n").encode()) dimension_process.stdin.flush() ### 3.1 : Receive and parse the observations returned by dimensions via stdout agent1res = json.loads(dimension_process.stderr.readline()) agent2res = json.loads(dimension_process.stderr.readline()) game_state._update(agent1res) # receive meta info such as global ID and map sizes for purposes of being able to start from specific state match_obs_meta = json.loads(dimension_process.stderr.readline()) match_status = json.loads(dimension_process.stderr.readline()) while True: try: line = q.get_nowait() except Empty: # no standard error received, break break else: # standard error output received, print it out print(line.decode(), file=sys.stderr, end='') ### 3.2 : Send observations to each agent through here. Like dimensions, first observation can include initialization stuff, then we do the looping player1.observation.updates = agent1res player1.observation.globalCityIDCount = match_obs_meta["globalCityIDCount"] player1.observation.globalUnitIDCount = match_obs_meta["globalUnitIDCount"] player1.observation.width = match_obs_meta["width"] player1.observation.height = match_obs_meta["height"] # player2.observation.updates = agent2res # duplicated and not added player1.observation.player = 0 player2.observation.player = 1 ### 3.3 : handle rewards # reward here is defined as the sum of number of city tiles player1.reward = compute_reward(game_state.players[0]) player2.reward = compute_reward(game_state.players[1]) player1.observation.reward = int(player1.reward) player2.observation.reward = int(player2.reward) ### 3.4 Handle finished match status if match_status["status"] == "finished": if player1.status == "ACTIVE": player1.status = "DONE" if player2.status == "ACTIVE": player2.status = "DONE" return state
class MainTab(SingleLinkableSetting, QFrame): set_progressbar_signal = pyqtSignal(int) # max progress increment_progressbar_signal = pyqtSignal(int) # increment value update_label_signal = pyqtSignal(str) write_to_terminal_signal = pyqtSignal(str) add_result_signal = pyqtSignal(object) # Result add_url_analysis_result_signal = pyqtSignal(object) # URLAnalysisResult add_run_to_queue_signal = pyqtSignal(object) # Run object (or a subclass) update_run_status_signal = pyqtSignal(int, str) # run_id, status_str print_results_signal = pyqtSignal( ) # called after a run finishes to flush the results queue before printing "Done" def __init__(self): QFrame.__init__(self) SingleLinkableSetting.__init__(self, "api_key") # lazy loaded, see self#library self._library = None self.print_results_signal.connect(self.print_results) self.write_to_terminal_signal.connect(self.write) self.q = Queue() # `AnalysisResult`s get put here when we get a url scheme event, we need # to create the visualizer from the main thread so we use a queue to # kick back the work to the main thread. This is checked at the same # time `self.q` is self.url_analysis_q = Queue() # reset at the beginning of every run, used to print something after # every run only if a cheat wasn't found self.show_no_cheat_found = True self.print_results_event = threading.Event() self.cg_q = Queue() self.helper_thread_running = False self.runs = [] # Run objects for cancelling runs self.run_id = 0 self.visualizer = None terminal = QTextEdit(self) terminal.setFocusPolicy(Qt.ClickFocus) terminal.setReadOnly(True) terminal.ensureCursorVisible() self.terminal = terminal self.run_button = RunButton() self.run_button.setFixedHeight(30) self.run_button.setText("Run") font = self.run_button.font() font.setPointSize(15) self.run_button.setFont(font) self.run_button.clicked.connect(self.add_circleguard_run) # disable button if no api_key is stored self.on_setting_changed("api_key", get_setting("api_key")) investigate_label = QLabel("Investigate For:") investigate_label.setFixedWidth(130) investigate_label.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed) self.investigation_checkboxes = InvestigationCheckboxes() investigations = WidgetCombiner(investigate_label, self.investigation_checkboxes, self) investigations.setFixedHeight(25) self.loadable_creation = LoadableCreation() self.investigation_checkboxes.similarity_cb.checkbox.stateChanged.connect( self.loadable_creation.similarity_cb_state_changed) layout = QGridLayout() layout.addWidget(investigations, 0, 0, 1, 16) layout.addWidget(self.loadable_creation, 1, 0, 4, 16) layout.addWidget(self.terminal, 5, 0, 2, 16) layout.addWidget(self.run_button, 7, 0, 1, 16) self.setLayout(layout) def on_setting_changed(self, setting, text): # TODO `setting_value` should be updated automatically by # `LinkableSetting` self.setting_value = text self.run_button.setEnabled(text != "") def write(self, message): self.terminal.append(str(message).strip()) self.scroll_to_bottom() def scroll_to_bottom(self): cursor = QTextCursor(self.terminal.document()) cursor.movePosition(QTextCursor.End) self.terminal.setTextCursor(cursor) @property def library(self): if not self._library: from slider import Library self._library = Library(get_setting("cache_dir")) return self._library def add_circleguard_run(self): # osu!api v2 uses a client secret which is still 40 chars long but # includes characters after f, unlike v1's key which is in hex. if re.search("[g-z]", self.setting_value): message_box = QMessageBox() message_box.setTextFormat(Qt.RichText) message_box.setText( "Your api key is invalid. You are likely using " "an api v2 key.\n" "Please ensure your api key comes from " "<a href=\"https://osu.ppy.sh/p/api\">https://osu.ppy.sh/p/api</a>." ) message_box.exec() return if not self.loadable_creation.check_and_mark_required_fields(): return # loadables can have their required fields filled, but with invalid # input, like having "1a" as a span. In this case we print the error # and early return. try: loadables = self.loadable_creation.cg_loadables() except ValueError as e: self.write_to_terminal_signal.emit( f"<div style='color:#ff5252'>Invalid arguments:</div> {str(e)}" ) return enabled_investigations = self.investigation_checkboxes.enabled_investigations( ) # if no loadables have been filled out, don't proceed if not loadables: return # similarly for investigations, but give a heads up since this mistake # is slightly subtler if not enabled_investigations: # nbsp is necessary because apparently ending with a closing tag # makes qt not recognize it and causes all text afterwards to be # affected by the div's color self.write_to_terminal_signal.emit( "<div style='color:#ff5252'>You must select " "at least one investigation before running</div> ") return run = Run(loadables, enabled_investigations, self.run_id, threading.Event()) self.runs.append(run) self.add_run_to_queue_signal.emit(run) self.cg_q.put(run) self.run_id += 1 # called every 1/4 seconds by timer, but force a recheck so we don't # wait for that delay self.check_circleguard_queue() def check_circleguard_queue(self): def _check_circleguard_queue(self): try: while True: run = self.cg_q.get_nowait() # occurs if run is canceled before being started, it will # still stop before actually loading anything but we don't # want the labels to flicker if run.event.wait(0): continue thread = threading.Thread(target=self.run_circleguard, args=[run]) self.helper_thread_running = True thread.start() # run sequentially to not confuse user with terminal output thread.join() except Empty: self.helper_thread_running = False return # don't launch another thread running cg if one is already running, # or else multiple runs will occur at once (defeats the whole purpose # of sequential runs) if not self.helper_thread_running: # have to do a double thread use if we start the threads in # the main thread and .join, it will block the gui thread (very bad). thread = threading.Thread(target=_check_circleguard_queue, args=[self]) thread.start() def run_circleguard(self, run): from circleguard import (Circleguard, UnknownAPIException, ReplayPath, NoInfoAvailableException, Loader, LoadableContainer, replay_pairs) class TrackerLoader(Loader, QObject): """ A circleguard.Loader subclass that emits a signal when the loader is ratelimited. It inherits from QObject to allow us to use qt signals. """ ratelimit_signal = pyqtSignal( int) # length of the ratelimit in seconds check_stopped_signal = pyqtSignal() # how often to emit check_stopped_signal when ratelimited, in seconds INTERVAL = 0.250 def __init__(self, key, cacher=None): Loader.__init__(self, key, cacher) QObject.__init__(self) def _ratelimit(self, length): # sometimes the ratelimit length can get very close to zero or # even negative due to network request time and rounding. As # displaying a zero or negative wait time is confusing, don't # wait for any less than 2 seconds. length = max(length, 2) self.ratelimit_signal.emit(length) # how many times to wait for 1/4 second (rng standing for range) # we do this loop in order to tell run_circleguard to check if # the run was canceled, or the application quit, instead of # hanging on a long time.sleep rng = math.ceil(length / self.INTERVAL) for _ in range(rng): time.sleep(self.INTERVAL) self.check_stopped_signal.emit() self.update_label_signal.emit("Loading Replays") self.update_run_status_signal.emit(run.run_id, "Loading Replays") # reset every run self.show_no_cheat_found = True event = run.event try: core_cache = get_setting("cache_dir") + "circleguard.db" slider_cache = get_setting("cache_dir") should_cache = get_setting("caching") cg = Circleguard(get_setting("api_key"), core_cache, slider_dir=slider_cache, cache=should_cache, loader=TrackerLoader) def _ratelimited(length): message = get_setting("message_ratelimited") ts = datetime.now() self.write_to_terminal_signal.emit( message.format(s=length, ts=ts)) self.update_label_signal.emit("Ratelimited") self.update_run_status_signal.emit(run.run_id, "Ratelimited") def _check_event(event): """ Checks the given event to see if it is set. If it is, the run has been canceled through the queue tab or by the application being quit, and this thread exits through sys.exit(0). If the event is not set, returns silently. """ if event.wait(0): self.update_label_signal.emit("Canceled") self.set_progressbar_signal.emit(-1) # the loadables of a run may be extremely large. We keep # this run object around in a list so people can cancel it; # if we want to prevent a memory leak we need to remove a # run's loadables (we don't need it anymore after this run # finishes). run.loadables = None # may seem dirty, but actually relatively clean since it only affects this thread. # Any cleanup we may want to do later can occur here as well sys.exit(0) cg.loader.ratelimit_signal.connect(_ratelimited) cg.loader.check_stopped_signal.connect(partial( _check_event, event)) if "Similarity" in run.enabled_investigations: loadables1 = [] loadables2 = [] for loadable in run.loadables: if loadable.sim_group == 1: loadables1.append(loadable) else: loadables2.append(loadable) lc1 = LoadableContainer(loadables1) lc2 = LoadableContainer(loadables2) else: lc = LoadableContainer(run.loadables) message_loading_info = get_setting("message_loading_info").format( ts=datetime.now()) self.write_to_terminal_signal.emit(message_loading_info) if "Similarity" in run.enabled_investigations: cg.load_info(lc1) cg.load_info(lc2) replays1 = lc1.all_replays() replays2 = lc2.all_replays() all_replays = replays1 + replays2 else: cg.load_info(lc) all_replays = lc.all_replays() replays1 = all_replays replays2 = [] num_replays = len(all_replays) self.set_progressbar_signal.emit(num_replays) message_loading_replays = get_setting( "message_loading_replays").format(ts=datetime.now(), num_replays=num_replays) self.write_to_terminal_signal.emit(message_loading_replays) def _skip_replay_with_message(replay, message): self.write_to_terminal_signal.emit(message) # the replay very likely (perhaps certainly) didn't get # loaded if the above exception fired. just remove it. # If two different loadables both contain this problematic # replay, we will attempt to remove it from the list twice. # Guard against this by checking for membership first. if replay in all_replays: all_replays.remove(replay) # check has already been initialized with the replay, # remove it here too or cg will try and load it again # when the check is ran if replay in replays1: replays1.remove(replay) if replay in replays2: replays2.remove(replay) # `[:]` implicitly copies the list, so we don't run into trouble # when removing elements from it while iterating for replay in all_replays[:]: _check_event(event) try: cg.load(replay) # circleparse sets replay_data to None if it's not a std # replay, which means replay.has_data() will be false, so # we need to do this check first to give a better error # message than "the replay is not available for download", # which is incorrect for local replays. if isinstance(replay, ReplayPath) and not replay.has_data(): _skip_replay_with_message( replay, "<div style='color:#ff5252'>The replay " + str(replay) + " is " + "not an osu!std replay.</div> We currently only support std replays. " "This replay has been skipped because of this.") elif not replay.has_data(): _skip_replay_with_message( replay, "<div style='color:#ff5252'>The replay " + str(replay) + " is " + "not available for download.</div> This is likely because it is not in the top 1k scores of " "the beatmap. This replay has been skipped because of this." ) except NoInfoAvailableException as e: _skip_replay_with_message( replay, "<div style='color:#ff5252'>The replay " + str(replay) + " does " "not exist.</div>\nDouble check your map and/or user id. This replay has " "been skipped because of this.") except UnknownAPIException as e: _skip_replay_with_message( replay, "<div style='color:#ff5252'>The osu! api provided an invalid " "response:</div> " + str(e) + ". The replay " + str(replay) + " has been skipped because of this.") except LZMAError as e: _skip_replay_with_message( replay, "<div style='color:#ff5252'>lzma error while parsing a replay:</div> " + str(e) + ". The replay is either corrupted or has no replay data. The replay " + str(replay) + " has been skipped because of this.") except Exception as e: # print full traceback here for more debugging info. # Don't do it for the previous exceptions because the # cause of those is well understood, but that's not # necessarily the case for generic exceptions here. # attempting to use divs/spans here to color the beginning # of this string made the traceback collapse down into # one line with none of the usual linebreaks, I'm not # sure why. So you win qt, no red color for this error. _skip_replay_with_message( replay, "error while loading a replay: " + str(e) + "\n" + traceback.format_exc() + "The replay " + str(replay) + " has been skipped because of this.") finally: self.increment_progressbar_signal.emit(1) if "Similarity" in run.enabled_investigations: lc1.loaded = True lc2.loaded = True else: lc.loaded = True # change progressbar into an undetermined state (animation with # stripes sliding horizontally) to indicate we're processing # the data self.set_progressbar_signal.emit(0) setting_end_dict = { "Similarity": "steal", "Unstable Rate": "relax", "Snaps": "correction", "Frametime": "timewarp", "Manual Analysis": "analysis" } for investigation in run.enabled_investigations: setting = "message_starting_" + setting_end_dict[investigation] message_starting_investigation = get_setting(setting).format( ts=datetime.now()) self.write_to_terminal_signal.emit( message_starting_investigation) if investigation == "Manual Analysis": map_ids = [r.map_id for r in all_replays] if len(set(map_ids)) > 1: self.write_to_terminal_signal.emit( "Manual analysis expected replays from a single map, " f"but got replays from maps {set(map_ids)}. Please use a different Manual Analysis " "Check for each map.") self.update_label_signal.emit( "Analysis Error (Multiple maps)") self.update_run_status_signal.emit( run.run_id, "Analysis Error (Multiple maps)") self.set_progressbar_signal.emit(-1) run.loadables = None sys.exit(0) # if a replay was removed from all_replays (eg if that replay was not available for download), # and that leaves all_replays with no replays, we don't want to add a result because # the rest of guard expects >=1 replay, leading to confusing errors. if len(all_replays) != 0: self.q.put(AnalysisResult(all_replays)) continue self.update_label_signal.emit("Investigating Replays...") self.update_run_status_signal.emit(run.run_id, "Investigating Replays") if investigation == "Similarity": # core relies on the second replays list being the one empty # if only one of the lists are empty, but we want to allow # users to select whichever group they want if they're only # using a single group, so switch the two if this is the # case. if replays2 and not replays1: replays1, replays2 = replays2, replays1 pairs = replay_pairs(replays1, replays2) for (replay1, replay2) in pairs: _check_event(event) sim = cg.similarity(replay1, replay2) result = StealResult(sim, replay1, replay2) self.q.put(result) if investigation == "Unstable Rate": for replay in all_replays: _check_event(event) # skip replays which have no map info if replay.map_info.available(): try: ur = cg.ur(replay) # Sometimes, a beatmap will have a bugged download where it returns an empty response # when we try to download it (https://github.com/ppy/osu-api/issues/171). Since peppy # doesn't plan on fixing this for unranked beatmaps, we just ignore / skip the error # in all cases. # StopIteration is a bit of a weird exception to catch here, but because of how slider # interacts with beatmaps it will attempt to call `next` on an empty generator if the # beatmap is empty, which of course raises StopIteration. except StopIteration: import requests if requests.get( f"https://osu.ppy.sh/osu/{replay.map_id}" ).content == b"": self.write_to_terminal_signal.emit( "<div style='color:#ff5252'>The " "map " + str(replay.map_id) + "'s download is bugged</div>, so its ur cannot " "be calculated. The replay " + str(replay) + " has been skipped because of this, " "but please report this to the developers through discord or github so it can be " "tracked.") break # If we happen to catch an unrelated error with this `except`, we still want to # raise that so it can be tracked and fixed. else: raise result = RelaxResult(ur, replay) self.q.put(result) else: self.write_to_terminal_signal.emit( "<div style='color:#ff5252'>The " "replay " + str(replay) + " has no map id</div>, so its ur cannot " "be calculated. This replay has been skipped because of this." ) if investigation == "Snaps": max_angle = get_setting("correction_max_angle") min_distance = get_setting("correction_min_distance") only_on_hitobjs = get_setting("ignore_snaps_off_hitobjs") for replay in all_replays: _check_event(event) snaps = cg.snaps(replay, max_angle, min_distance, only_on_hitobjs) result = CorrectionResult(snaps, replay) self.q.put(result) if investigation == "Frametime": for replay in all_replays: _check_event(event) frametime = cg.frametime(replay) frametimes = cg.frametimes(replay) result = TimewarpResult(frametime, frametimes, replay) self.q.put(result) # flush self.q. Since the next investigation will be processed # so quickly afterwards, we actually need to forcibly wait # until the results have been printed before proceeding. self.print_results_event.clear() self.print_results_signal.emit() self.print_results_event.wait() self.set_progressbar_signal.emit(-1) # empty progressbar # this event is necessary because `print_results` will set # `show_no_cheat_found`, and since it happens asynchronously we need # to wait for it to finish before checking it. So we clear it here, # then wait for it to get set before proceeding. self.print_results_event.clear() # 'flush' self.q so there's no more results left and message_finished_investigation # won't print before results from that investigation which looks strange. self.print_results_signal.emit() self.print_results_event.wait() if self.show_no_cheat_found: self.write_to_terminal_signal.emit( get_setting("message_no_cheat_found").format( ts=datetime.now())) self.write_to_terminal_signal.emit( get_setting("message_finished_investigation").format( ts=datetime.now())) # prevents an error when a user closes the application. Because # we're running inside a new thread, if we don't do this, cg (and) # the library) will get gc'd in another thread. Because library's # ``__del__`` closes the sqlite connection, this causes: # ``` # Traceback (most recent call last): # File "/Users/tybug/Desktop/coding/osu/slider/slider/library.py", line 98, in __del__ # self.close() # File "/Users/tybug/Desktop/coding/osu/slider/slider/library.py", line 94, in close # self._db.close() # sqlite3.ProgrammingError: SQLite objects created in a thread can only be used in that same thread. # The object was created in thread id 123145483210752 and this is thread id 4479481280. # ``` cg.library.close() except Exception: # if the error happens before we set the progressbar it stays at # 100%. make sure we reset it here self.set_progressbar_signal.emit(-1) log.exception( "Error while running circlecore. Please " "report this to the developers through discord or github.\n") self.update_label_signal.emit("Idle") self.update_run_status_signal.emit(run.run_id, "Finished") # necessary to prevent memory leaks, as we keep the run object around # after the run finishes, but don't need its loadables anymore. run.loadables = None def print_results(self): try: while True: result = self.q.get_nowait() whitelisted_ids = [] whitelist_file = get_setting("whitelist_file_location") # whitelist_file is the empty strign if no file has been specified if whitelist_file: whitelist_file = Path(whitelist_file) with open(whitelist_file) as f: lines = f.readlines() for line in lines: # remove comment if there is one line = line.rsplit("#")[0] # allow whitespace on either end of the id line = line.strip() # allow empty lines anywhere in the file if line == "": continue # TODO: nicer error if the line is an invalid id? try: line = int(line) except ValueError: self.write_to_terminal_signal.emit( "<div style='color:#ff5252'>Invalid " f"user id in whitelist file:</div> \"{line}\" is not a valid user id." ) continue whitelisted_ids.append(line) ts = datetime.now() # ts = timestamp message = None ischeat = False if isinstance(result, StealResult): if result.earlier_replay.user_id in whitelisted_ids: pass elif result.similarity < get_setting("steal_max_sim"): ischeat = True message = get_setting("message_steal_found").format( ts=ts, sim=result.similarity, r=result, replay1=result.replay1, replay2=result.replay2, earlier_replay_mods_short_name=result. earlier_replay.mods.short_name(), earlier_replay_mods_long_name=result. earlier_replay.mods.long_name(), later_replay_mods_short_name=result.later_replay. mods.short_name(), later_replay_mods_long_name=result.later_replay. mods.long_name()) elif result.similarity < get_setting( "steal_max_sim_display"): message = get_setting( "message_steal_found_display").format( ts=ts, sim=result.similarity, r=result, replay1=result.replay1, earlier_replay_mods_short_name=result. earlier_replay.mods.short_name(), earlier_replay_mods_long_name=result. earlier_replay.mods.long_name(), later_replay_mods_short_name=result. later_replay.mods.short_name(), later_replay_mods_long_name=result. later_replay.mods.long_name()) if isinstance(result, RelaxResult): if result.replay.user_id in whitelisted_ids: pass elif result.ur < get_setting("relax_max_ur"): ischeat = True message = get_setting("message_relax_found").format( ts=ts, r=result, replay=result.replay, ur=result.ur, mods_short_name=result.replay.mods.short_name(), mods_long_name=result.replay.mods.long_name()) elif result.ur < get_setting("relax_max_ur_display"): message = get_setting( "message_relax_found_display").format( ts=ts, r=result, replay=result.replay, ur=result.ur, mods_short_name=result.replay.mods.short_name( ), mods_long_name=result.replay.mods.long_name()) if isinstance(result, CorrectionResult): if result.replay.user_id in whitelisted_ids: pass elif len(result.snaps) >= get_setting( "correction_min_snaps"): ischeat = True snap_message = get_setting("message_correction_snaps") snap_text = "\n".join([ snap_message.format(time=snap.time, angle=snap.angle, distance=snap.distance) for snap in result.snaps ]) message = get_setting( "message_correction_found").format( ts=ts, r=result, replay=result.replay, snaps=snap_text, mods_short_name=result.replay.mods.short_name( ), mods_long_name=result.replay.mods.long_name()) elif len(result.snaps) >= get_setting( "correction_min_snaps_display"): snap_message = get_setting("message_correction_snaps") snap_text = "\n".join([ snap_message.format(time=snap.time, angle=snap.angle, distance=snap.distance) for snap in result.snaps ]) message = get_setting( "message_correction_found_display").format( ts=ts, r=result, replay=result.replay, snaps=snap_text, mods_short_name=result.replay.mods.short_name( ), mods_long_name=result.replay.mods.long_name()) if isinstance(result, TimewarpResult): if result.replay.user_id in whitelisted_ids: pass elif result.frametime < get_setting( "timewarp_max_frametime"): ischeat = True message = get_setting("message_timewarp_found").format( ts=ts, r=result, replay=result.replay, frametime=result.frametime, mods_short_name=result.replay.mods.short_name(), mods_long_name=result.replay.mods.long_name()) elif result.frametime < get_setting( "timewarp_max_frametime_display"): message = get_setting( "message_timewarp_found_display").format( ts=ts, r=result, replay=result.replay, frametime=result.frametime, mods_short_name=result.replay.mods.short_name( ), mods_long_name=result.replay.mods.long_name()) if message or isinstance(result, AnalysisResult): self.show_no_cheat_found = False if message: self.write(message) if isinstance(result, AnalysisResult): self.add_result_signal.emit(result) else: if ischeat: if get_setting("alert_on_result"): QApplication.beep() QApplication.alert(self) # add to Results Tab so it can be played back on demand self.add_result_signal.emit(result) except Empty: try: result = self.url_analysis_q.get_nowait() self.add_url_analysis_result_signal.emit(result) except Empty: pass self.print_results_event.set() def visualize(self, replays, beatmap_id, result, start_at=0): from circlevis import BeatmapInfo # only run one instance at a time if self.visualizer is not None: self.visualizer.close() snaps = [] if isinstance(result, CorrectionResult): snaps = [snap.time for snap in result.snaps] beatmap_info = BeatmapInfo(map_id=beatmap_id) CGVisualizer = get_visualizer() self.visualizer = CGVisualizer(beatmap_info, replays, snaps, self.library) self.visualizer.show() if start_at != 0: self.visualizer.seek_to(start_at) self.visualizer.pause() def visualize_from_url(self, result): """ called when our url scheme (circleguard://) was entered, giving us a replay to visualize """ map_id = result.replays[0].map_id if self.visualizer and self.visualizer.isVisible( ) and self.visualizer.replays and self.visualizer.replays[ 0].map_id == map_id: # pause even if we're currently playing - it's important that this # happens before seeking, or else the new frame won't correctly # display self.visualizer.force_pause() # if we're visualizing the same replay that's in the url, just jump # to the new timestamp self.visualizer.seek_to(result.timestamp) return # otherwise visualize as normal (which will close any existing # visualizers) self.visualize(result.replays, map_id, result, start_at=result.timestamp)
class Palette2: def __init__(self, config): self.printer = config.get_printer() self.reactor = self.printer.get_reactor() try: self.virtual_sdcard = self.printer.load_object( config, "virtual_sdcard") except config.error: raise self.printer.config_error( "Palette 2 requires [virtual_sdcard] to work," " please add it to your config!") try: self.pause_resume = self.printer.load_object( config, "pause_resume") except config.error: raise self.printer.config_error( "Palette 2 requires [pause_resume] to work," " please add it to your config!") self.gcode_move = self.printer.load_object(config, 'gcode_move') self.gcode = self.printer.lookup_object("gcode") self.gcode.register_command("PALETTE_CONNECT", self.cmd_Connect, desc=self.cmd_Connect_Help) self.gcode.register_command("PALETTE_DISCONNECT", self.cmd_Disconnect, desc=self.cmd_Disconnect_Help) self.gcode.register_command("PALETTE_CLEAR", self.cmd_Clear, desc=self.cmd_Clear_Help) self.gcode.register_command("PALETTE_CUT", self.cmd_Cut, desc=self.cmd_Cut_Help) self.gcode.register_command("PALETTE_SMART_LOAD", self.cmd_Smart_Load, desc=self.cmd_Smart_Load_Help) self.serial = None self.serial_port = config.get("serial") if not self.serial_port: raise config.error("Invalid serial port specific for Palette 2") self.baud = config.getint("baud", default=115200) self.feedrate_splice = config.getfloat("feedrate_splice", default=0.8, minval=0., maxval=1.) self.feedrate_normal = config.getfloat("feedrate_normal", default=1.0, minval=0., maxval=1.) self.auto_load_speed = config.getint("auto_load_speed", 2) self.auto_cancel_variation = config.getfloat("auto_cancel_variation", default=None, minval=0.01, maxval=0.2) # Omega code matchers self.omega_header = [None] * 9 omega_handlers = ["O" + str(i) for i in range(33)] for cmd in omega_handlers: func = getattr(self, 'cmd_' + cmd, None) desc = getattr(self, 'cmd_' + cmd + '_help', None) if func: self.gcode.register_command(cmd, func, desc=desc) else: self.gcode.register_command(cmd, self.cmd_OmegaDefault) self._reset() self.read_timer = None self.read_buffer = "" self.read_queue = Queue() self.write_timer = None self.write_queue = Queue() self.heartbeat_timer = None self.heartbeat = None self.signal_disconnect = False self.is_printing = False self.smart_load_timer = None def _reset(self): self.files = [] self.is_setup_complete = False self.is_splicing = False self.is_loading = False self.remaining_load_length = None self.omega_algorithms = [] self.omega_algorithms_counter = 0 self.omega_splices = [] self.omega_splices_counter = 0 self.omega_pings = [] self.omega_pongs = [] self.omega_current_ping = None self.omega_header = [None] * 9 self.omega_header_counter = 0 self.omega_last_command = "" self.omega_drivers = [] def _check_P2(self, gcmd=None): if self.serial: return True if gcmd: gcmd.respond_info(INFO_NOT_CONNECTED) return False cmd_Connect_Help = ("Connect to the Palette 2") def cmd_Connect(self, gcmd): if self.serial: gcmd.respond_info( "Palette 2 serial port is already active, disconnect first") return self.signal_disconnect = False logging.info("Connecting to Palette 2 on port (%s) at (%s)" % (self.serial_port, self.baud)) try: self.serial = serial.Serial(self.serial_port, self.baud, timeout=0, write_timeout=0) except SerialException: gcmd.respond_info("Unable to connect to the Palette 2") return with self.write_queue.mutex: self.write_queue.queue.clear() with self.read_queue.mutex: self.read_queue.queue.clear() self.read_timer = self.reactor.register_timer(self._run_Read, self.reactor.NOW) self.write_timer = self.reactor.register_timer(self._run_Write, self.reactor.NOW) self.heartbeat_timer = self.reactor.register_timer( self._run_Heartbeat, self.reactor.NOW) # Tell the device we're alive self.write_queue.put("\n") self.write_queue.put(COMMAND_FIRMWARE) self._wait_for_heartbeat() cmd_Disconnect_Help = ("Disconnect from the Palette 2") def cmd_Disconnect(self, gmcd=None): self.gcode.respond_info("Disconnecting from Palette 2") if self.serial: self.serial.close() self.serial = None self.reactor.unregister_timer(self.read_timer) self.reactor.unregister_timer(self.write_timer) self.reactor.unregister_timer(self.heartbeat_timer) self.read_timer = None self.write_timer = None self.heartbeat = None self.is_printing = False cmd_Clear_Help = ("Clear the input and output of the Palette 2") def cmd_Clear(self, gcmd): logging.info("Clearing Palette 2 input and output") if self._check_P2(gcmd): for l in COMMAND_CLEAR: self.write_queue.put(l) cmd_Cut_Help = ("Cut the outgoing filament") def cmd_Cut(self, gcmd): logging.info("Cutting outgoing filament in Palette 2") if self._check_P2(gcmd): self.write_queue.put(COMMAND_CUT) cmd_Smart_Load_Help = ("Automatically load filament through the extruder") def cmd_Smart_Load(self, gcmd): if self._check_P2(gcmd): if not self.is_loading: gcmd.respond_info( "Cannot auto load when the Palette 2 is not ready") return self.p2cmd_O102(params=None) def cmd_OmegaDefault(self, gcmd): logging.debug("Omega Code: %s" % (gcmd.get_command())) if self._check_P2(gcmd): self.write_queue.put(gcmd.get_commandline()) def _wait_for_heartbeat(self): startTs = self.reactor.monotonic() currTs = startTs while self.heartbeat is None and self.heartbeat < ( currTs - SETUP_TIMEOUT) and startTs > (currTs - SETUP_TIMEOUT): currTs = self.reactor.pause(currTs + 1.) if self.heartbeat < (currTs - SETUP_TIMEOUT): self.signal_disconnect = True raise self.printer.command_error("No response from Palette 2") cmd_O1_help = ( "Initialize the print, and check connection with the Palette 2") def cmd_O1(self, gcmd): logging.info("Initializing print with Pallete 2") if not self._check_P2(gcmd): raise self.printer.command_error( "Cannot initialize print, palette 2 is not connected") self.reactor.update_timer(self.heartbeat_timer, self.reactor.NOW) self._wait_for_heartbeat() self.write_queue.put(gcmd.get_commandline()) self.gcode.respond_info("Palette 2 waiting on user to complete setup") self.pause_resume.send_pause_command() cmd_O9_help = ("Reset print information") def cmd_O9(self, gcmd): logging.info("Print finished, resetting Palette 2 state") if self._check_P2(gcmd): self.write_queue.put(gcmd.get_commandline()) self.is_printing = False def cmd_O21(self, gcmd): logging.debug("Omega version: %s" % (gcmd.get_commandline())) self._reset() self.omega_header[0] = gcmd.get_commandline() self.is_printing = True def cmd_O22(self, gcmd): logging.debug("Omega printer profile: %s" % (gcmd.get_commandline())) self.omega_header[1] = gcmd.get_commandline() def cmd_O23(self, gcmd): logging.debug("Omega slicer profile: %s" % (gcmd.get_commandline())) self.omega_header[2] = gcmd.get_commandline() def cmd_O24(self, gcmd): logging.debug("Omega PPM: %s" % (gcmd.get_commandline())) self.omega_header[3] = gcmd.get_commandline() def cmd_O25(self, gcmd): logging.debug("Omega inputs: %s" % (gcmd.get_commandline())) self.omega_header[4] = gcmd.get_commandline() drives = self.omega_header[4][4:].split() for idx in range(len(drives)): state = drives[idx][:2] if state == "D1": drives[idx] = "U" + str(60 + idx) self.omega_drives = [d for d in drives if d != "D0"] logging.info("Omega drives: %s" % self.omega_drives) def cmd_O26(self, gcmd): logging.debug("Omega splices %s" % (gcmd.get_commandline())) self.omega_header[5] = gcmd.get_commandline() def cmd_O27(self, gcmd): logging.debug("Omega pings: %s" % (gcmd.get_commandline())) self.omega_header[6] = gcmd.get_commandline() def cmd_O28(self, gcmd): logging.debug("Omega MSF NA: %s" % (gcmd.get_commandline())) self.omega_header[7] = gcmd.get_commandline() def cmd_O29(self, gcmd): logging.debug("Omega MSF NH: %s" % (gcmd.get_commandline())) self.omega_header[8] = gcmd.get_commandline() def cmd_O30(self, gcmd): try: param_drive = gcmd.get_commandline()[5:6] param_distance = gcmd.get_commandline()[8:] except IndexError: gmcd.respond_info( "Incorrect number of arguments for splice command") try: self.omega_splices.append((int(param_drive), param_distance)) except ValueError: gcmd.respond_info("Incorrectly formatted splice command") logging.debug("Omega splice command drive %s distance %s" % (param_drive, param_distance)) def cmd_O31(self, gcmd): if self._check_P2(gcmd): self.omega_current_ping = gcmd.get_commandline() logging.debug("Omega ping command: %s" % (gcmd.get_commandline())) self.write_queue.put(COMMAND_PING) self.gcode.create_gcode_command("G4", "G4", {"P": "10"}) def cmd_O32(self, gcmd): logging.debug("Omega algorithm: %s" % (gcmd.get_commandline())) self.omega_algorithms.append(gcmd.get_commandline()) def p2cmd_O20(self, params): if not self.is_printing: return # First print, we can ignore if params[0] == "D5": logging.info("First print on Palette") return try: n = int(params[0][1:]) except (TypeError, IndexError): logging.error("O20 command has invalid parameters") return if n == 0: logging.info("Sending omega header %s" % self.omega_header_counter) self.write_queue.put(self.omega_header[self.omega_header_counter]) self.omega_header_counter = self.omega_header_counter + 1 elif n == 1: logging.info("Sending splice info %s" % self.omega_splices_counter) splice = self.omega_splices[self.omega_splices_counter] self.write_queue.put("O30 D%d D%s" % (splice[0], splice[1])) self.omega_splices_counter = self.omega_splices_counter + 1 elif n == 2: logging.info("Sending current ping info %s" % self.omega_current_ping) self.write_queue.put(self.omega_current_ping) elif n == 4: logging.info("Sending algorithm info %s" % self.omega_algorithms_counter) self.write_queue.put( self.omega_algorithms[self.omega_algorithms_counter]) self.omega_algorithms_counter = self.omega_algorithms_counter + 1 elif n == 8: logging.info("Resending the last command to Palette 2") self.write_queue.put(self.omega_last_command) def p2cmd_O34(self, params): if not self.is_printing: return def check_ping_variation(last_ping): if self.auto_cancel_variation is not None: ping_max = 100. + (self.auto_cancel_variation * 100.) ping_min = 100. - (self.auto_cancel_variation * 100.) if last_ping < ping_min or last_ping > ping_max: logging.info("Ping variation is too high, " "cancelling print") self.gcode.run_script("CANCEL_PRINT") if len(params) > 2: percent = float(params[1][1:]) if params[0] == "D1": number = len(self.omega_pings) + 1 d = {"number": number, "percent": percent} logging.info("Ping %d, %d percent" % (number, percent)) self.omega_pings.append(d) check_ping_variation(percent) elif params[0] == "D2": number = len(self.omega_pongs) + 1 d = {"number": number, "percent": percent} logging.info("Pong %d, %d percent" % (number, percent)) self.omega_pongs.append(d) def p2cmd_O40(self, params): logging.info("Resume request from Palette 2") self.pause_resume.send_resume_command() def p2cmd_O50(self, params): if len(params) > 1: try: fw = params[0][1:] logging.info("Palette 2 firmware version %s detected" % os.fwalk) except (TypeError, IndexError): logging.error("Unable to parse firmware version") if fw < "9.0.9": raise self.printer.command_error( "Palette 2 firmware version is too old, " "update to at least 9.0.9") else: self.files = [ file for (file, size) in self.virtual_sdcard.get_file_list( check_subdirs=True) if ".mcf.gcode" in file ] for file in self.files: self.write_queue.put("%s D%s" % (COMMAND_FILENAME, file)) self.write_queue.put(COMMAND_FILENAMES_DONE) def p2cmd_O53(self, params): if len(params) > 1 and params[0] == "D1": try: idx = int(params[1][1:], 16) file = self.files[::-1][idx] self.gcode.run_script("SDCARD_PRINT_FILE FILENAME=%s" % file) except (TypeError, IndexError): logging.error("O53 has invalid command parameters") def p2cmd_O88(self, params): logging.error("Palette 2 error detected") try: error = int(params[0][1:], 16) logging.error("Palette 2 error code %d" % error) except (TypeError, IndexError): logging.error("Unable to parse Palette 2 error") def p2cmd_O97(self, params): def printCancelling(params): logging.info("Print Cancelling") self.gcode.run_script("CLEAR_PAUSE") self.gcode.run_script("CANCEL_PRINT") def printCancelled(params): logging.info("Print Cancelled") self._reset() def loadingOffsetStart(params): logging.info("Waiting for user to load filament into printer") self.is_loading = True def loadingOffset(params): self.remaining_load_length = int(params[1][1:]) logging.debug("Loading filamant remaining %d" % self.remaining_load_length) if self.remaining_load_length >= 0 and self.smart_load_timer: logging.info("Smart load filament is complete") self.reactor.unregister_timer(self.smart_load_timer) self.smart_load_timer = None self.is_loading = False def feedrateStart(params): logging.info("Setting feedrate to %f for splice" % self.feedrate_splice) self.is_splicing = True self.gcode.run_script("M220 S%d" % (self.feedrate_splice * 100)) def feedrateEnd(params): logging.info("Setting feedrate to %f splice done" % self.feedrate_normal) self.is_splicing = False self.gcode.run_script("M220 S%d" % (self.feedrate_normal * 100)) matchers = [] if self.is_printing: matchers = matchers + [ [printCancelling, 2, "U0", "D2"], [printCancelled, 2, "U0", "D3"], [loadingOffset, 2, "U39"], [loadingOffsetStart, 1, "U39"], ] matchers.append([feedrateStart, 3, "U25", "D0"]) matchers.append([feedrateEnd, 3, "U25", "D1"]) self._param_Matcher(matchers, params) def p2cmd_O100(self, params): logging.info("Pause request from Palette 2") self.is_setup_complete = True self.pause_resume.send_pause_command() def p2cmd_O102(self, params): toolhead = self.printer.lookup_object("toolhead") if not toolhead.get_extruder().get_heater().can_extrude: self.write_queue.put(COMMAND_SMART_LOAD_STOP) self.gcode.respond_info( "Unable to auto load filament, extruder is below minimum temp") return if self.smart_load_timer is None: logging.info("Smart load starting") self.smart_load_timer = self.reactor.register_timer( self._run_Smart_Load, self.reactor.NOW) def p2cmd(self, line): t = line.split() ocode = t[0] params = t[1:] params_count = len(params) if params_count: res = [i for i in params if i[0] == "D" or i[0] == "U"] if not all(res): logging.error("Omega parameters are invalid") return func = getattr(self, 'p2cmd_' + ocode, None) if func is not None: func(params) def _param_Matcher(self, matchers, params): # Match the command with the handling table for matcher in matchers: if len(params) >= matcher[1]: match_params = matcher[2:] res = all([ match_params[i] == params[i] for i in range(len(match_params)) ]) if res: matcher[0](params) return True return False def _run_Read(self, eventtime): if self.signal_disconnect: self.cmd_Disconnect() return self.reactor.NEVER # Do non-blocking reads from serial and try to find lines while True: try: raw_bytes = self.serial.read() except SerialException: logging.error("Unable to communicate with the Palette 2") self.cmd_Disconnect() return self.reactor.NEVER if len(raw_bytes): text_buffer = self.read_buffer + raw_bytes.decode() while True: i = text_buffer.find("\n") if i >= 0: line = text_buffer[0:i + 1] self.read_queue.put(line.strip()) text_buffer = text_buffer[i + 1:] else: break self.read_buffer = text_buffer else: break # Process any decoded lines from the device while not self.read_queue.empty(): try: text_line = self.read_queue.get_nowait() except Empty: pass heartbeat_strings = [COMMAND_HEARTBEAT, "Connection Okay"] if not any(x in text_line for x in heartbeat_strings): logging.debug("%0.3f P2 -> : %s" % (eventtime, text_line)) # Received a heartbeat from the device if text_line == COMMAND_HEARTBEAT: self.heartbeat = eventtime elif text_line[0] == "O": self.p2cmd(text_line) return eventtime + SERIAL_TIMER def _run_Heartbeat(self, eventtime): self.write_queue.put(COMMAND_HEARTBEAT) eventtime = self.reactor.pause(eventtime + 5) if self.heartbeat and self.heartbeat < (eventtime - HEARTBEAT_TIMEOUT): logging.error("P2 has not responded to heartbeat") if not self.is_printing or self.is_setup_complete: self.cmd_Disconnect() return self.reactor.NEVER return eventtime + HEARTBEAT_SEND def _run_Write(self, eventtime): while not self.write_queue.empty(): try: text_line = self.write_queue.get_nowait() except Empty: continue if text_line: self.omega_last_command = text_line l = text_line.strip() if COMMAND_HEARTBEAT not in l: logging.debug("%s -> P2 : %s" % (self.reactor.monotonic(), l)) terminated_line = "%s\n" % (l) try: self.serial.write(terminated_line.encode()) except SerialException: logging.error("Unable to communicate with the Palette 2") self.signal_disconnect = True return self.reactor.NEVER return eventtime + SERIAL_TIMER def _run_Smart_Load(self, eventtime): if not self.is_splicing and self.remaining_load_length < 0: # Make sure toolhead class isn't busy toolhead = self.printer.lookup_object("toolhead") print_time, est_print_time, lookahead_empty = toolhead.check_busy( eventtime) idle_time = est_print_time - print_time if not lookahead_empty or idle_time < 0.5: return eventtime + \ max(0., min(1., print_time - est_print_time)) extrude = abs(self.remaining_load_length) extrude = min(50, extrude / 2) if extrude <= 10: extrude = 1 logging.info("Smart loading %dmm filament with %dmm remaining" % (extrude, abs(self.remaining_load_length))) self.gcode.run_script("G92 E0") self.gcode.run_script("G1 E%d F%d" % (extrude, self.auto_load_speed * 60)) return self.reactor.NOW return eventtime + AUTOLOAD_TIMER def get_status(self, eventtime=None): status = { "ping": self.omega_pings[-1], "remaining_load_length": self.remaining_load_length, "is_splicing": self.is_splicing } return status
class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ TagBrowserMixin, CoverFlowMixin, LibraryViewMixin, SearchBoxMixin, SavedSearchBoxMixin, SearchRestrictionMixin, LayoutMixin, UpdateMixin, EbookDownloadMixin ): 'The main GUI' proceed_requested = pyqtSignal(object, object) book_converted = pyqtSignal(object, object) shutting_down = False def __init__(self, opts, parent=None, gui_debug=None): global _gui MainWindow.__init__(self, opts, parent=parent, disable_automatic_gc=True) self.setWindowIcon(QApplication.instance().windowIcon()) self.jobs_pointer = Pointer(self) self.proceed_requested.connect(self.do_proceed, type=Qt.QueuedConnection) self.proceed_question = ProceedQuestion(self) self.job_error_dialog = JobError(self) self.keyboard = Manager(self) _gui = self self.opts = opts self.device_connected = None self.gui_debug = gui_debug self.iactions = OrderedDict() # Actions for action in interface_actions(): if opts.ignore_plugins and action.plugin_path is not None: continue try: ac = self.init_iaction(action) except: # Ignore errors in loading user supplied plugins import traceback traceback.print_exc() if action.plugin_path is None: raise continue ac.plugin_path = action.plugin_path ac.interface_action_base_plugin = action self.add_iaction(ac) self.load_store_plugins() def init_iaction(self, action): ac = action.load_actual_plugin(self) ac.plugin_path = action.plugin_path ac.interface_action_base_plugin = action action.actual_iaction_plugin_loaded = True return ac def add_iaction(self, ac): acmap = self.iactions if ac.name in acmap: if ac.priority >= acmap[ac.name].priority: acmap[ac.name] = ac else: acmap[ac.name] = ac def load_store_plugins(self): from calibre.gui2.store.loader import Stores self.istores = Stores() for store in available_store_plugins(): if self.opts.ignore_plugins and store.plugin_path is not None: continue try: st = self.init_istore(store) self.add_istore(st) except: # Ignore errors in loading user supplied plugins import traceback traceback.print_exc() if store.plugin_path is None: raise continue self.istores.builtins_loaded() def init_istore(self, store): st = store.load_actual_plugin(self) st.plugin_path = store.plugin_path st.base_plugin = store store.actual_istore_plugin_loaded = True return st def add_istore(self, st): stmap = self.istores if st.name in stmap: if st.priority >= stmap[st.name].priority: stmap[st.name] = st else: stmap[st.name] = st def initialize(self, library_path, db, listener, actions, show_gui=True): opts = self.opts self.preferences_action, self.quit_action = actions self.library_path = library_path self.library_broker = GuiLibraryBroker(db) self.content_server = None self.server_change_notification_timer = t = QTimer(self) self.server_changes = Queue() t.setInterval(1000), t.timeout.connect(self.handle_changes_from_server_debounced), t.setSingleShot(True) self._spare_pool = None self.must_restart_before_config = False self.listener = Listener(listener) self.check_messages_timer = QTimer() self.check_messages_timer.timeout.connect(self.another_instance_wants_to_talk) self.check_messages_timer.start(1000) for ac in list(self.iactions.values()): try: ac.do_genesis() except Exception: # Ignore errors in third party plugins import traceback traceback.print_exc() if getattr(ac, 'plugin_path', None) is None: raise self.donate_action = QAction(QIcon(I('donate.png')), _('&Donate to support calibre'), self) for st in list(self.istores.values()): st.do_genesis() MainWindowMixin.init_main_window_mixin(self, db) # Jobs Button {{{ self.job_manager = JobManager() self.jobs_dialog = JobsDialog(self, self.job_manager) self.jobs_button = JobsButton(parent=self) self.jobs_button.initialize(self.jobs_dialog, self.job_manager) # }}} LayoutMixin.init_layout_mixin(self) DeviceMixin.init_device_mixin(self) self.progress_indicator = ProgressIndicator(self) self.progress_indicator.pos = (0, 20) self.verbose = opts.verbose self.get_metadata = GetMetadata() self.upload_memory = {} self.metadata_dialogs = [] self.default_thumbnail = None self.tb_wrapper = textwrap.TextWrapper(width=40) self.viewers = collections.deque() self.system_tray_icon = None if config['systray_icon']: self.system_tray_icon = factory(app_id='com.calibre-ebook.gui').create_system_tray_icon(parent=self, title='calibre') if self.system_tray_icon is not None: self.system_tray_icon.setIcon(QIcon(I('lt.png', allow_user_override=False))) if not (iswindows or isosx): self.system_tray_icon.setIcon(QIcon.fromTheme('calibre-gui', self.system_tray_icon.icon())) self.system_tray_icon.setToolTip(self.jobs_button.tray_tooltip()) self.system_tray_icon.setVisible(True) self.jobs_button.tray_tooltip_updated.connect(self.system_tray_icon.setToolTip) elif config['systray_icon']: prints('Failed to create system tray icon, your desktop environment probably does not support the StatusNotifier spec') self.system_tray_menu = QMenu(self) self.toggle_to_tray_action = self.system_tray_menu.addAction(QIcon(I('page.png')), '') self.toggle_to_tray_action.triggered.connect(self.system_tray_icon_activated) self.system_tray_menu.addAction(self.donate_action) self.eject_action = self.system_tray_menu.addAction( QIcon(I('eject.png')), _('&Eject connected device')) self.eject_action.setEnabled(False) self.addAction(self.quit_action) self.system_tray_menu.addAction(self.quit_action) self.keyboard.register_shortcut('quit calibre', _('Quit calibre'), default_keys=('Ctrl+Q',), action=self.quit_action) if self.system_tray_icon is not None: self.system_tray_icon.setContextMenu(self.system_tray_menu) self.system_tray_icon.activated.connect(self.system_tray_icon_activated) self.quit_action.triggered[bool].connect(self.quit) self.donate_action.triggered[bool].connect(self.donate) self.minimize_action = QAction(_('Minimize the calibre window'), self) self.addAction(self.minimize_action) self.keyboard.register_shortcut('minimize calibre', self.minimize_action.text(), default_keys=(), action=self.minimize_action) self.minimize_action.triggered.connect(self.showMinimized) self.esc_action = QAction(self) self.addAction(self.esc_action) self.keyboard.register_shortcut('clear current search', _('Clear the current search'), default_keys=('Esc',), action=self.esc_action) self.esc_action.triggered.connect(self.esc) self.shift_esc_action = QAction(self) self.addAction(self.shift_esc_action) self.keyboard.register_shortcut('focus book list', _('Focus the book list'), default_keys=('Shift+Esc',), action=self.shift_esc_action) self.shift_esc_action.triggered.connect(self.shift_esc) self.ctrl_esc_action = QAction(self) self.addAction(self.ctrl_esc_action) self.keyboard.register_shortcut('clear virtual library', _('Clear the virtual library'), default_keys=('Ctrl+Esc',), action=self.ctrl_esc_action) self.ctrl_esc_action.triggered.connect(self.ctrl_esc) self.alt_esc_action = QAction(self) self.addAction(self.alt_esc_action) self.keyboard.register_shortcut('clear additional restriction', _('Clear the additional restriction'), default_keys=('Alt+Esc',), action=self.alt_esc_action) self.alt_esc_action.triggered.connect(self.clear_additional_restriction) # ###################### Start spare job server ######################## QTimer.singleShot(1000, self.create_spare_pool) # ###################### Location Manager ######################## self.location_manager.location_selected.connect(self.location_selected) self.location_manager.unmount_device.connect(self.device_manager.umount_device) self.location_manager.configure_device.connect(self.configure_connected_device) self.location_manager.update_device_metadata.connect(self.update_metadata_on_device) self.eject_action.triggered.connect(self.device_manager.umount_device) # ################### Update notification ################### UpdateMixin.init_update_mixin(self, opts) # ###################### Search boxes ######################## SearchRestrictionMixin.init_search_restriction_mixin(self) SavedSearchBoxMixin.init_saved_seach_box_mixin(self) # ###################### Library view ######################## LibraryViewMixin.init_library_view_mixin(self, db) SearchBoxMixin.init_search_box_mixin(self) # Requires current_db self.library_view.model().count_changed_signal.connect( self.iactions['Choose Library'].count_changed) if not gprefs.get('quick_start_guide_added', False): try: add_quick_start_guide(self.library_view) except: import traceback traceback.print_exc() for view in ('library', 'memory', 'card_a', 'card_b'): v = getattr(self, '%s_view' % view) v.selectionModel().selectionChanged.connect(self.update_status_bar) v.model().count_changed_signal.connect(self.update_status_bar) self.library_view.model().count_changed() self.bars_manager.database_changed(self.library_view.model().db) self.library_view.model().database_changed.connect(self.bars_manager.database_changed, type=Qt.QueuedConnection) # ########################## Tags Browser ############################## TagBrowserMixin.init_tag_browser_mixin(self, db) self.library_view.model().database_changed.connect(self.populate_tb_manage_menu, type=Qt.QueuedConnection) # ######################## Search Restriction ########################## if db.prefs['virtual_lib_on_startup']: self.apply_virtual_library(db.prefs['virtual_lib_on_startup']) self.rebuild_vl_tabs() # ########################## Cover Flow ################################ CoverFlowMixin.init_cover_flow_mixin(self) self._calculated_available_height = min(max_available_height()-15, self.height()) self.resize(self.width(), self._calculated_available_height) self.build_context_menus() for ac in list(self.iactions.values()): try: ac.gui_layout_complete() except: import traceback traceback.print_exc() if ac.plugin_path is None: raise if config['autolaunch_server']: self.start_content_server() self.read_settings() self.finalize_layout() self.bars_manager.start_animation() self.set_window_title() for ac in list(self.iactions.values()): try: ac.initialization_complete() except: import traceback traceback.print_exc() if ac.plugin_path is None: raise self.set_current_library_information(current_library_name(), db.library_id, db.field_metadata) register_keyboard_shortcuts() self.keyboard.finalize() if show_gui: # Note this has to come after restoreGeometry() because of # https://bugreports.qt.io/browse/QTBUG-56831 self.show() if self.system_tray_icon is not None and self.system_tray_icon.isVisible() and opts.start_in_tray: self.hide_windows() self.auto_adder = AutoAdder(gprefs['auto_add_path'], self) self.save_layout_state() # Collect cycles now gc.collect() QApplication.instance().shutdown_signal_received.connect(self.quit) if show_gui and self.gui_debug is not None: QTimer.singleShot(10, self.show_gui_debug_msg) self.iactions['Connect Share'].check_smartdevice_menus() QTimer.singleShot(1, self.start_smartdevice) QTimer.singleShot(100, self.update_toggle_to_tray_action) def show_gui_debug_msg(self): info_dialog(self, _('Debug mode'), '<p>' + _('You have started calibre in debug mode. After you ' 'quit calibre, the debug log will be available in ' 'the file: %s<p>The ' 'log will be displayed automatically.')%self.gui_debug, show=True) def esc(self, *args): self.search.clear() def shift_esc(self): self.current_view().setFocus(Qt.OtherFocusReason) def ctrl_esc(self): self.apply_virtual_library() self.current_view().setFocus(Qt.OtherFocusReason) def start_smartdevice(self): message = None if self.device_manager.get_option('smartdevice', 'autostart'): try: message = self.device_manager.start_plugin('smartdevice') except: message = 'start smartdevice unknown exception' prints(message) import traceback traceback.print_exc() if message: if not self.device_manager.is_running('Wireless Devices'): error_dialog(self, _('Problem starting the wireless device'), _('The wireless device driver had problems starting. ' 'It said "%s"')%message, show=True) self.iactions['Connect Share'].set_smartdevice_action_state() def start_content_server(self, check_started=True): from calibre.srv.embedded import Server if not gprefs.get('server3_warning_done', False): gprefs.set('server3_warning_done', True) if os.path.exists(os.path.join(config_dir, 'server.py')): try: os.remove(os.path.join(config_dir, 'server.py')) except EnvironmentError: pass warning_dialog(self, _('Content server changed!'), _( 'calibre 3 comes with a completely re-written content server.' ' As such any custom configuration you have for the content' ' server no longer applies. You should check and refresh your' ' settings in Preferences->Sharing->Sharing over the net'), show=True) self.content_server = Server(self.library_broker, Dispatcher(self.handle_changes_from_server)) self.content_server.state_callback = Dispatcher( self.iactions['Connect Share'].content_server_state_changed) if check_started: self.content_server.start_failure_callback = \ Dispatcher(self.content_server_start_failed) self.content_server.start() def handle_changes_from_server(self, library_path, change_event): if DEBUG: prints('Received server change event: {} for {}'.format(change_event, library_path)) if self.library_broker.is_gui_library(library_path): self.server_changes.put((library_path, change_event)) self.server_change_notification_timer.start() def handle_changes_from_server_debounced(self): if self.shutting_down: return changes = [] while True: try: library_path, change_event = self.server_changes.get_nowait() except Empty: break if self.library_broker.is_gui_library(library_path): changes.append(change_event) if changes: handle_changes(changes, self) def content_server_start_failed(self, msg): self.content_server = None error_dialog(self, _('Failed to start Content server'), _('Could not start the Content server. Error:\n\n%s')%msg, show=True) def resizeEvent(self, ev): MainWindow.resizeEvent(self, ev) self.search.setMaximumWidth(self.width()-150) def create_spare_pool(self, *args): if self._spare_pool is None: num = min(detect_ncpus(), int(config['worker_limit']/2.0)) self._spare_pool = Pool(max_workers=num, name='GUIPool') def spare_pool(self): ans, self._spare_pool = self._spare_pool, None QTimer.singleShot(1000, self.create_spare_pool) return ans def do_proceed(self, func, payload): if callable(func): func(payload) def no_op(self, *args): pass def system_tray_icon_activated(self, r=False): if r in (QSystemTrayIcon.Trigger, QSystemTrayIcon.MiddleClick, False): if self.isVisible(): if self.isMinimized(): self.showNormal() else: self.hide_windows() else: self.show_windows() if self.isMinimized(): self.showNormal() @property def is_minimized_to_tray(self): return getattr(self, '__systray_minimized', False) def ask_a_yes_no_question(self, title, msg, det_msg='', show_copy_button=False, ans_when_user_unavailable=True, skip_dialog_name=None, skipped_value=True): if self.is_minimized_to_tray: return ans_when_user_unavailable return question_dialog(self, title, msg, det_msg=det_msg, show_copy_button=show_copy_button, skip_dialog_name=skip_dialog_name, skip_dialog_skipped_value=skipped_value) def update_toggle_to_tray_action(self, *args): if hasattr(self, 'toggle_to_tray_action'): self.toggle_to_tray_action.setText( _('Hide main window') if self.isVisible() else _('Show main window')) def hide_windows(self): for window in QApplication.topLevelWidgets(): if isinstance(window, (MainWindow, QDialog)) and \ window.isVisible(): window.hide() setattr(window, '__systray_minimized', True) self.update_toggle_to_tray_action() def show_windows(self, *args): for window in QApplication.topLevelWidgets(): if getattr(window, '__systray_minimized', False): window.show() setattr(window, '__systray_minimized', False) self.update_toggle_to_tray_action() def test_server(self, *args): if self.content_server is not None and \ self.content_server.exception is not None: error_dialog(self, _('Failed to start Content server'), str(self.content_server.exception)).exec_() @property def current_db(self): return self.library_view.model().db def refresh_all(self): m = self.library_view.model() m.db.data.refresh(clear_caches=False, do_search=False) self.saved_searches_changed(recount=False) m.resort() m.research() self.tags_view.recount() def another_instance_wants_to_talk(self): try: msg = self.listener.queue.get_nowait() except Empty: return if msg.startswith('launched:'): import json try: argv = json.loads(msg[len('launched:'):]) except ValueError: prints('Failed to decode message from other instance: %r' % msg) if DEBUG: error_dialog(self, 'Invalid message', 'Received an invalid message from other calibre instance.' ' Do you have multiple versions of calibre installed?', det_msg='Invalid msg: %r' % msg, show=True) argv = () if isinstance(argv, (list, tuple)) and len(argv) > 1: files = [os.path.abspath(p) for p in argv[1:] if not os.path.isdir(p) and os.access(p, os.R_OK)] if files: self.iactions['Add Books'].add_filesystem_book(files) self.setWindowState(self.windowState() & ~Qt.WindowMinimized|Qt.WindowActive) self.show_windows() self.raise_() self.activateWindow() elif msg.startswith('refreshdb:'): m = self.library_view.model() m.db.new_api.reload_from_db() self.refresh_all() elif msg.startswith('shutdown:'): self.quit(confirm_quit=False) elif msg.startswith('bookedited:'): parts = msg.split(':')[1:] try: book_id, fmt, library_id = parts[:3] book_id = int(book_id) m = self.library_view.model() db = m.db.new_api if m.db.library_id == library_id and db.has_id(book_id): db.format_metadata(book_id, fmt, allow_cache=False, update_db=True) db.update_last_modified((book_id,)) m.refresh_ids((book_id,)) except Exception: import traceback traceback.print_exc() else: print(msg) def current_view(self): '''Convenience method that returns the currently visible view ''' idx = self.stack.currentIndex() if idx == 0: return self.library_view if idx == 1: return self.memory_view if idx == 2: return self.card_a_view if idx == 3: return self.card_b_view def booklists(self): return self.memory_view.model().db, self.card_a_view.model().db, self.card_b_view.model().db def library_moved(self, newloc, copy_structure=False, allow_rebuild=False): if newloc is None: return with self.library_broker: default_prefs = None try: olddb = self.library_view.model().db if copy_structure: default_prefs = olddb.prefs except: olddb = None if copy_structure and olddb is not None and default_prefs is not None: default_prefs['field_metadata'] = olddb.new_api.field_metadata.all_metadata() db = self.library_broker.prepare_for_gui_library_change(newloc) if db is None: try: db = LibraryDatabase(newloc, default_prefs=default_prefs) except apsw.Error: if not allow_rebuild: raise import traceback repair = question_dialog(self, _('Corrupted database'), _('The library database at %s appears to be corrupted. Do ' 'you want calibre to try and rebuild it automatically? ' 'The rebuild may not be completely successful.') % force_unicode(newloc, filesystem_encoding), det_msg=traceback.format_exc() ) if repair: from calibre.gui2.dialogs.restore_library import repair_library_at if repair_library_at(newloc, parent=self): db = LibraryDatabase(newloc, default_prefs=default_prefs) else: return else: return self.library_path = newloc prefs['library_path'] = self.library_path self.book_on_device(None, reset=True) db.set_book_on_device_func(self.book_on_device) self.library_view.set_database(db) self.tags_view.set_database(db, self.alter_tb) self.library_view.model().set_book_on_device_func(self.book_on_device) self.status_bar.clear_message() self.search.clear() self.saved_search.clear() self.book_details.reset_info() # self.library_view.model().count_changed() db = self.library_view.model().db self.iactions['Choose Library'].count_changed(db.count()) self.set_window_title() self.apply_named_search_restriction('') # reset restriction to null self.saved_searches_changed(recount=False) # reload the search restrictions combo box if db.prefs['virtual_lib_on_startup']: self.apply_virtual_library(db.prefs['virtual_lib_on_startup']) self.rebuild_vl_tabs() for action in list(self.iactions.values()): action.library_changed(db) self.library_broker.gui_library_changed(db, olddb) if self.device_connected: self.set_books_in_library(self.booklists(), reset=True) self.refresh_ondevice() self.memory_view.reset() self.card_a_view.reset() self.card_b_view.reset() self.set_current_library_information(current_library_name(), db.library_id, db.field_metadata) self.library_view.set_current_row(0) # Run a garbage collection now so that it does not freeze the # interface later gc.collect() def set_window_title(self): db = self.current_db restrictions = [x for x in (db.data.get_base_restriction_name(), db.data.get_search_restriction_name()) if x] restrictions = ' :: '.join(restrictions) font = QFont() if restrictions: restrictions = ' :: ' + restrictions font.setBold(True) font.setItalic(True) self.virtual_library.setFont(font) title = '{0} - || {1}{2} ||'.format( __appname__, self.iactions['Choose Library'].library_name(), restrictions) self.setWindowTitle(title) def location_selected(self, location): ''' Called when a location icon is clicked (e.g. Library) ''' page = 0 if location == 'library' else 1 if location == 'main' else 2 if location == 'carda' else 3 self.stack.setCurrentIndex(page) self.book_details.reset_info() for x in ('tb', 'cb'): splitter = getattr(self, x+'_splitter') splitter.button.setEnabled(location == 'library') for action in list(self.iactions.values()): action.location_selected(location) if location == 'library': self.virtual_library_menu.setEnabled(True) self.highlight_only_button.setEnabled(True) self.vl_tabs.setEnabled(True) else: self.virtual_library_menu.setEnabled(False) self.highlight_only_button.setEnabled(False) self.vl_tabs.setEnabled(False) # Reset the view in case something changed while it was invisible self.current_view().reset() self.set_number_of_books_shown() self.update_status_bar() def job_exception(self, job, dialog_title=_('Conversion error'), retry_func=None): if not hasattr(self, '_modeless_dialogs'): self._modeless_dialogs = [] minz = self.is_minimized_to_tray if self.isVisible(): for x in list(self._modeless_dialogs): if not x.isVisible(): self._modeless_dialogs.remove(x) try: if 'calibre.ebooks.DRMError' in job.details: if not minz: from calibre.gui2.dialogs.drm_error import DRMErrorMessage d = DRMErrorMessage(self, _('Cannot convert') + ' ' + job.description.split(':')[-1].partition('(')[-1][:-1]) d.setModal(False) d.show() self._modeless_dialogs.append(d) return if 'calibre.ebooks.oeb.transforms.split.SplitError' in job.details: title = job.description.split(':')[-1].partition('(')[-1][:-1] msg = _('<p><b>Failed to convert: %s')%title msg += '<p>'+_(''' Many older e-book reader devices are incapable of displaying EPUB files that have internal components over a certain size. Therefore, when converting to EPUB, calibre automatically tries to split up the EPUB into smaller sized pieces. For some files that are large undifferentiated blocks of text, this splitting fails. <p>You can <b>work around the problem</b> by either increasing the maximum split size under <i>EPUB output</i> in the conversion dialog, or by turning on Heuristic Processing, also in the conversion dialog. Note that if you make the maximum split size too large, your e-book reader may have trouble with the EPUB. ''') if not minz: d = error_dialog(self, _('Conversion Failed'), msg, det_msg=job.details) d.setModal(False) d.show() self._modeless_dialogs.append(d) return if 'calibre.ebooks.mobi.reader.mobi6.KFXError:' in job.details: if not minz: title = job.description.split(':')[-1].partition('(')[-1][:-1] msg = _('<p><b>Failed to convert: %s') % title idx = job.details.index('calibre.ebooks.mobi.reader.mobi6.KFXError:') msg += '<p>' + re.sub(r'(https:\S+)', r'<a href="\1">{}</a>'.format(_('here')), job.details[idx:].partition(':')[2].strip()) d = error_dialog(self, _('Conversion failed'), msg, det_msg=job.details) d.setModal(False) d.show() self._modeless_dialogs.append(d) return if 'calibre.web.feeds.input.RecipeDisabled' in job.details: if not minz: msg = job.details msg = msg[msg.find('calibre.web.feeds.input.RecipeDisabled:'):] msg = msg.partition(':')[-1] d = error_dialog(self, _('Recipe Disabled'), '<p>%s</p>'%msg) d.setModal(False) d.show() self._modeless_dialogs.append(d) return if 'calibre.ebooks.conversion.ConversionUserFeedBack:' in job.details: if not minz: import json payload = job.details.rpartition( 'calibre.ebooks.conversion.ConversionUserFeedBack:')[-1] payload = json.loads('{' + payload.partition('{')[-1]) d = {'info':info_dialog, 'warn':warning_dialog, 'error':error_dialog}.get(payload['level'], error_dialog) d = d(self, payload['title'], '<p>%s</p>'%payload['msg'], det_msg=payload['det_msg']) d.setModal(False) d.show() self._modeless_dialogs.append(d) return except: pass if job.killed: return try: prints(job.details, file=sys.stderr) except: pass if not minz: self.job_error_dialog.show_error(dialog_title, _('<b>Failed</b>')+': '+str(job.description), det_msg=job.details, retry_func=retry_func) def read_settings(self): geometry = config['main_window_geometry'] if geometry is not None: self.restoreGeometry(geometry) self.read_layout_settings() def write_settings(self): with gprefs: # Only write to gprefs once config.set('main_window_geometry', self.saveGeometry()) dynamic.set('sort_history', self.library_view.model().sort_history) self.save_layout_state() self.stack.tb_widget.save_state() def quit(self, checked=True, restart=False, debug_on_restart=False, confirm_quit=True): if self.shutting_down: return if confirm_quit and not self.confirm_quit(): return try: self.shutdown() except: pass self.restart_after_quit = restart self.debug_on_restart = debug_on_restart QApplication.instance().quit() def donate(self, *args): open_url(QUrl('https://calibre-ebook.com/donate')) def confirm_quit(self): if self.job_manager.has_jobs(): msg = _('There are active jobs. Are you sure you want to quit?') if self.job_manager.has_device_jobs(): msg = '<p>'+__appname__ + \ _(''' is communicating with the device!<br> Quitting may cause corruption on the device.<br> Are you sure you want to quit?''')+'</p>' if not question_dialog(self, _('Active jobs'), msg): return False if self.proceed_question.questions: msg = _('There are library updates waiting. Are you sure you want to quit?') if not question_dialog(self, _('Library updates waiting'), msg): return False from calibre.db.delete_service import has_jobs if has_jobs(): msg = _('Some deleted books are still being moved to the Recycle ' 'Bin, if you quit now, they will be left behind. Are you ' 'sure you want to quit?') if not question_dialog(self, _('Active jobs'), msg): return False return True def shutdown(self, write_settings=True): self.shutting_down = True self.show_shutdown_message() self.server_change_notification_timer.stop() from calibre.customize.ui import has_library_closed_plugins if has_library_closed_plugins(): self.show_shutdown_message( _('Running database shutdown plugins. This could take a few seconds...')) self.grid_view.shutdown() db = None try: db = self.library_view.model().db cf = db.clean except: pass else: cf() # Save the current field_metadata for applications like calibre2opds # Goes here, because if cf is valid, db is valid. db.new_api.set_pref('field_metadata', db.field_metadata.all_metadata()) db.commit_dirty_cache() db.prefs.write_serialized(prefs['library_path']) for action in list(self.iactions.values()): if not action.shutting_down(): return if write_settings: self.write_settings() self.check_messages_timer.stop() if hasattr(self, 'update_checker'): self.update_checker.shutdown() self.listener.close() self.job_manager.server.close() self.job_manager.threaded_server.close() self.device_manager.keep_going = False self.auto_adder.stop() # Do not report any errors that happen after the shutdown # We cannot restore the original excepthook as that causes PyQt to # call abort() on unhandled exceptions import traceback def eh(t, v, tb): try: traceback.print_exception(t, v, tb, file=sys.stderr) except: pass sys.excepthook = eh mb = self.library_view.model().metadata_backup if mb is not None: mb.stop() self.library_view.model().close() try: try: if self.content_server is not None: # If the Content server has any sockets being closed then # this can take quite a long time (minutes). Tell the user that it is # happening. self.show_shutdown_message( _('Shutting down the Content server. This could take a while...')) s = self.content_server self.content_server = None s.exit() except: pass except KeyboardInterrupt: pass self.hide_windows() if self._spare_pool is not None: self._spare_pool.shutdown() from calibre.db.delete_service import shutdown shutdown() time.sleep(2) self.istores.join() return True def run_wizard(self, *args): if self.confirm_quit(): self.run_wizard_b4_shutdown = True self.restart_after_quit = True try: self.shutdown(write_settings=False) except: pass QApplication.instance().quit() def closeEvent(self, e): if self.shutting_down: return self.write_settings() if self.system_tray_icon is not None and self.system_tray_icon.isVisible(): if not dynamic['systray_msg'] and not isosx: info_dialog(self, 'calibre', 'calibre '+ _('will keep running in the system tray. To close it, ' 'choose <b>Quit</b> in the context menu of the ' 'system tray.'), show_copy_button=False).exec_() dynamic['systray_msg'] = True self.hide_windows() e.ignore() else: if self.confirm_quit(): try: self.shutdown(write_settings=False) except: import traceback traceback.print_exc() e.accept() else: e.ignore()
class InventoryModule(BaseInventoryPlugin, Constructable): NAME = 'azure.azcollection.azure_rm' def __init__(self): super(InventoryModule, self).__init__() self._serializer = Serializer() self._deserializer = Deserializer() self._hosts = [] self._filters = None # FUTURE: use API profiles with defaults self._compute_api_version = '2017-03-30' self._network_api_version = '2015-06-15' self._default_header_parameters = { 'Content-Type': 'application/json; charset=utf-8' } self._request_queue = Queue() self.azure_auth = None self._batch_fetch = False def verify_file(self, path): ''' :param loader: an ansible.parsing.dataloader.DataLoader object :param path: the path to the inventory config file :return the contents of the config file ''' if super(InventoryModule, self).verify_file(path): if re.match(r'.{0,}azure_rm\.y(a)?ml$', path): return True # display.debug("azure_rm inventory filename must end with 'azure_rm.yml' or 'azure_rm.yaml'") return False def parse(self, inventory, loader, path, cache=True): super(InventoryModule, self).parse(inventory, loader, path) self._read_config_data(path) if self.get_option('use_contrib_script_compatible_sanitization'): self._sanitize_group_name = self._legacy_script_compatible_group_sanitization self._batch_fetch = self.get_option('batch_fetch') self._legacy_hostnames = self.get_option('plain_host_names') self._filters = self.get_option( 'exclude_host_filters') + self.get_option('default_host_filters') try: self._credential_setup() self._get_hosts() except Exception: raise def _credential_setup(self): auth_options = dict( auth_source=self.get_option('auth_source'), profile=self.get_option('profile'), subscription_id=self.get_option('subscription_id'), client_id=self.get_option('client_id'), secret=self.get_option('secret'), tenant=self.get_option('tenant'), ad_user=self.get_option('ad_user'), password=self.get_option('password'), cloud_environment=self.get_option('cloud_environment'), cert_validation_mode=self.get_option('cert_validation_mode'), api_profile=self.get_option('api_profile'), adfs_authority_url=self.get_option('adfs_authority_url')) self.azure_auth = AzureRMAuth(**auth_options) self._clientconfig = AzureRMRestConfiguration( self.azure_auth.azure_credentials, self.azure_auth.subscription_id, self.azure_auth._cloud_environment.endpoints.resource_manager) self._client = ServiceClient(self._clientconfig.credentials, self._clientconfig) def _enqueue_get(self, url, api_version, handler, handler_args=None): if not handler_args: handler_args = {} self._request_queue.put_nowait( UrlAction(url=url, api_version=api_version, handler=handler, handler_args=handler_args)) def _enqueue_vm_list(self, rg='*'): if not rg or rg == '*': url = '/subscriptions/{subscriptionId}/providers/Microsoft.Compute/virtualMachines' else: url = '/subscriptions/{subscriptionId}/resourceGroups/{rg}/providers/Microsoft.Compute/virtualMachines' url = url.format(subscriptionId=self._clientconfig.subscription_id, rg=rg) self._enqueue_get(url=url, api_version=self._compute_api_version, handler=self._on_vm_page_response) def _enqueue_vmss_list(self, rg=None): if not rg or rg == '*': url = '/subscriptions/{subscriptionId}/providers/Microsoft.Compute/virtualMachineScaleSets' else: url = '/subscriptions/{subscriptionId}/resourceGroups/{rg}/providers/Microsoft.Compute/virtualMachineScaleSets' url = url.format(subscriptionId=self._clientconfig.subscription_id, rg=rg) self._enqueue_get(url=url, api_version=self._compute_api_version, handler=self._on_vmss_page_response) def _get_hosts(self): for vm_rg in self.get_option('include_vm_resource_groups'): self._enqueue_vm_list(vm_rg) for vmss_rg in self.get_option('include_vmss_resource_groups'): self._enqueue_vmss_list(vmss_rg) if self._batch_fetch: self._process_queue_batch() else: self._process_queue_serial() constructable_config_strict = boolean( self.get_option('fail_on_template_errors')) constructable_config_compose = self.get_option('hostvar_expressions') constructable_config_groups = self.get_option('conditional_groups') constructable_config_keyed_groups = self.get_option('keyed_groups') for h in self._hosts: inventory_hostname = self._get_hostname(h) if self._filter_host(inventory_hostname, h.hostvars): continue self.inventory.add_host(inventory_hostname) # FUTURE: configurable default IP list? can already do this via hostvar_expressions self.inventory.set_variable( inventory_hostname, "ansible_host", next( chain(h.hostvars['public_ipv4_addresses'], h.hostvars['private_ipv4_addresses']), None)) for k, v in iteritems(h.hostvars): # FUTURE: configurable hostvar prefix? Makes docs harder... self.inventory.set_variable(inventory_hostname, k, v) # constructable delegation self._set_composite_vars(constructable_config_compose, h.hostvars, inventory_hostname, strict=constructable_config_strict) self._add_host_to_composed_groups( constructable_config_groups, h.hostvars, inventory_hostname, strict=constructable_config_strict) self._add_host_to_keyed_groups(constructable_config_keyed_groups, h.hostvars, inventory_hostname, strict=constructable_config_strict) # FUTURE: fix underlying inventory stuff to allow us to quickly access known groupvars from reconciled host def _filter_host(self, inventory_hostname, hostvars): self.templar.available_variables = hostvars for condition in self._filters: # FUTURE: should warn/fail if conditional doesn't return True or False conditional = "{{% if {0} %}} True {{% else %}} False {{% endif %}}".format( condition) try: if boolean(self.templar.template(conditional)): return True except Exception as e: if boolean(self.get_option('fail_on_template_errors')): raise AnsibleParserError( "Error evaluating filter condition '{0}' for host {1}: {2}" .format(condition, inventory_hostname, to_native(e))) continue return False def _get_hostname(self, host): # FUTURE: configurable hostname sources return host.default_inventory_hostname def _process_queue_serial(self): try: while True: item = self._request_queue.get_nowait() resp = self.send_request(item.url, item.api_version) item.handler(resp, **item.handler_args) except Empty: pass def _on_vm_page_response(self, response, vmss=None): next_link = response.get('nextLink') if next_link: self._enqueue_get(url=next_link, api_version=self._compute_api_version, handler=self._on_vm_page_response) if 'value' in response: for h in response['value']: # FUTURE: add direct VM filtering by tag here (performance optimization)? self._hosts.append( AzureHost(h, self, vmss=vmss, legacy_name=self._legacy_hostnames)) def _on_vmss_page_response(self, response): next_link = response.get('nextLink') if next_link: self._enqueue_get(url=next_link, api_version=self._compute_api_version, handler=self._on_vmss_page_response) # FUTURE: add direct VMSS filtering by tag here (performance optimization)? for vmss in response['value']: url = '{0}/virtualMachines'.format(vmss['id']) # VMSS instances look close enough to regular VMs that we can share the handler impl... self._enqueue_get(url=url, api_version=self._compute_api_version, handler=self._on_vm_page_response, handler_args=dict(vmss=vmss)) # use the undocumented /batch endpoint to bulk-send up to 500 requests in a single round-trip # def _process_queue_batch(self): while True: batch_requests = [] batch_item_index = 0 batch_response_handlers = dict() try: while batch_item_index < 100: item = self._request_queue.get_nowait() name = str(uuid.uuid4()) query_parameters = {'api-version': item.api_version} req = self._client.get(item.url, query_parameters) batch_requests.append( dict(httpMethod="GET", url=req.url, name=name)) batch_response_handlers[name] = item batch_item_index += 1 except Empty: pass if not batch_requests: break batch_resp = self._send_batch(batch_requests) key_name = None if 'responses' in batch_resp: key_name = 'responses' elif 'value' in batch_resp: key_name = 'value' else: raise AnsibleError( "didn't find expected key responses/value in batch response" ) for idx, r in enumerate(batch_resp[key_name]): status_code = r.get('httpStatusCode') returned_name = r['name'] result = batch_response_handlers[returned_name] if status_code != 200: # FUTURE: error-tolerant operation mode (eg, permissions) raise AnsibleError( "a batched request failed with status code {0}, url {1}" .format(status_code, result.url)) # FUTURE: store/handle errors from individual handlers result.handler(r['content'], **result.handler_args) def _send_batch(self, batched_requests): url = '/batch' query_parameters = {'api-version': '2015-11-01'} body_obj = dict(requests=batched_requests) body_content = self._serializer.body(body_obj, 'object') header = {'x-ms-client-request-id': str(uuid.uuid4())} header.update(self._default_header_parameters) request = self._client.post(url, query_parameters) initial_response = self._client.send(request, header, body_content) # FUTURE: configurable timeout? poller = ARMPolling(timeout=2) poller.initialize( client=self._client, initial_response=initial_response, deserialization_callback=lambda r: self._deserializer('object', r)) poller.run() return poller.resource() def send_request(self, url, api_version): query_parameters = {'api-version': api_version} req = self._client.get(url, query_parameters) resp = self._client.send(req, self._default_header_parameters, stream=False) resp.raise_for_status() content = resp.content return json.loads(content) @staticmethod def _legacy_script_compatible_group_sanitization(name): # note that while this mirrors what the script used to do, it has many issues with unicode and usability in python regex = re.compile(r"[^A-Za-z0-9\_\-]") return regex.sub('_', name)
class SingleMachineBatchSystem(BatchSystemSupport): """ The interface for running jobs on a single machine, runs all the jobs you give it as they come in, but in parallel. Uses a single "daddy" thread to manage a fleet of child processes. Communication with the daddy thread happens via two queues: one queue of jobs waiting to be run (the input queue), and one queue of jobs that are finished/stopped and need to be returned by getUpdatedBatchJob (the output queue). When the batch system is shut down, the daddy thread is stopped. If running in debug-worker mode, jobs are run immediately as they are sent to the batch system, in the sending thread, and the daddy thread is not run. But the queues are still used. """ @classmethod def supportsAutoDeployment(cls): return False @classmethod def supportsWorkerCleanup(cls): return True numCores = cpu_count() minCores = 0.1 """ The minimal fractional CPU. Tasks with a smaller core requirement will be rounded up to this value. """ physicalMemory = toil.physicalMemory() def __init__(self, config, maxCores, maxMemory, maxDisk): self.config = config # Limit to the smaller of the user-imposed limit and what we actually # have on this machine for each resource. # # If we don't have up to the limit of the resource (and the resource # isn't the inlimited sentinel), warn. if maxCores > self.numCores: if maxCores != sys.maxsize: # We have an actually specified limit and not the default log.warning( 'Not enough cores! User limited to %i but we only have %i.', maxCores, self.numCores) maxCores = self.numCores if maxMemory > self.physicalMemory: if maxMemory != sys.maxsize: # We have an actually specified limit and not the default log.warning( 'Not enough memory! User limited to %i bytes but we only have %i bytes.', maxMemory, self.physicalMemory) maxMemory = self.physicalMemory workdir = Toil.getLocalWorkflowDir( config.workflowID, config.workDir ) # config.workDir may be None; this sets a real directory self.physicalDisk = toil.physicalDisk(workdir) if maxDisk > self.physicalDisk: if maxDisk != sys.maxsize: # We have an actually specified limit and not the default log.warning( 'Not enough disk space! User limited to %i bytes but we only have %i bytes.', maxDisk, self.physicalDisk) maxDisk = self.physicalDisk super(SingleMachineBatchSystem, self).__init__(config, maxCores, maxMemory, maxDisk) assert self.maxCores >= self.minCores assert self.maxMemory >= 1 # The scale allows the user to apply a factor to each task's cores requirement, thereby # squeezing more tasks onto each core (scale < 1) or stretching tasks over more cores # (scale > 1). self.scale = config.scale if config.badWorker > 0 and config.debugWorker: # We can't throw SIGUSR1 at the worker because it is also going to # be the leader and/or test harness. raise RuntimeError( "Cannot use badWorker and debugWorker together; " "worker would have to kill the leader") self.debugWorker = config.debugWorker # A counter to generate job IDs and a lock to guard it self.jobIndex = 0 self.jobIndexLock = Lock() # A dictionary mapping IDs of submitted jobs to the command line self.jobs = {} """ :type: dict[str,toil.job.JobDescription] """ # A queue of jobs waiting to be executed. Consumed by the daddy thread. self.inputQueue = Queue() # A queue of finished jobs. Produced by the daddy thread. self.outputQueue = Queue() # A dictionary mapping IDs of currently running jobs to their Info objects self.runningJobs = {} """ :type: dict[str,Info] """ # These next two are only used outside debug-worker mode # A dict mapping PIDs to Popen objects for running jobs. # Jobs that don't fork are executed one at a time in the main thread. self.children = {} """ :type: dict[int,subprocess.Popen] """ # A dict mapping child PIDs to the Job IDs they are supposed to be running. self.childToJob = {} """ :type: dict[int,str] """ # A pool representing available CPU in units of minCores self.coreFractions = ResourcePool(int(self.maxCores / self.minCores), 'cores') # A pool representing available memory in bytes self.memory = ResourcePool(self.maxMemory, 'memory') # A pool representing the available space in bytes self.disk = ResourcePool(self.maxDisk, 'disk') # If we can't schedule something, we fill this in with a reason why self.schedulingStatusMessage = None # We use this event to signal shutdown self.shuttingDown = Event() # A thread in charge of managing all our child processes. # Also takes care of resource accounting. self.daddyThread = None # If it breaks it will fill this in self.daddyException = None if self.debugWorker: log.debug('Started in worker debug mode.') else: self.daddyThread = Thread(target=self.daddy, daemon=True) self.daddyThread.start() log.debug('Started in normal mode.') def daddy(self): """ Be the "daddy" thread. Our job is to look at jobs from the input queue. If a job fits in the available resources, we allocate resources for it and kick off a child process. We also check on our children. When a child finishes, we reap it, release its resources, and put its information in the output queue. """ try: log.debug('Started daddy thread.') while not self.shuttingDown.is_set(): # Main loop while not self.shuttingDown.is_set(): # Try to start as many jobs as we can try to start try: # Grab something from the input queue if available. args = self.inputQueue.get_nowait() jobCommand, jobID, jobCores, jobMemory, jobDisk, environment = args coreFractions = int(jobCores / self.minCores) # Try to start the child result = self._startChild(jobCommand, jobID, coreFractions, jobMemory, jobDisk, environment) if result is None: # We did not get the resources to run this job. # Requeue last, so we can look at the next job. # TODO: Have some kind of condition the job can wait on, # but without threads (queues for jobs needing # cores/memory/disk individually)? self.inputQueue.put(args) break # Otherwise it's a PID if it succeeded, or False if it couldn't # start. But we don't care either way here. except Empty: # Nothing to run. Stop looking in the queue. break # Now check on our children. for done_pid in self._pollForDoneChildrenIn(self.children): # A child has actually finished. # Clean up after it. self._handleChild(done_pid) # Then loop again: start and collect more jobs. # TODO: It would be good to be able to wait on a new job or a finished child, whichever comes first. # For now we just sleep and loop. time.sleep(0.01) # When we get here, we are shutting down. for popen in self.children.values(): # Kill all the children, going through popen to avoid signaling re-used PIDs. popen.kill() for popen in self.children.values(): # Reap all the children popen.wait() # Then exit the thread. return except Exception as e: log.critical('Unhandled exception in daddy thread: %s', traceback.format_exc()) # Pass the exception back to the main thread so it can stop the next person who calls into us. self.daddyException = e raise def _checkOnDaddy(self): if self.daddyException is not None: # The daddy thread broke and we cannot do our job log.critical( 'Propagating unhandled exception in daddy thread to main thread' ) exc = self.daddyException self.daddyException = None raise exc def _pollForDoneChildrenIn(self, pid_to_popen): """ See if any children represented in the given dict from PID to Popen object have finished. Return a collection of their PIDs. Guarantees that each child's exit code will be gettable via wait() on the child's Popen object (i.e. does not reap the child, unless via Popen). """ # We keep our found PIDs in a set so we can work around waitid showing # us the same one repeatedly. ready = set() # Find the waitid function waitid = getattr(os, 'waitid', None) if callable(waitid): # waitid exists (not Mac) while True: # Poll for any child to have exit, but don't reap it. Leave reaping # to the Popen. # TODO: What if someone else in Toil wants to do this syscall? # TODO: Is this one-notification-per-done-child with WNOHANG? Or # can we miss some? Or do we see the same one repeatedly until it # is reaped? try: siginfo = waitid(os.P_ALL, -1, os.WEXITED | os.WNOWAIT | os.WNOHANG) except ChildProcessError: # This happens when there is nothing to wait on right now, # instead of the weird C behavior of overwriting a field in # a pointed-to struct. siginfo = None if siginfo is not None and siginfo.si_pid in pid_to_popen and siginfo.si_pid not in ready: # Something new finished ready.add(siginfo.si_pid) else: # Nothing we own that we haven't seen before has finished. return ready else: # On Mac there's no waitid and no way to wait and not reap. # Fall back on polling all the Popen objects. # To make this vaguely efficient we have to return done children in # batches. for pid, popen in pid_to_popen.items(): if popen.poll() is not None: # Process is done ready.add(pid) log.debug('Child %d has stopped', pid) # Return all the done processes we found return ready def _runDebugJob(self, jobCommand, jobID, environment): """ Run the jobCommand right now, in the current thread. May only be called in debug-worker mode. Assumes resources are available. """ assert self.debugWorker # TODO: It is not possible to kill running jobs in forkless mode, # because they are run immediately in the main thread. info = Info(time.time(), None, None, killIntended=False) self.runningJobs[jobID] = info if jobCommand.startswith("_toil_worker "): # We can actually run in this thread jobName, jobStoreLocator, jobStoreID = jobCommand.split()[ 1:] # Parse command jobStore = Toil.resumeJobStore(jobStoreLocator) toil_worker.workerScript( jobStore, jobStore.config, jobName, jobStoreID, redirectOutputToLogFile=not self.debugWorker ) # Call the worker else: # Run synchronously. If starting or running the command fails, let the exception stop us. subprocess.check_call(jobCommand, shell=True, env=dict(os.environ, **environment)) self.runningJobs.pop(jobID) if not info.killIntended: self.outputQueue.put( UpdatedBatchJobInfo(jobID=jobID, exitStatus=0, wallTime=time.time() - info.time, exitReason=None)) def getSchedulingStatusMessage(self): # Implement the abstractBatchSystem's scheduling status message API return self.schedulingStatusMessage def _setSchedulingStatusMessage(self, message): """ If we can't run a job, we record a short message about why not. If the leader wants to know what is up with us (for example, to diagnose a deadlock), it can ask us for the message. """ self.schedulingStatusMessage = message def _startChild(self, jobCommand, jobID, coreFractions, jobMemory, jobDisk, environment): """ Start a child process for the given job. Allocate its required resources and save it and save it in our bookkeeping structures. If the job is started, returns its PID. If the job fails to start, reports it as failed and returns False. If the job cannot get the resources it needs to start, returns None. """ # We fill this in if we manage to actually start the child. popen = None # This is when we started working on the job. startTime = time.time() # See if we can fit the job in our resource pools right now. if self.coreFractions.acquireNow(coreFractions): # We got some cores if self.memory.acquireNow(jobMemory): # We got some memory if self.disk.acquireNow(jobDisk): # We got the final resource, disk. # Actually run the job. # When it finishes we will release what it was using. # So it is important to not lose track of the child process. try: # Launch the job popen = subprocess.Popen(jobCommand, shell=True, env=dict( os.environ, **environment)) except Exception: # If the job can't start, make sure we release resources now self.coreFractions.release(coreFractions) self.memory.release(jobMemory) self.disk.release(jobDisk) log.error('Could not start job %s: %s', jobID, traceback.format_exc()) # Report as failed. self.outputQueue.put( UpdatedBatchJobInfo( jobID=jobID, exitStatus=EXIT_STATUS_UNAVAILABLE_VALUE, wallTime=0, exitReason=None)) # Free resources self.coreFractions.release(coreFractions) self.memory.release(jobMemory) self.disk.release(jobDisk) # Complain it broke. return False else: # If the job did start, record it self.children[popen.pid] = popen # Make sure we can look it up by PID later self.childToJob[popen.pid] = jobID # Record that the job is running, and the resources it is using info = Info(startTime, popen, (coreFractions, jobMemory, jobDisk), killIntended=False) self.runningJobs[jobID] = info log.debug('Launched job %s as child %d', jobID, popen.pid) # Report success starting the job # Note that if a PID were somehow 0 it would look like False assert popen.pid != 0 return popen.pid else: # We can't get disk, so free cores and memory self.coreFractions.release(coreFractions) self.memory.release(jobMemory) self._setSchedulingStatusMessage( 'Not enough disk to run job %s' % jobID) else: # Free cores, since we can't get memory self.coreFractions.release(coreFractions) self._setSchedulingStatusMessage( 'Not enough memory to run job %s' % jobID) else: self._setSchedulingStatusMessage('Not enough cores to run job %s' % jobID) # If we get here, we didn't succeed or fail starting the job. # We didn't manage to get the resources. # Report that. return None def _handleChild(self, pid): """ Handle a child process PID that has finished. The PID must be for a child job we started. Not thread safe to run at the same time as we are making more children. Remove the child from our bookkeeping structures and free its resources. """ # Look up the child popen = self.children[pid] jobID = self.childToJob[pid] info = self.runningJobs[jobID] # Unpack the job resources (coreFractions, jobMemory, jobDisk) = info.resources # Clean up our records of the job. self.runningJobs.pop(jobID) self.childToJob.pop(pid) self.children.pop(pid) # See how the child did, and reap it. statusCode = popen.wait() if statusCode != 0 and not info.killIntended: log.error("Got exit code %i (indicating failure) " "from job %s.", statusCode, self.jobs[jobID]) if not info.killIntended: # Report if the job failed and we didn't kill it. # If we killed it then it shouldn't show up in the queue. self.outputQueue.put( UpdatedBatchJobInfo(jobID=jobID, exitStatus=statusCode, wallTime=time.time() - info.time, exitReason=None)) # Free up the job's resources. self.coreFractions.release(coreFractions) self.memory.release(jobMemory) self.disk.release(jobDisk) log.debug('Child %d for job %s succeeded', pid, jobID) def issueBatchJob(self, jobDesc): """Adds the command and resources to a queue to be run.""" self._checkOnDaddy() # Round cores to minCores and apply scale. # Make sure to give minCores even if asked for 0 cores, or negative or something. cores = max( math.ceil(jobDesc.cores * self.scale / self.minCores) * self.minCores, self.minCores) # Don't do our own assertions about job size vs. our configured size. # The abstract batch system can handle it. self.checkResourceRequest(jobDesc.memory, cores, jobDesc.disk, job_name=jobDesc.jobName, detail=f'Scale is set to {self.scale}.') log.debug( f"Issuing the command: {jobDesc.command} with " f"memory: {jobDesc.memory}, cores: {cores}, disk: {jobDesc.disk}") with self.jobIndexLock: jobID = self.jobIndex self.jobIndex += 1 self.jobs[jobID] = jobDesc.command if self.debugWorker: # Run immediately, blocking for return. # Ignore resource requirements; we run one job at a time self._runDebugJob(jobDesc.command, jobID, self.environment.copy()) else: # Queue the job for later self.inputQueue.put( (jobDesc.command, jobID, cores, jobDesc.memory, jobDesc.disk, self.environment.copy())) return jobID def killBatchJobs(self, jobIDs): """Kills jobs by ID.""" self._checkOnDaddy() log.debug('Killing jobs: {}'.format(jobIDs)) for jobID in jobIDs: if jobID in self.runningJobs: info = self.runningJobs[jobID] info.killIntended = True if info.popen is not None: log.debug('Send kill to PID %s', info.popen.pid) info.popen.kill() log.debug('Sent kill to PID %s', info.popen.pid) else: # No popen if running in forkless mode currently assert self.debugWorker log.critical("Can't kill job: %s in debug mode" % jobID) while jobID in self.runningJobs: pass def getIssuedBatchJobIDs(self): """Just returns all the jobs that have been run, but not yet returned as updated.""" self._checkOnDaddy() return list(self.jobs.keys()) def getRunningBatchJobIDs(self): self._checkOnDaddy() now = time.time() return { jobID: now - info.time for jobID, info in list(self.runningJobs.items()) } def shutdown(self): """ Cleanly terminate and join daddy thread. """ if self.daddyThread is not None: # Tell the daddy thread to stop. self.shuttingDown.set() # Wait for it to stop. self.daddyThread.join() BatchSystemSupport.workerCleanup(self.workerCleanupInfo) def getUpdatedBatchJob(self, maxWait): """Returns a tuple of a no-longer-running job, the return value of its process, and its runtime, or None.""" self._checkOnDaddy() try: item = self.outputQueue.get(timeout=maxWait) except Empty: return None self.jobs.pop(item.jobID) log.debug("Ran jobID: %s with exit value: %i", item.jobID, item.exitStatus) return item @classmethod def setOptions(cls, setOption): setOption("scale", default=1)
class _PrefetchingIter(object): def __init__(self, dataloader, dataloader_it, num_threads=None): self.queue = Queue(1) self.dataloader_it = dataloader_it self.dataloader = dataloader self.num_threads = num_threads self.use_thread = dataloader.use_prefetch_thread self.use_alternate_streams = dataloader.use_alternate_streams self.device = self.dataloader.device if self.use_alternate_streams and self.device.type == 'cuda': self.stream = torch.cuda.Stream(device=self.device) else: self.stream = None self._shutting_down = False if self.use_thread: self._done_event = threading.Event() thread = threading.Thread( target=_prefetcher_entry, args=(dataloader_it, dataloader, self.queue, num_threads, self.stream, self._done_event), daemon=True) thread.start() self.thread = thread def __iter__(self): return self def _shutdown(self): # Sometimes when Python is exiting complicated operations like # self.queue.get_nowait() will hang. So we set it to no-op and let Python handle # the rest since the thread is daemonic. # PyTorch takes the same solution. if PYTHON_EXIT_STATUS is True or PYTHON_EXIT_STATUS is None: return if not self._shutting_down: try: self._shutting_down = True self._done_event.set() try: self.queue.get_nowait() # In case the thread is blocking on put(). except: # pylint: disable=bare-except pass self.thread.join() except: # pylint: disable=bare-except pass def __del__(self): if self.use_thread: self._shutdown() def _next_non_threaded(self): batch = next(self.dataloader_it) batch = recursive_apply(batch, restore_parent_storage_columns, self.dataloader.graph) batch, feats, stream_event = _prefetch(batch, self.dataloader, self.stream) return batch, feats, stream_event def _next_threaded(self): try: batch, feats, stream_event, exception = self.queue.get(timeout=prefetcher_timeout) except Empty: raise RuntimeError( f'Prefetcher thread timed out at {prefetcher_timeout} seconds.') if batch is None: self.thread.join() if exception is None: raise StopIteration exception.reraise() return batch, feats, stream_event def __next__(self): batch, feats, stream_event = \ self._next_non_threaded() if not self.use_thread else self._next_threaded() batch = recursive_apply_pair(batch, feats, _assign_for) if stream_event is not None: stream_event.wait() return batch
async def get_item(self, queue: Queue): return queue.get_nowait()
def search_overseer_thread(args, method, new_location_queue, pause_bit, encryption_lib_path, db_updates_queue, wh_queue): log.info('Search overseer starting') search_items_queue = Queue() threadStatus = {} threadStatus['Overseer'] = { 'message': 'Initializing', 'type': 'Overseer', 'method': 'Hex Grid' if method == 'hex' else 'Spawn Point' } if (args.print_status): log.info('Starting status printer thread') t = Thread(target=status_printer, name='status_printer', args=(threadStatus, search_items_queue, db_updates_queue, wh_queue)) t.daemon = True t.start() # Create a search_worker_thread per account log.info('Starting search worker threads') for i, account in enumerate(args.accounts): log.debug('Starting search worker thread %d for user %s', i, account['username']) workerId = 'Worker {:03}'.format(i) threadStatus[workerId] = { 'type': 'Worker', 'message': 'Creating thread...', 'success': 0, 'fail': 0, 'noitems': 0, 'skip': 0, 'user': account['username'] } t = Thread(target=search_worker_thread, name='search-worker-{}'.format(i), args=(args, account, search_items_queue, pause_bit, encryption_lib_path, threadStatus[workerId], db_updates_queue, wh_queue)) t.daemon = True t.start() ''' For hex scanning, we can generate the full list of scan points well in advance. When then can queue them all up to be searched as fast as the threads will allow. With spawn point scanning (sps) we can come up with the order early on, and we can populate the entire queue, but the individual threads will need to wait until the point is available (and ensure it is not to late as well). ''' # A place to track the current location current_location = False # Used to tell SPS to scan for all CURRENT pokemon instead # of, like during a normal loop, just finding the next one # which will appear (since you've already scanned existing # locations in the prior loop) # Needed in a first loop and pausing/changing location. sps_scan_current = True # The real work starts here but will halt on pause_bit.set() while True: # paused; clear queue if needed, otherwise sleep and loop while pause_bit.is_set(): if not search_items_queue.empty(): try: while True: search_items_queue.get_nowait() except Empty: pass threadStatus['Overseer']['message'] = 'Scanning is paused' sps_scan_current = True time.sleep(1) # If a new location has been passed to us, get the most recent one if not new_location_queue.empty(): log.info('New location caught, moving search grid') sps_scan_current = True try: while True: current_location = new_location_queue.get_nowait() except Empty: pass # We (may) need to clear the search_items_queue if not search_items_queue.empty(): try: while True: search_items_queue.get_nowait() except Empty: pass # If there are no search_items_queue either the loop has finished (or been # cleared above) -- either way, time to fill it back up if search_items_queue.empty(): log.debug('Search queue empty, restarting loop') # locations = [((lat, lng, alt), ts_appears, ts_leaves),...] if method == 'hex': locations = get_hex_location_list(args, current_location) else: locations = get_sps_location_list(args, current_location, sps_scan_current) sps_scan_current = False if len(locations) == 0: log.warning('Nothing to scan!') threadStatus['Overseer']['message'] = 'Queuing steps' for step, step_location in enumerate(locations, 1): log.debug('Queueing step %d @ %f/%f/%f', step, step_location[0][0], step_location[0][1], step_location[0][2]) search_args = (step, step_location[0], step_location[1], step_location[2]) search_items_queue.put(search_args) else: nextitem = search_items_queue.queue[0] threadStatus['Overseer'][ 'message'] = 'Processing search queue, next item is {:6f},{:6f}'.format( nextitem[1][0], nextitem[1][1]) # If times are specified, print the time of the next queue item, and how many seconds ahead/behind realtime if nextitem[2]: threadStatus['Overseer']['message'] += ' @ {}'.format( time.strftime('%H:%M:%S', time.localtime(nextitem[2]))) if nextitem[2] > now(): threadStatus['Overseer'][ 'message'] += ' ({}s ahead)'.format(nextitem[2] - now()) else: threadStatus['Overseer'][ 'message'] += ' ({}s behind)'.format(now() - nextitem[2]) # Now we just give a little pause here time.sleep(1)
class FFMPEGNVR(Thread): nvr_list: List[object] = [] def __init__(self, config, detector, detector_queue, post_processors, mqtt_queue=None): Thread.__init__(self) self.setup_loggers(config) self._logger.debug("Initializing NVR thread") # Use FFMPEG to read from camera. Used for reading/recording self.camera = FFMPEGCamera(config) self._mqtt = MQTT(config, mqtt_queue) self.config = config self.kill_received = False self.camera_grabber = None self._objects_in_fov = [] self._labels_in_fov = [] self._reported_label_count = {} self.idle_frames = 0 self._motion_frames = 0 self._motion_detected = False self._motion_only_frames = 0 self._motion_max_timeout_reached = False self.detector = detector self._post_processors = post_processors self._object_decoder_queue = Queue(maxsize=2) self._motion_decoder_queue = Queue(maxsize=2) motion_queue = Queue(maxsize=2) self.object_return_queue = Queue(maxsize=2) self.motion_return_queue = Queue(maxsize=2) if config.motion_detection.trigger_detector: self.camera.scan_for_motion.set() self.camera.scan_for_objects.clear() else: self.camera.scan_for_objects.set() self.camera.scan_for_motion.clear() self._object_filters = {} for object_filter in config.object_detection.labels: self._object_filters[object_filter.label] = Filter(object_filter) self._zones = [] for zone in config.camera.zones: self._zones.append( Zone( zone, self.camera.resolution, config, self._mqtt.mqtt_queue, post_processors, )) # Motion detector class. if config.motion_detection.timeout or config.motion_detection.trigger_detector: self.motion_detector = MotionDetection(config, self.camera.resolution) self.motion_thread = Thread( target=self.motion_detector.motion_detection, args=(motion_queue, )) self.motion_thread.daemon = True self.motion_thread.start() self.motion_decoder = Thread( target=self.camera.decoder, args=( self._motion_decoder_queue, motion_queue, config.motion_detection.width, config.motion_detection.height, ), ) self.motion_decoder.daemon = True self.motion_decoder.start() self.object_decoder = Thread( target=self.camera.decoder, args=( self._object_decoder_queue, detector_queue, detector.model_width, detector.model_height, ), ) self.object_decoder.daemon = True self.object_decoder.start() self.start_camera() # Initialize recorder self._trigger_recorder = False self._start_recorder = False self.recorder = FFMPEGRecorder(config, self.detector.detection_lock, mqtt_queue) self.nvr_list.append({config.camera.mqtt_name: self}) self._logger.debug("NVR thread initialized") def setup_loggers(self, config): self._logger = logging.getLogger(__name__ + "." + config.camera.name_slug) if getattr(config.camera.logging, "level", None): self._logger.setLevel(config.camera.logging.level) self._motion_logger = logging.getLogger(__name__ + "." + config.camera.name_slug + ".motion") if getattr(config.motion_detection.logging, "level", None): self._motion_logger.setLevel(config.motion_detection.logging.level) elif getattr(config.camera.logging, "level", None): self._motion_logger.setLevel(config.camera.logging.level) self._object_logger = logging.getLogger(__name__ + "." + config.camera.name_slug + ".object") if getattr(config.object_detection.logging, "level", None): self._object_logger.setLevel(config.object_detection.logging.level) elif getattr(config.camera.logging, "level", None): self._object_logger.setLevel(config.camera.logging.level) def on_connect(self, client): """Called when MQTT connection is established""" subscriptions = self._mqtt.on_connect(client) self.recorder.on_connect(client) for zone in self._zones: zone.on_connect(client) # We subscribe to the switch topic to toggle camera on/off subscriptions[self._mqtt.devices["switch"].command_topic].append( self.toggle_camera) return subscriptions def toggle_camera(self, message): if message.payload.decode() == "ON": self.start_camera() return if message.payload.decode() == "OFF": self.stop_camera() return def start_camera(self): if not self.camera_grabber or not self.camera_grabber.is_alive(): self._logger.debug("Starting camera") self.camera_grabber = Thread( target=self.camera.capture_pipe, args=( self.config.object_detection.interval, self._object_decoder_queue, self.object_return_queue, self.config.motion_detection.interval, self._motion_decoder_queue, self.motion_return_queue, ), ) self.camera_grabber.daemon = True self.camera_grabber.start() def stop_camera(self): self._logger.debug("Stopping camera") self.camera.release() self.camera_grabber.join() if self.recorder.is_recording: self.recorder.stop_recording() def event_over(self): if self._trigger_recorder or any(zone.trigger_recorder for zone in self._zones): self._motion_max_timeout_reached = False self._motion_only_frames = 0 return False if self.config.motion_detection.timeout and self.motion_detected: # Only allow motion to keep event active for a specified period of time if self._motion_only_frames >= ( self.camera.stream.fps * self.config.motion_detection.max_timeout): if not self._motion_max_timeout_reached: self._motion_max_timeout_reached = True self._logger.debug( "Motion has stalled recorder for longer than max_timeout, " "event considered over anyway") return True self._motion_only_frames += 1 return False return True def start_recording(self, frame): recorder_thread = Thread( target=self.recorder.start_recording, args=(frame, self.objects_in_fov, self.camera.resolution), ) recorder_thread.start() if (self.config.motion_detection.timeout and not self.camera.scan_for_motion.is_set()): self.camera.scan_for_motion.set() self._logger.info("Starting motion detector") def stop_recording(self): if self.idle_frames % self.camera.stream.fps == 0: self._logger.info("Stopping recording in: {}".format( int(self.config.recorder.timeout - (self.idle_frames / self.camera.stream.fps)))) if self.idle_frames >= (self.camera.stream.fps * self.config.recorder.timeout): if not self.config.motion_detection.trigger_detector: self.camera.scan_for_motion.clear() self._logger.info("Pausing motion detector") self.recorder.stop_recording() def get_processed_object_frame(self): """ Returns a frame along with its detections which has been processed by the object detector """ try: return self.object_return_queue.get_nowait()["frame"] except Empty: return None def filter_fov(self, frame): objects_in_fov = [] labels_in_fov = [] self._trigger_recorder = False for obj in frame.objects: if self._object_filters.get(obj.label) and self._object_filters[ obj.label].filter_object(obj): obj.relevant = True objects_in_fov.append(obj) labels_in_fov.append(obj.label) if self._object_filters[obj.label].triggers_recording: self._trigger_recorder = True # Send detection to configured post processors if self._object_filters[obj.label].post_processor: send_to_post_processor( self._logger, self.config, self._post_processors, self._object_filters[obj.label].post_processor, frame, obj, ) self.objects_in_fov = objects_in_fov self.labels_in_fov = labels_in_fov @property def objects_in_fov(self): return self._objects_in_fov @objects_in_fov.setter def objects_in_fov(self, objects): if objects == self._objects_in_fov: return if self._mqtt.mqtt_queue: attributes = {} attributes["objects"] = [obj.formatted for obj in objects] self._mqtt.devices["object_detected"].publish( bool(objects), attributes) self._objects_in_fov = objects @property def labels_in_fov(self): return self._labels_in_fov @labels_in_fov.setter def labels_in_fov(self, labels): self._labels_in_fov, self._reported_label_count = report_labels( labels, self._labels_in_fov, self._reported_label_count, self._mqtt.mqtt_queue, self._mqtt.devices, ) def filter_zones(self, frame): for zone in self._zones: zone.filter_zone(frame) def get_processed_motion_frame(self): """ Returns a frame along with its motion contours which has been processed by the motion detector """ try: return self.motion_return_queue.get_nowait()["frame"] except Empty: return None def filter_motion(self, motion_contours): _motion_found = bool( motion_contours.max_area > self.config.motion_detection.area) if _motion_found: self._motion_frames += 1 self._motion_logger.debug("Consecutive frames with motion: {}, " "max area size: {}".format( self._motion_frames, motion_contours.max_area)) if self._motion_frames >= self.config.motion_detection.frames: if not self.motion_detected: self.motion_detected = True return else: self._motion_frames = 0 if self.motion_detected: self.motion_detected = False @property def motion_detected(self): return self._motion_detected @motion_detected.setter def motion_detected(self, motion_detected): self._motion_detected = motion_detected self._motion_logger.debug( "Motion detected" if motion_detected else "Motion stopped") if self._mqtt.mqtt_queue: self._mqtt.devices["motion_detected"].publish(motion_detected) def process_object_event(self): if self._trigger_recorder or any(zone.trigger_recorder for zone in self._zones): if not self.recorder.is_recording: self._start_recorder = True def process_motion_event(self): if self.motion_detected: if (self.config.motion_detection.trigger_detector and not self.camera.scan_for_objects.is_set()): self.camera.scan_for_objects.set() self._logger.debug("Starting object detector") elif (self.camera.scan_for_objects.is_set() and not self.recorder.is_recording and self.config.motion_detection.trigger_detector): self._logger.debug("Not recording, pausing object detector") self.camera.scan_for_objects.clear() def update_status_sensor(self): status = "unknown" if self.recorder.is_recording: status = "recording" elif self.camera.scan_for_objects.is_set(): status = "scanning_for_objects" elif self.camera.scan_for_motion.is_set(): status = "scanning_for_motion" attributes = {} attributes["last_recording_start"] = self.recorder.last_recording_start attributes["last_recording_end"] = self.recorder.last_recording_end if (status != self._mqtt.status_state or attributes != self._mqtt.status_attributes): self._mqtt.status_attributes = attributes self._mqtt.status_state = status def run(self): """ Main thread. It handles starting/stopping of recordings and publishes to MQTT if object is detected. Speed is determined by FPS""" self._logger.debug("Waiting for first frame") self.camera.frame_ready.wait() self._logger.debug("First frame received") self.idle_frames = 0 while not self.kill_received: self.update_status_sensor() self.camera.frame_ready.wait() # Filter returned objects processed_object_frame = self.get_processed_object_frame() if processed_object_frame: # Filter objects in the FoV self.filter_fov(processed_object_frame) # Filter objects in each zone self.filter_zones(processed_object_frame) if self._object_logger.level == LOG_LEVELS["DEBUG"]: if self.config.object_detection.log_all_objects: objs = [ obj.formatted for obj in processed_object_frame.objects ] self._object_logger.debug(f"All objects: {objs}") else: objs = [obj.formatted for obj in self.objects_in_fov] self._object_logger.debug(f"Objects: {objs}") # Filter returned motion contours processed_motion_frame = self.get_processed_motion_frame() if processed_motion_frame: # self._logger.debug(processed_motion_frame.motion_contours) self.filter_motion(processed_motion_frame.motion_contours) self.process_object_event() self.process_motion_event() if (processed_object_frame or processed_motion_frame ) and self.config.camera.publish_image: self._mqtt.publish_image( processed_object_frame, processed_motion_frame, self._zones, self.camera.resolution, ) # If we are recording and no object is detected if self._start_recorder: self._start_recorder = False self.start_recording(processed_object_frame) elif self.recorder.is_recording and self.event_over(): self.idle_frames += 1 self.stop_recording() continue self.idle_frames = 0 self._logger.info("Exiting NVR thread") def stop(self): self._logger.info("Stopping NVR thread") self.kill_received = True # Stop frame grabber self.camera.release() self.camera_grabber.join() # Stop potential recording if self.recorder.is_recording: self.recorder.stop_recording()