class CaptureProcess(Process): """A process that fills a queue with images as captured from a camera feed""" def __init__(self, capture, imageQueue): Process.__init__(self, name="Capture") self.imageQueue = imageQueue self.capture = capture self.keepGoing = Event() self.keepGoing.set() self.daemon = True def run(self): print "CaptureProcess pid: %s" % (self.pid,) while self.keepGoing.is_set(): image = captureImage(self.capture) # sys.stdout.write(".") try: self.imageQueue.put(serializeImage(image), block=True, timeout=0.25) except FullException: try: _ = self.imageQueue.get_nowait() except: pass # Try to clear the queue, but don't worry if someone snatches it first def stop(self): self.keepGoing.clear()
class BroadcastClient(Process): def __init__(self, port, datagram_size, name="BroadcastClient"): Process.__init__(self, name=name) self.logger = multiprocessing.get_logger() self.event = Event() self.port = port self.datagram_size = datagram_size self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.sock.settimeout(1) self.sock.bind(("", self.port)) def run(self): self.event.set() self.logger.debug("PID: %d" % multiprocessing.current_process().pid) while self.event.is_set(): try: message, (ip, port) = self.sock.recvfrom(self.datagram_size) teacher_discovered.send(sender=self) self.logger.debug("Received: %s from: %s" % (message, ip)) except socket.timeout: self.logger.debug("%s timeout" % multiprocessing.current_process().name) time.sleep(1) def stop(self): self.logger.debug("Client will halt.") self.event.clear() self.sock.close() self.terminate()
def wait_for(self, key, value): d = Manager().dict() d[key] = None v = Manager().Value('s', ' ') e = Event() p_state = Process(target=self.state_refresher, args=(self, d, e)) p_input = Process(target=self.input_waiter, args=(self, v, e)) p_state.start() p_input.start() while v.value != 'exit' and dict(d.items())[key] != value: e.wait() e.clear() self.state = d['state'] self.current = d['current'] self.enemy = d['enemy'] self.battlefield = d['battlefield'] p_state.terminate() p_input.terminate() curses.endwin() p_state.join() p_input.join() return True if dict(d.items())[key] == value else False
class SharedFile(object): def __init__(self, filename): self.filename = filename self.fevent = Event() # self.state = Value('i', 0) self.fevent.set() def write(self, mode, data): # print("Write {}".format(inspect.stack()[1][3])) self.wait_freedom_and_lock() f = open(self.filename, mode) f.write(data) f.close self.unlock() def read(self): # print("Read {}".format(inspect.stack()[1][3])) self.wait_freedom_and_lock() f = open(self.filename, 'r') data = f.read() f.close self.unlock() return data def wait_freedom_and_lock(self): self.fevent.wait() self.fevent.clear() # return def unlock(self): self.fevent.set()
class ClassifierWorkerPool(object): def __init__(self): self.queue = Queue(100) self.workers = [] self.stop = Event() self.stop.clear() self.queue_feeder = QueueFeeder(self.queue, self.stop) row = TrainedClassifiers.objects(name=config.classifier).first() if not row: raise Exception("Classifier %s does not exists" % config.classifier) self.trained_classifier = row.get_classifier() def start(self): self.queue_feeder.start() for i in range(0, config.classifier_pool_size): worker = ClassifierWorker(self.trained_classifier, self.queue, self.stop) worker.start() self.workers.append(worker) def terminate(self): self.stop.set() self.queue_feeder.join() for w in self.workers: w.join()
class TestProxyData(TestData): def setup(self): create_link('dummyX', 'dummy') t_url = 'unix://\0%s' % (uuid.uuid4()) p_url = 'unix://\0%s' % (uuid.uuid4()) self.connect = Event() self.release = Event() target = Process(target=_run_remote_uplink, args=(t_url, self.connect, self.release)) target.daemon = True target.start() self.connect.wait() self.connect.clear() proxy = Process(target=_run_remote_uplink, args=(p_url, self.connect, self.release)) proxy.daemon = True proxy.start() self.connect.wait() self.connect.clear() self.ip = IPRoute(do_connect=False) link, proxy = self.ip.connect(p_url) self.ip.register('bala', proxy) link, host = self.ip.connect(t_url, addr=proxy) service = self.ip.discover(self.ip.default_target, addr=host) self.ip.default_peer = host self.ip.default_dport = service self.dev = self.ip.link_lookup(ifname='dummyX')
class Logger(object): def __init__(self, filename): self.qtag = Queue() self.done = Event() self.tag = None self.filename = filename self.file = None def start(self): self.file = open(self.filename, 'w') print 'Opened',self.filename,'for writing.' def set_tag(self, tag): self.qtag.put(tag) def set_done(self): self.done.set() def log(self, nodeid, msgid, data): if not self.qtag.empty(): self.tag = self.qtag.get() if self.done.is_set(): self.done.clear() return True L = ['%f'%time.time(), '%d'%nodeid, '%d'%msgid] + map(str,data) if self.tag: L.append(self.tag) print >>self.file, ','.join(L) self.file.flush() def close(self): if self.file: self.file.close() print 'File closed.'
class BroadcastServer(Process): def __init__(self, ip, port, message, name="BroadcastServer"): Process.__init__(self, name=name) self.logger = multiprocessing.get_logger() self.event = Event() self.message = message self.ip = ip self.port = port self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.sock.bind(("", 0)) def run(self): self.event.set() self.logger.debug("PID: %d" % multiprocessing.current_process().pid) while self.event.is_set(): self.logger.debug("Sending: %s" % self.message) self.sock.sendto(self.message, (self.ip, self.port)) time.sleep(1) def stop(self): self.logger.debug("Server will halt.") self.event.clear() self.terminate()
def SetupBricks(): allsuccess = False allprocs = [] startvols = False events = {} ev = None for brick in BRICKS_IPADDRS: ev = Event () ev.clear() events[brick] = ev for brick in BRICKS_IPADDRS: mychan,brickchan = Pipe() startvols = False if brick == NFSSERVER_ADDR: startvols = True proc = Process (target=SetupBrick, args=(brick, brickchan, startvols, events,)) allprocs.append ((proc, mychan, brickchan)) if FUSE and NFSCLIENT_ADDR not in BRICKS_IPADDRS: ev = Event () ev.clear() events[NFSCLIENT_ADDR] = ev mychan,brickchan = Pipe() startvols = False proc = Process (target=SetupBrick, args=(NFSCLIENT_ADDR, brickchan, startvols, events,)) allprocs.append ((proc, mychan, brickchan)) allsuccess = RunProcMonitorLoop(allprocs) return allsuccess
class MistProcess(Process): def __init__(self, gpio, sleep=1, name='MistProcess'): Process.__init__(self, name=name) self.logger = multiprocessing.get_logger() self.event = Event() self.name = name self.gpio = gpio self.sleep = sleep self.mist = mraa.Gpio(self.gpio) self.mist.dir(mraa.DIR_OUT) def _mist_on(self): self.logger.debug('Mist on') self.mist.write(1) def _mist_off(self): self.logger.debug('Mist off') if self.mist: self.mist.write(0) def run(self): self.event.set() self.logger.debug('PID: %d' % multiprocessing.current_process().pid) while self.event.is_set(): self._mist_on() time.sleep(self.sleep) def stop(self): self.logger.debug('Process {} will halt.'.format(self.name)) self.event.clear() self._mist_off()
class StoppableProcess(Process): """ Base class for Processes which require the ability to be stopped by a process-safe method call """ def __init__(self): self._should_stop = Event() self._should_stop.clear() super(StoppableProcess, self).__init__() def join(self, timeout=0): """ Joins the current process and forces it to stop after the timeout if necessary :param timeout: Timeout duration in seconds """ self._should_stop.wait(timeout) if not self.should_stop(): self.stop() super(StoppableProcess, self).join(0) def stop(self): self._should_stop.set() def should_stop(self): return self._should_stop.is_set() def __repr__(self): return "<%s(should_stop=%s)>" % ( self.__class__.__name__, self.should_stop())
class fmanager: def __init__(self,data,fn): self.sf = Event() self.sf.clear() self.nproc=cpu_count() self.pipes = [Pipe() for i in xrange(self.nproc)] self.e = [evaluator(self.pipes[i][1],self.sf,data,fn) for i in xrange(self.nproc)] null = [i.start() for i in self.e] return def __del__(self): self.sf.set() null = [i.join() for i in self.e] null = [i.terminate() for i in self.e] return def eval(self,x): nd = len(x) for i in xrange(nd): self.pipes[i % self.nproc][0].send([i, x[i]]) solns = [] while len(solns) < nd: for i in xrange(self.nproc): if self.pipes[i][0].poll(0.005): solns.append(self.pipes[i][0].recv()) solns.sort(key=lambda i: i[0]) return [i[1] for i in solns]
class ChildChecker(threading.Thread): def __init__(self, killEvent): super(ChildChecker, self).__init__() self.killEvent = killEvent self.event = Event() self.process = Process(target=childsPlay, args=(self.event,)) def run(self): self.process.start() while not self.killEvent.is_set(): self.event.wait() print "Child checked, and is done playing" if raw_input("Do again? y/n:") == "y": self.event.clear() self.process = Process(target=endlessChildsPlay, args=(self.event,)) self.process.start() else: self.cleanChild() self.killEvent.set() def join(self): print "Joining child process" # Timeout on 5 seconds self.process.join(5) if self.process.is_alive(): print "Child did not join! Killing.." self.process.terminate() print "Joining ChildChecker thread" super(ChildChecker, self).join() def cleanChild(self): print "Cleaning up the child..."
class QueueTask: def __init__(self): self.queue = JoinableQueue() self.event = Event() atexit.register( self.queue.join ) process = Process(target=self.work) process.daemon = True process.start() def work(self): while True: func, args, wait_for = self.queue.get() for evt in wait_for: evt.wait() func(*args) self.event.set() self.queue.task_done() def enqueue(self, func, args=[], wait_for=[]): self.event.clear() self.queue.put( (func, args, wait_for) ) return self.event
def face_proc(self, child_face_recog: Pipe, e_new_person: Event): """ Parallel process of saving people for face recognition Arguments: child_face_recog {Pipe} -- pipe for communication with parent process, recieve ROI ndarray type of recognized object """ if not os.path.exists('humans'): print('created', os.getcwd()) os.mkdir('humans') else: print('exist') os.chdir(os.path.join(os.getcwd(), 'humans')) is_first = True counter = 0 while True: if e_new_person.is_set(): counter = 0 if not is_first: os.chdir('..') new_dir = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") os.mkdir(new_dir) print('Created', os.getcwd() + new_dir) os.chdir(os.path.join(os.getcwd(), new_dir)) e_new_person.clear() is_first = False image = child_face_recog.recv() cv.imwrite(filename=str(counter) + '.jpg', img=image) print('image saved:', os.getcwd() + str(counter) + '.jpg') counter += 1
class KafkaQueue(object): def __init__(self, client, topic, partitions, producer_config={}, consumer_config={}): """ KafkaQueue a Queue-like object backed by a Kafka producer and some number of consumers Params ====== client: KafkaClient object topic: str, the topic name partitions: list of ints, the partions to consume from producer_config: dict, see below consumer_config: dict, see below Consumer Config =============== consumer_sleep: int, time in milliseconds a consumer should sleep when it reaches the end of a partition. Default is 200 Producer Config =============== producer_timeout: int, time in milliseconds a producer should wait for messages to enqueue for producing. Default is 100 producer_flush_timeout: int, time in milliseconds a producer should allow messages to accumulate before sending to Kafka. Default is 2000 producer_flush_buffer: int, number of messages a producer should allow to accumulate. Default is 500 """ self.in_queue = Queue() self.out_queue = Queue() self.consumers = [] self.barrier = Event() # Initialize and start consumer threads for partition in partitions: consumer = KafkaConsumerProcess(client, topic, partition, self.in_queue, self.barrier, **consumer_config) consumer.start() self.consumers.append(consumer) # Initialize and start producer thread self.producer = KafkaProducerProcess(client, topic, self.out_queue, self.barrier, **producer_config) self.producer.start() # Trigger everything to start self.barrier.set() def get(self, block=True, timeout=None): return self.in_queue.get(block, timeout).payload def put(self, msg, block=True, timeout=None): return self.out_queue.put(msg, block, timeout) def close(self): self.in_queue.close() self.out_queue.close() self.barrier.clear() self.producer.join() for consumer in self.consumers: consumer.join()
def add_poster(self, num=1): if num <= 0: return else: evt_death = Event() evt_death.clear() self._posters.append( Poster(self.name + "_Poster_" + str(num), self.out_dir, self.in_dir, self.q_gpu2s3, evt_death, self.sqs_name,self.in_sqs_q, self.s3bucket_name)) self._reaper.append(evt_death) self._posters[-1].daemon = True self._posters[-1].start() self.add_poster(num - 1)
def add_retriever(self, num=1): if num <= 0: return else: evt_death = Event() evt_death.clear() self._retrievers.append( Retriever(self.name + "_Retriever_" + str(num), self.in_dir, self.q_ret2gpu, evt_death, self.sqs_name, self.s3bucket_name, max_q_size=10*(num+1) ) ) self._reaper.append(evt_death) self._retrievers[-1].daemon = True self._retrievers[-1].start() self.add_retriever(num - 1)
def parent_run(): # Make sure we can connect conn = psycopg2.connect(conn_string) curs = conn.cursor() curs.execute(dummy_statement) curs.fetchall() curs.close() conn.close() # Spawn workers children = [] result_queue = Queue() start_lock = Event() for i in xrange(0, n_workers): done_event = Event() child = Process(target=child_run, args=(i,result_queue,start_lock,done_event)) children.append(child) child.done_event = done_event for child in children: child.start() # and start them working start_lock.set() # wait a while for children to thrash the system # then ask them to finish time.sleep(10) start_lock.clear() # Wait until all children have reported in sys.stderr.write("Waiting for children to complete...") sys.stderr.flush() for child in children: child.done_event.wait() sys.stderr.write(" done\n") # Read the results results = [] for i in range(0, len(children)): (worker_number, num_tx) = result_queue.get(False) results.append(num_tx) total_tx = sum(results) mean_tx = float(total_tx) / float(len(results)) # and wait for children to exit for child in children: child.join() sys.stderr.write("Done, executed %i statements with %i conns and %i workers\n" % (total_tx, n_conns, n_workers)) sys.stderr.write("Mean tx per worker was %.2f\n" % mean_tx)
class EngineManager(ObjectsManager, Process): """Uses header values to handle directed requests and results.""" def __init__(self, *args, **kwargs): super(EngineManager, self).__init__(*args, **kwargs) self.manager = Manager() self.results = self.manager.dict() self.block = Event() def retrieve(self, jid=None, inputs=False, timeout=None, block=True): """Retrieve a job given a jid""" if jid is None: jid = self.cjid if block: self.block.wait() hdr, func, args, kwargs, rvalue = self.results[jid] self.block.clear() if inputs: return func, args, kwargs, rvalue else: return rvalue def ready(self, jid): """Check if a job id is ready.""" return jid in self.results def __getattr__(self, attr): """Call a method on the underlying threaded object""" def method(*args, **kwargs): """A threaded method""" jid = id((attr, args, kwargs)) self.cjid = jid self.hdr["jid"] = jid self.input.put((self.hdr, attr, args, kwargs)) return jid return method def start(self): """Start""" super(EngineManager, self).start() def run(self): """Run the subthread to move things off the output.""" running = True while running: hdr, func, args, kwargs, rvalue = self.output.get() if "stop" in hdr: running = False else: self.results[hdr["jid"]] = (hdr, func, args, kwargs, rvalue) self.block.set()
def main(): # Setup process pooling manager = Manager() # Make queue for processes to grab a thread number from # also used so that the last worker knows it's last. thread_number = manager.Queue() [thread_number.put(t) for t in range(getthreads())] # Set event for handling worker ready condition workers_ready = Event() workers_idle = Event() workers_idle.set() # Set event for handling daemon quit condition for stuff that doesnt exit clean daemon_exit = Event() # Set event for handling new file condition new_file = Event() # Setup workqueue queue = manager.Queue() print("Starting daemons") watcherd = Process(name="watcherd", target=watcher, daemon=True, args=(queue, new_file,)) watcherd.start() print("Daemons started.") print("Initializing workers...") workers = Pool(processes=thread_number.qsize(), initializer=initworker, initargs=(thread_number, workers_ready,)) workers_ready.wait() print("All workers ready") # Set signal handler on main process def exit_handler(signum, frame): cleanup(workers, daemon_exit, workers_idle) signal(SIGINT, exit_handler) signal(SIGTERM, exit_handler) # Main loop while not daemon_exit.is_set(): # wait until new file is on queue, forced check every 5 minutes. status = new_file.wait(timeout=300.0) if status: new_file.clear() workers_idle.clear() # Send to workers workers.map(compressors.dyncompress,[queue.get() for q in range(queue.qsize())]) workers_idle.set()
class DeviceWatcher(object): def __init__(self, port=None, watcher='arduino'): ''' Initialize a new instance of the class ''' self.ipc_queue = Queue() self.halt_event = Event() arguments = (self.halt_event, self.ipc_queue, port, watcher) self.process = Process(target=self.__watcher, args=arguments) self.process.daemon = True def start(self): ''' Start the processor running :returns: The intermessage queue to read from ''' _logger.info("starting the device monitor") self.halt_event.clear() self.process.start() return self.ipc_queue def stop(self): ''' Stop the processor from running ''' _logger.info("stopping the device monitor") self.halt_event.set() self.process.join(5) if self.process.is_alive(): _logger.info("force stopping the device monitor") self.process.terminate() @classmethod def __watcher(cls, halt_event, ipc_queue, port, watcher): ''' The main runner for pulling messages off the serial bus and throwing them into the database ''' if watcher == 'arduino': client = ArduinoWatcher(port=port, baudrate=9600, timeout=5) else: client = TinyosWatcher(port) _logger.info("device monitor initialized") try: for message in client: if halt_event.is_set(): _logger.info("device monitor was told to stop") break # someone told us to stop if message: ipc_queue.put(message) except Exception: _logger.exception("something bad happend in the device monitor") ipc_queue.put(None) # we are broke, force an exit client.close() _logger.info("device monitor exiting")
class conductor(Process): def __init__(self, bus_data_queue, cld_e, park_e, open_once_e): Process.__init__(self) self.park_e = park_e self.closedoor_e = cld_e self.open_once_e = open_once_e self.bus_data_queue = bus_data_queue def run(self): once = 0 while True: self.park_e.wait() self.open_once_e.wait() self.open_once_e.clear() self.opendoor() self.closedoor_e.set() self.closedoor() self.selltickets() def opendoor(self): msg = 'open the door' self.bus_data_queue.put(msg) timeit = random.random() time.sleep(timeit) def closedoor(self): msg = 'close the door' self.bus_data_queue.put(msg) timeit = random.random() time.sleep(timeit) def selltickets(self): msg = 'selling tickets...' self.bus_data_queue.put(msg) timeit = random.random() time.sleep(timeit) self.closedoor_e = Event() self.park_e = Event() self.open_once_e = Event() self.closedoor_e.set() self.park_e.clear() self.open_once_e.set() self.driver1 = driver(self.bus_data_queue, self.closedoor_e, self.park_e, self.open_once_e) self.driver1.start() self.conductor1 = conductor(self.bus_data_queue, self.closedoor_e, self.park_e, self.open_once_e) self.conductor1.start()
def main(): event = Event() queue = Queue() proc_pool = [MyProcess(queue, event, top_func) for _ in range(4)] event.set() for i in range(1000): queue.put(i) for proc in proc_pool: proc.start() while not queue.empty(): time.sleep(1) event.clear() for proc in proc_pool: proc.join()
class IndexerFather(object): def __init__(self, count): Process.__init__(self) self.is_alive = False self.count = count # Indexers count self.alive_event = Event() # Event to alive Indexers self.work_event = Event() # Event to signal Indexers work self.queue = Queue() # Next text to process self.index_queue = [] # List for saving tasks self.indexer_pool = [] def start(self): if not self.alive_event.is_set(): for task in self.index_queue: self.queue.put(task) for i in range(self.count): self.indexer_pool.append(Indexer(self.queue, self.alive_event, self.work_event)) self.alive_event.set() self.work_event.set() self.is_alive = True for indexer in self.indexer_pool: indexer.start() def stop(self): self.work_event.clear() is_stop_working = False while not is_stop_working: for indexer in self.indexer_pool: is_stop_working = not (indexer.is_working and is_stop_working) while not self.queue.empty(): self.index_queue.append(self.queue.get()) self.alive_event.clear() for indexer in self.indexer_pool: indexer.join() self.indexer_pool.clear() # After every join processes must stoped logging.debug("All processes is stoped") return self.index_queue def add_text(self, url, text): self.queue.put((url, text))
def test_worker(self): queue = Queue(100) stop = Event() stop.clear() w = ClassifierWorker(self.cls, queue, stop) w.start() for d in self.data: queue.put(d) while not queue.empty(): time.sleep(0.1) time.sleep(0.5) stop.set() w.join() self.assertEqual(ClassifiedStream.objects.count(), 5)
class Processer(object): """docstring for Worker""" def __init__(self, arg ): super(Processer, self).__init__() self.arg = arg self.be_exit = Event() self.be_exit.clear() def stop_main_loop(self): self.be_exit.set() def main_loop(self): while not self.be_exit.is_set(): try: GetTaskFromTBus( self.arg ) time.sleep(1) except Exception, e: logger.exception( e )
class WeatherProcess(Process): def __init__(self, city, name='WeatherProcess'): Process.__init__(self, name=name) self.logger = multiprocessing.get_logger() self.event = Event() self.name = name self.city = city def run(self): self.event.set() self.logger.debug('PID: %d' % multiprocessing.current_process().pid) while self.event.is_set(): time.sleep(1) def stop(self): self.logger.debug('Process {} will halt.'.format(self.name)) self.event.clear()
def make_server_manager(self, port, ip, authkey): """ Create a manager for the server, listening on the given port. Return a manager object with get_job_q and get_result_q methods. """ try: # Make_server_manager definition try block job_q = Queue(maxsize=100) result_q = Queue() shutdown = Event() shutdown.clear() mode_bit_1 = Event() mode_bit_2 = Event() try: # JobQueueManager/Lambda functions Try Block self.JobQueueManager.register('get_job_q', callable=lambda: job_q) self.JobQueueManager.register('get_result_q', callable=lambda: result_q) self.JobQueueManager.register('get_shutdown', callable=lambda: shutdown) self.JobQueueManager.register('get_mode_bit_1', callable=lambda: mode_bit_1) self.JobQueueManager.register('get_mode_bit_2', callable=lambda: mode_bit_2) except Exception as inst: print "============================================================================================" print "ERROR: An exception was thrown in Make_server_Manager: " \ + "JobQueueManager/Lambda functions Try Block" #the exception instance print type(inst) #arguments stored in .args print inst.args #_str_ allows args tto be printed directly print inst print "============================================================================================" manager = self.JobQueueManager(address=(ip, port), authkey=authkey) manager.start() print 'Server started at port %s' % port return manager except Exception as inst: print "=============================================================================================" print "ERROR: An exception was thrown in Make_server_manager definition Try block" #the exception instance print type(inst) #arguments stored in .args print inst.args #_str_ allows args tto be printed directly print inst print "============================================================================================="
class ConsumerProxy(Process): """ Consumer Proxy Consumer Proxy act as a broker for consumer in Producer-Consumer Pattern. It deals with the process model for the real consumer. """ def __init__(self, queue, consumer): Process.__init__(self) self._queue = queue self._consumer = consumer self._stop = Event() self._stop.clear() def consume(self, task): self._consumer.consume(task) def close(self): self._consumer.close() def stop(self): self._stop.set() def run(self): signal.signal(signal.SIGINT, signal.SIG_IGN) while True: if self._stop.is_set(): self.close() break try: task = self._queue.get(timeout=BLOCK_TIMEOUT) except Queue.Empty: continue try: self.consume(task) except Exception: traceback.print_exc() finally: self._queue.task_done()
class RegistryServer(Process): __KEY = b'NETAPPapiS3CR3Tk8' __PORT = int(os.environ.get('NETAPP_API_CR_PORT', '12342')) __HOST = 'localhost' def __init__(self): super(RegistryServer, self).__init__() self.notify = Event() def wait_complete(self): while not self.notify.is_set(): self.notify.wait(1) self.notify.clear() @staticmethod def GET_CLIENT_REGISTRY(): RegistryManager.register('get_datadict') RegistryManager.register('get_datalock') RegistryManager.register('get_sd_event') manager = RegistryManager(address=(RegistryServer.__HOST, RegistryServer.__PORT), authkey=RegistryServer.__KEY) manager.connect() return ClientRegistry(manager.get_datadict(), manager.get_datalock(), manager.get_sd_event()) def run(self): LOGGER.debug('starting up registry server') data_dict = { 'apistatus': 'TBD', # the status of the NetApp api 'grpc': { 'accessmsg': 'TBD' # the string to return for connection }, 'clientsessionstatus': { # for each client --> ID : status .. [RUNNING, SHUTDOWN] } } data_lock = Lock() sd_event = Event() RegistryManager.register('get_datadict', callable=lambda: data_dict, proxytype=mp_Managers.DictProxy) RegistryManager.register('get_datalock', callable=lambda: data_lock, proxytype=mp_Managers.AcquirerProxy) RegistryManager.register('get_sd_event', callable=lambda: sd_event, proxytype=mp_Managers.EventProxy) manager = RegistryManager(address=(RegistryServer.__HOST, RegistryServer.__PORT), authkey=RegistryServer.__KEY) LOGGER.debug('registry server created') manager.start() LOGGER.debug('registry server started @ %s:%s', RegistryServer.__HOST, RegistryServer.__PORT) # indicate to outside world that start is complete self.notify.set() # sleep until shutdown is issued while not sd_event.is_set(): sd_event.wait(1) LOGGER.debug('registry server shutting down') # shutdown the manager manager.shutdown() manager.join() LOGGER.debug('registry server exiting') # indicate to outside world that stop is complete self.notify.set()
class Publisher: """ Publisher class in which the queue for publishing messages is defined and also a separated process is started. It is important to have a new process, since the serialization/deserialization of messages from the QUEUE may be a bottleneck for performance. """ def __init__(self, port=5555): self.logger = get_logger(name=__name__) self._port = port self._queue = Queue( ) # The publisher will grab and broadcast the messages from this queue self._event = Event() # This event is used to stop the process self._process = Process(target=publisher, args=[self._queue, self._event, self._port]) self.logger.info('Initialized published on port {}'.format(port)) def start(self): """ Start a new process that will be responsible for broadcasting the messages. """ self._event.clear() self._process.start() sleep(1) # This forces the start to block until the publisher is ready def stop(self): self._event.set() self.empty_queue() def empty_queue(self): """ If the publisher stops before broadcasting all the messages, the Queue may still be using some memory. This method is simply getting all the elements in order to free memory. Can be useful for garbage collection or better control of the downstream program. """ self.logger.info('Emptying the queue of the publisher') self.logger.debug('Queue length: {}'.format(self._queue.qsize())) self._queue.close() def publish(self, topic, data): """ Adapts the data to make it faster to broadcast :param str topic: Topic in which to publish the data :param data: Data to be published :return: None """ self.logger.debug('Adding data of type {} to topic {}'.format( type(data), topic)) try: self._queue.put({'topic': topic, 'data': data}) except AssertionError: # This means the queue has been closed already pass @property def port(self): return self._port @port.setter def port(self, new_port): if new_port != self._port: self._port = new_port self.logger.warning( 'Changing the port requires restarting the publisher process') self.logger.debug( 'Setting the new publisher port to {}'.format(new_port)) self.stop() self._process.join() self._process = Process( target=publisher, args=[self._queue, self._event, self._port]) self.start() else: self.logger.warning( 'New port {} is the same as the old port'.format(new_port)) def join(self, timeout=0): if self._process.is_alive(): self.logger.debug('Waiting for Publisher process to finish') self._process.join(timeout)
def main(q): v = 0 t = time() while v != (size * 3): try: if time() - t > 0.4: print 'Queue.qsize:', q.qsize() t = time() q.put('test' + str(v)) v += 1 except Full: pass s = time() p1 = Process(target = worker, args=(q, run)) p1.start() p2 = Process(target = worker, args=(q, run)) p2.start() main(q) qs = q.qsize() while qs > 0: print 'Queue.qsize:', qs sleep(0.4) qs = q.qsize() run.clear() p1.join() p2.join() print 'nox.shm.Queue put/get: %s' %(time() - s)
class KafkaQueue(object): def __init__(self, client, topic, partitions, producer_config=None, consumer_config=None): """ KafkaQueue a Queue-like object backed by a Kafka producer and some number of consumers Messages are eagerly loaded by the consumer in batches of size consumer_fetch_size. Messages are buffered in the producer thread until producer_flush_timeout or producer_flush_buffer is reached. Params ====== client: KafkaClient object topic: str, the topic name partitions: list of ints, the partions to consume from producer_config: dict, see below consumer_config: dict, see below Consumer Config =============== consumer_fetch_size: int, number of bytes to fetch in one call to Kafka. Default is 1024 consumer_sleep: int, time in milliseconds a consumer should sleep when it reaches the end of a partition. Default is 200 Producer Config =============== producer_timeout: int, time in milliseconds a producer should wait for messages to enqueue for producing. Default is 100 producer_flush_timeout: int, time in milliseconds a producer should allow messages to accumulate before sending to Kafka. Default is 2000 producer_flush_buffer: int, number of messages a producer should allow to accumulate. Default is 500 """ producer_config = {} if producer_config is None else producer_config consumer_config = {} if consumer_config is None else consumer_config self.in_queue = Queue() self.out_queue = Queue() self.consumers = [] self.barrier = Event() # Initialize and start consumer threads for partition in partitions: consumer = KafkaConsumerProcess(client, topic, partition, self.in_queue, self.barrier, **consumer_config) consumer.start() self.consumers.append(consumer) # Initialize and start producer thread self.producer = KafkaProducerProcess(client, topic, self.out_queue, self.barrier, **producer_config) self.producer.start() # Trigger everything to start self.barrier.set() def get(self, block=True, timeout=None): """ Consume a message from Kafka Params ====== block: boolean, default True timeout: int, number of seconds to wait when blocking, default None Returns ======= msg: str, the payload from Kafka """ return self.in_queue.get(block, timeout).payload def put(self, msg, block=True, timeout=None): """ Send a message to Kafka Params ====== msg: std, the message to send block: boolean, default True timeout: int, number of seconds to wait when blocking, default None """ self.out_queue.put(msg, block, timeout) def close(self): """ Close the internal queues and Kafka consumers/producer """ self.in_queue.close() self.out_queue.close() self.barrier.clear() self.producer.join() for consumer in self.consumers: consumer.join()
log("{} timed out.".format(name)) to_remove.append(i) avail_jobs.append(job_index) elif not proc.is_alive(): to_remove.append(i) proc.join() avail_jobs.append(job_index) # Reverse index order so as to pop elements correctly for i in reversed(to_remove): workers.pop(i) time.sleep(0.01) # Conclude and write results to file. run_signal.clear() flush_proc.join(1) # it's important to count the total after proc.join total = len(res_list) log(f"\nFinished {total} contracts...\n") vulnerability_counts = defaultdict(int) analytics_sums = defaultdict(int) meta_counts = defaultdict(int) all_files = set() for contract, files, meta, analytics in res_list: for f in files: all_files.add(f) for m in meta: meta_counts[m] += 1 for k, a in analytics.items():
class ClockBaseInterruptBehavior(ClockBaseBehavior): '''A kivy clock which can be interrupted during a frame to execute events. ''' interupt_next_only = False _event = None _get_min_timeout_func = None def __init__(self, interupt_next_only=False, **kwargs): super(ClockBaseInterruptBehavior, self).__init__(**kwargs) self._event = MultiprocessingEvent() if PY2 else ThreadingEvent() self.interupt_next_only = interupt_next_only self._get_min_timeout_func = self.get_min_timeout def usleep(self, microseconds): self._event.clear() self._event.wait(microseconds / 1000000.) def on_schedule(self, event): fps = self._max_fps if not fps: return if not event.timeout or ( not self.interupt_next_only and event.timeout <= 1 / fps - # remaining time (self.time() - self._last_tick) + # elapsed time 4 / 5. * self.get_resolution()): # resolution fudge factor self._event.set() def idle(self): fps = self._max_fps event = self._event resolution = self.get_resolution() if fps > 0: done, sleeptime = self._check_ready(fps, resolution, 4 / 5. * resolution) if not done: event.wait(sleeptime) current = self.time() self._dt = current - self._last_tick self._last_tick = current event.clear() # anything scheduled from now on, if scheduled for the upcoming frame # will cause a timeout of the event on the next idle due to on_schedule # `self._last_tick = current` must happen before clear, otherwise the # on_schedule computation is wrong when exec between the clear and # the `self._last_tick = current` bytecode. return current def _check_ready(self, fps, min_sleep, undershoot): if self._event.is_set(): return True, 0 t = self._get_min_timeout_func() if not t: return True, 0 if not self.interupt_next_only: curr_t = self.time() sleeptime = min(1 / fps - (curr_t - self._last_tick), t - curr_t) else: sleeptime = 1 / fps - (self.time() - self._last_tick) return sleeptime - undershoot <= min_sleep, sleeptime - undershoot
class GenericCam(Process): def __init__(self, outQ=None, recorderpar=None, refreshperiod=1 / 20.): super(GenericCam, self).__init__() self.name = '' self.cam_id = None self.h = None self.w = None self.nchan = 1 self.close_event = Event() self.start_trigger = Event() self.stop_trigger = Event() self.saving = Event() self.nframes = Value('i', 0) self.queue = outQ self.camera_ready = Event() self.eventsQ = Queue() self._init_controls() self._init_ctrevents() self.cam_is_running = False self.was_saving = False self.recorderpar = recorderpar self.recorder = None self.refresh_period = refreshperiod self._tupdate = time.time() self.daemon = True #self.memmanager = Manager() #self.memlist = self.memmanager.list() #self.memlist.append(None) self.lasttime = 0 def get_img(self): return self.img #self.memlist[0] def stop_saving(self): # This will send a stop to stop saving and close the writer. #if self.saving.is_set(): self.saving.clear() def _init_controls(self): return def _init_ctrevents(self): if hasattr(self, 'ctrevents'): for c in self.ctrevents.keys(): self.ctrevents[c][ 'call'] = 'self.' + self.ctrevents[c]['function'] def _init_variables(self, dtype=np.uint8): if dtype == np.uint8: cdtype = ctypes.c_ubyte else: cdtype = ctypes.c_ushort self.frame = Array( cdtype, np.zeros([self.h, self.w, self.nchan], dtype=dtype).flatten()) self.img = np.frombuffer(self.frame.get_obj(), dtype=cdtype).reshape( [self.h, self.w, self.nchan]) def _start_recorder(self): if not self.recorderpar is None: extrapar = {} if 'binary' in self.recorderpar['recorder'].lower(): from .io import BinaryCamWriter as rec elif 'tiff' in self.recorderpar['recorder'].lower(): from .io import TiffCamWriter as rec elif 'ffmpeg' in self.recorderpar['recorder'].lower(): from .io import FFMPEGCamWriter as rec if 'hwaccel' in self.recorderpar: if 'hwaccel' in self.recorderpar.keys(): extrapar['hwaccel'] = self.recorderpar['hwaccel'] if 'compression' in self.recorderpar.keys(): extrapar['compression'] = self.recorderpar[ 'compression'] else: display('Recorder {0} not implemented'.format( self.recorderpar['recorder'])) if 'rec' in dir(): self.recorder = rec( self, inQ=self.queue, filename=self.recorderpar['filename'], pathformat=self.recorderpar['pathformat'], dataname=self.recorderpar['dataname'], datafolder=self.recorderpar['datafolder'], framesperfile=self.recorderpar['framesperfile'], incrementruns=True, **extrapar) def run(self): self._init_ctrevents() self.img = np.frombuffer(self.frame.get_obj(), dtype=self.dtype).reshape( [self.h, self.w, self.nchan]) self.close_event.clear() self._start_recorder() while not self.close_event.is_set(): self._cam_init() if self.stop_trigger.is_set(): break self._cam_waitsoftwaretrigger() if not self.stop_trigger.is_set(): self._cam_startacquisition() self.cam_is_running = True while not self.stop_trigger.is_set(): frame, metadata = self._cam_loop() self._handle_frame(frame, metadata) self._parse_command_queue() # to be able to pause acquisition on software trigger if not self.start_trigger.is_set( ) and not self.stop_trigger.is_set(): self._cam_stopacquisition() self._cam_waitsoftwaretrigger() if not self.stop_trigger.is_set(): self._cam_startacquisition() self.cam_is_running = True display('[Camera] Stop trigger set.') self.start_trigger.clear() self._cam_close() self.cam_is_running = False if self.was_saving: self.was_saving = False if self.recorder is None: display('[Camera] Sending stop signal to the recorder.') self.queue.put(['STOP']) else: self.recorder.close_run() self.stop_trigger.clear() if self.close_event.is_set(): break def _handle_frame(self, frame, metadata): #display('loop rate : {0}'.format(1./(timestamp - self.lasttime))) if self.saving.is_set(): self.was_saving = True if not frame is None: if not metadata[0] == self.lastframeid: if not self.recorder is None: self.recorder.save(frame, metadata) else: self.queue.put((frame, metadata)) elif self.was_saving: if self.recorder is None: self.was_saving = False display('[Camera] Sending stop signal to the recorder.') self.queue.put(['STOP']) else: self.was_saving = False self.recorder.close_run() if not frame is None: frameID, timestamp = metadata[:2] if not frameID == self.lastframeid: t = time.time() if (t - self._tupdate) > self.refresh_period: self._update_buffer(frame, frameID) self._tupdate = t #self.nframes.value += 1 self.lastframeid = frameID self.lasttime = timestamp def _update_buffer(self, frame, frameID): self.img[:] = np.reshape(frame, self.img.shape)[:] #self.memlist[0] = np.reshape(frame,[self.h,self.w,self.nchan]) def _parse_command_queue(self): if not self.eventsQ.empty(): cmd = self.eventsQ.get() if '=' in cmd: cmd = cmd.split('=') if hasattr(self, 'ctrevents'): self._call_event(cmd[0], cmd[1]) if cmd[0] == 'filename': if not self.recorder is None: if hasattr(self, 'recorder'): self.recorder.set_filename(cmd[1]) self.recorderpar['filename'] = cmd[1] elif cmd[0] == 'log': msg = '# {0},{1} - {2}'.format(self.lastframeid, self.lasttime, cmd[1]) if self.recorder is None: self.queue.put([msg]) else: self.recorder.logfile.write(msg) def _call_event(self, eventname, eventvalue): if eventname in self.ctrevents.keys(): val = eval(self.ctrevents[eventname]['type'] + '(' + str(eventvalue) + ')') eval(self.ctrevents[eventname]['call'] + '(val)') #print(self.ctrevents[eventname]) # else: # display('No event found {0} {1}'.format(eventname,eventvalue)) def _cam_init(self): '''initialize the camera''' pass def _cam_startacquisition(self): '''start camera acq''' pass def _cam_stopacquisition(self): '''stop camera acq''' pass def _cam_close(self): '''close cam - release driver''' pass def _cam_loop(self): '''get a frame and move on, returns frame,(frameID,timestamp)''' pass def _cam_waitsoftwaretrigger(self): '''wait for software trigger''' display('[{0} {1}] waiting for software trigger.'.format( self.drivername, self.cam_id)) while not self.start_trigger.is_set() or self.stop_trigger.is_set(): # limits resolution to 1 ms time.sleep(0.001) if self.close_event.is_set() or self.stop_trigger.is_set(): break self._handle_frame( None, None) # to stop saving while waiting for triggers if self.close_event.is_set() or self.stop_trigger.is_set(): return self.camera_ready.clear() def stop_acquisition(self): self.stop_trigger.set() def close(self): self.close_event.set() self.stop_acquisition()
raise AvExpt( 0, 'movie_info', ) if (avnum_queue.empty() is True) and (proc_event.is_set()): break if __name__ == '__main__': parse_genre_proc = [] log_expt_proc = [] genre_proc_queue = Queue() avnum_proc_queue = Queue() expt_proc_queue = Queue() parse_genre_task_done = Event() parse_genre_task_done.clear() genres = database.get_genre_id() for genre_item in genres: genre_proc_queue.put(genre_item[0]) freeze_support() for i in range(GENRE_PROC_NUM): proc_name = 'Genre process ' + str(i + 1) parse_genre_proc.append( Process(name=proc_name, target=parse_genre, args=(genre_proc_queue, avnum_proc_queue, expt_proc_queue, i, parse_genre_task_done))) log_movie_proc = Process(name='log_movie', target=log_movie_info, args=(avnum_proc_queue, expt_proc_queue, parse_genre_task_done, 'log_movie'))
if 'WifiOnOff' in actions.keys(): total_actions += 1 if priority is None: wifi_bt_sync_event.set() else: priority = 'toggle' p1 = Process(name="WifiOnOff", target=toggle_wifi, args=(serial, wifi_bt_sync_event), kwargs={'iteration': actions["WifiOnOff"], 'q': result_q}) # p1.start() process.append(p1) if "A2dpBtConnect" in actions.keys(): total_actions += 1 priority = "a2dp" wifi_bt_sync_event.clear() p1 = Process(name="A2dpConnect", target=pair_a2dp_device, args=(serial, bt_a2dp_name, wifi_bt_sync_event), kwargs={'q': result_q}) process.append(p1) # if 'ConnectToAp' in actions.keys(): # total_actions += 1 # p1 = Process(name="ConnectAp", target=connectToAp, args=(serial, ), # kwargs={'iteration': actions["ConnectToAp"], 'q': result_q}) # #p1.start() # process.append(p1) if multiple_iteration_count > 1: log.error("Invalid iteration count as only one operation can iterate " "multiple times")
class Artifacts(object): max_preview_size_bytes = 65536 _flush_frequency_sec = 300. # notice these two should match _save_format = '.csv.gz' _compression = 'gzip' # hashing constants _hash_block_size = 65536 _pd_artifact_type = 'data-audit-table' class _ProxyDictWrite(dict): """ Dictionary wrapper that updates an arguments instance on any item set in the dictionary """ def __init__(self, artifacts_manager, *args, **kwargs): super(Artifacts._ProxyDictWrite, self).__init__(*args, **kwargs) self._artifacts_manager = artifacts_manager # list of artifacts we should not upload (by name & weak-reference) self.artifact_metadata = {} # list of hash columns to calculate uniqueness for the artifacts self.artifact_hash_columns = {} def __setitem__(self, key, value): # check that value is of type pandas if pd and isinstance(value, pd.DataFrame): super(Artifacts._ProxyDictWrite, self).__setitem__(key, value) if self._artifacts_manager: self._artifacts_manager.flush() else: raise ValueError( 'Artifacts currently support pandas.DataFrame objects only' ) def unregister_artifact(self, name): self.artifact_metadata.pop(name, None) self.pop(name, None) def add_metadata(self, name, metadata): self.artifact_metadata[name] = deepcopy(metadata) def get_metadata(self, name): return self.artifact_metadata.get(name) def add_hash_columns(self, artifact_name, hash_columns): self.artifact_hash_columns[artifact_name] = hash_columns def get_hash_columns(self, artifact_name): return self.artifact_hash_columns.get(artifact_name) @property def registered_artifacts(self): # type: () -> Dict[str, Artifact] return self._artifacts_container @property def summary(self): # type: () -> str return self._summary def __init__(self, task): self._task = task # notice the double link, this is important since the Artifact # dictionary needs to signal the Artifacts base on changes self._artifacts_container = self._ProxyDictWrite(self) self._last_artifacts_upload = {} self._unregister_request = set() self._thread = None self._flush_event = Event() self._exit_flag = False self._summary = '' self._temp_folder = [] self._task_artifact_list = [] self._task_edit_lock = RLock() self._storage_prefix = None def register_artifact(self, name, artifact, metadata=None, uniqueness_columns=True): # type: (str, DataFrame, Optional[dict], Union[bool, Sequence[str]]) -> () """ :param str name: name of the artifacts. Notice! it will override previous artifacts if name already exists. :param pandas.DataFrame artifact: artifact object, supported artifacts object types: pandas.DataFrame :param dict metadata: dictionary of key value to store with the artifact (visible in the UI) :param list uniqueness_columns: list of columns for artifact uniqueness comparison criteria. The default value is True, which equals to all the columns (same as artifact.columns). """ # currently we support pandas.DataFrame (which we will upload as csv.gz) if name in self._artifacts_container: LoggerRoot.get_base_logger().info( 'Register artifact, overwriting existing artifact \"{}\"'. format(name)) self._artifacts_container.add_hash_columns( name, list(artifact.columns if uniqueness_columns is True else uniqueness_columns)) self._artifacts_container[name] = artifact if metadata: self._artifacts_container.add_metadata(name, metadata) def unregister_artifact(self, name): # type: (str) -> () # Remove artifact from the watch list self._unregister_request.add(name) self.flush() def upload_artifact(self, name, artifact_object=None, metadata=None, preview=None, delete_after_upload=False, auto_pickle=True, wait_on_upload=False): # type: (str, Optional[object], Optional[dict], Optional[str], bool, bool, bool) -> bool if not Session.check_min_api_version('2.3'): LoggerRoot.get_base_logger().warning( 'Artifacts not supported by your ClearML-server version, ' 'please upgrade to the latest server version') return False if name in self._artifacts_container: raise ValueError( "Artifact by the name of {} is already registered, use register_artifact" .format(name)) # cast preview to string if preview is not None and not (isinstance(preview, bool) and preview is False): preview = str(preview) # evaluate lazy proxy object if isinstance(artifact_object, LazyEvalWrapper): # noinspection PyProtectedMember artifact_object = LazyEvalWrapper._load_object(artifact_object) pathlib_types = ( Path, pathlib_Path, ) if pathlib_Path is not None else (Path, ) local_filename = None # try to convert string Path object (it might reference a file/folder) # dont not try to serialize long texts. if isinstance(artifact_object, six.string_types ) and artifact_object and len(artifact_object) < 2048: # noinspection PyBroadException try: artifact_path = Path(artifact_object) if artifact_path.exists(): artifact_object = artifact_path elif '*' in artifact_object or '?' in artifact_object: # hackish, detect wildcard in tr files folder = Path('').joinpath(*artifact_path.parts[:-1]) if folder.is_dir() and folder.parts: wildcard = artifact_path.parts[-1] if list(Path(folder).rglob(wildcard)): artifact_object = artifact_path except Exception: pass store_as_pickle = False artifact_type_data = tasks.ArtifactTypeData() artifact_type_data.preview = '' override_filename_in_uri = None override_filename_ext_in_uri = None uri = None if np and isinstance(artifact_object, np.ndarray): artifact_type = 'numpy' artifact_type_data.content_type = 'application/numpy' artifact_type_data.preview = preview or str( artifact_object.__repr__()) override_filename_ext_in_uri = '.npz' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) np.savez_compressed(local_filename, **{name: artifact_object}) delete_after_upload = True elif pd and isinstance(artifact_object, pd.DataFrame): artifact_type = 'pandas' artifact_type_data.content_type = 'text/csv' artifact_type_data.preview = preview or str( artifact_object.__repr__()) override_filename_ext_in_uri = self._save_format override_filename_in_uri = name fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) artifact_object.to_csv(local_filename, compression=self._compression) delete_after_upload = True elif isinstance(artifact_object, Image.Image): artifact_type = 'image' artifact_type_data.content_type = 'image/png' desc = str(artifact_object.__repr__()) artifact_type_data.preview = preview or desc[1:desc.find(' at ')] override_filename_ext_in_uri = '.png' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) artifact_object.save(local_filename) delete_after_upload = True elif isinstance(artifact_object, dict): artifact_type = 'JSON' artifact_type_data.content_type = 'application/json' # noinspection PyBroadException try: json_text = json.dumps(artifact_object, sort_keys=True, indent=4) except Exception: if not auto_pickle: raise LoggerRoot.get_base_logger().warning( "JSON serialization of artifact \'{}\' failed, reverting to pickle" .format(name)) store_as_pickle = True json_text = None if json_text is not None: override_filename_ext_in_uri = '.json' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp( prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.write(fd, bytes(json_text.encode())) os.close(fd) preview = preview or json_text if len(preview) < self.max_preview_size_bytes: artifact_type_data.preview = preview else: artifact_type_data.preview = '# full json too large to store, storing first {}kb\n{}'.format( self.max_preview_size_bytes // 1024, preview[:self.max_preview_size_bytes]) delete_after_upload = True elif isinstance(artifact_object, pathlib_types): # check if single file artifact_object = Path(artifact_object) artifact_object.expanduser().absolute() # noinspection PyBroadException try: create_zip_file = not artifact_object.is_file() except Exception: # Hack for windows pathlib2 bug, is_file isn't valid. create_zip_file = True else: # We assume that this is not Windows os if artifact_object.is_dir(): # change to wildcard artifact_object /= '*' if create_zip_file: folder = Path('').joinpath(*artifact_object.parts[:-1]) if not folder.is_dir() or not folder.parts: raise ValueError( "Artifact file/folder '{}' could not be found".format( artifact_object.as_posix())) wildcard = artifact_object.parts[-1] files = list(Path(folder).rglob(wildcard)) override_filename_ext_in_uri = '.zip' override_filename_in_uri = folder.parts[ -1] + override_filename_ext_in_uri fd, zip_file = mkstemp( prefix=quote(folder.parts[-1], safe="") + '.', suffix=override_filename_ext_in_uri) try: artifact_type_data.content_type = 'application/zip' archive_preview = 'Archive content {}:\n'.format( artifact_object.as_posix()) with ZipFile(zip_file, 'w', allowZip64=True, compression=ZIP_DEFLATED) as zf: for filename in sorted(files): if filename.is_file(): relative_file_name = filename.relative_to( folder).as_posix() archive_preview += '{} - {}\n'.format( relative_file_name, format_size(filename.stat().st_size)) zf.write(filename.as_posix(), arcname=relative_file_name) except Exception as e: # failed uploading folder: LoggerRoot.get_base_logger().warning( 'Exception {}\nFailed zipping artifact folder {}'. format(folder, e)) return False finally: os.close(fd) artifact_type_data.preview = preview or archive_preview artifact_object = zip_file artifact_type = 'archive' artifact_type_data.content_type = mimetypes.guess_type( artifact_object)[0] local_filename = artifact_object delete_after_upload = True else: if not artifact_object.is_file(): raise ValueError( "Artifact file '{}' could not be found".format( artifact_object.as_posix())) override_filename_in_uri = artifact_object.parts[-1] artifact_type_data.preview = preview or '{} - {}\n'.format( artifact_object, format_size( artifact_object.stat().st_size)) artifact_object = artifact_object.as_posix() artifact_type = 'custom' artifact_type_data.content_type = mimetypes.guess_type( artifact_object)[0] local_filename = artifact_object elif isinstance(artifact_object, (list, tuple)) and all( isinstance(p, pathlib_types) for p in artifact_object): # find common path if exists list_files = [Path(p) for p in artifact_object] override_filename_ext_in_uri = '.zip' override_filename_in_uri = quote( name, safe="") + override_filename_ext_in_uri common_path = get_common_path(list_files) fd, zip_file = mkstemp(prefix='artifact_folder.', suffix=override_filename_ext_in_uri) try: artifact_type_data.content_type = 'application/zip' archive_preview = 'Archive content:\n' with ZipFile(zip_file, 'w', allowZip64=True, compression=ZIP_DEFLATED) as zf: for filename in sorted(list_files): if filename.is_file(): relative_file_name = filename.relative_to(Path(common_path)).as_posix() \ if common_path else filename.as_posix() archive_preview += '{} - {}\n'.format( relative_file_name, format_size(filename.stat().st_size)) zf.write(filename.as_posix(), arcname=relative_file_name) else: LoggerRoot.get_base_logger().warning( "Failed zipping artifact file '{}', file not found!" .format(filename.as_posix())) except Exception as e: # failed uploading folder: LoggerRoot.get_base_logger().warning( 'Exception {}\nFailed zipping artifact files {}'.format( artifact_object, e)) return False finally: os.close(fd) artifact_type_data.preview = preview or archive_preview artifact_object = zip_file artifact_type = 'archive' artifact_type_data.content_type = mimetypes.guess_type( artifact_object)[0] local_filename = artifact_object delete_after_upload = True elif (isinstance(artifact_object, six.string_types) and len(artifact_object) < 4096 and urlparse(artifact_object).scheme in remote_driver_schemes): # we should not upload this, just register local_filename = None uri = artifact_object artifact_type = 'custom' artifact_type_data.content_type = mimetypes.guess_type( artifact_object)[0] if preview: artifact_type_data.preview = preview elif isinstance(artifact_object, six.string_types): # if we got here, we should store it as text file. artifact_type = 'string' artifact_type_data.content_type = 'text/plain' if preview: artifact_type_data.preview = preview elif len(artifact_object) < self.max_preview_size_bytes: artifact_type_data.preview = artifact_object else: artifact_type_data.preview = '# full text too large to store, storing first {}kb\n{}'.format( self.max_preview_size_bytes // 1024, artifact_object[:self.max_preview_size_bytes]) if artifact_object: delete_after_upload = True override_filename_ext_in_uri = '.txt' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp( prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) # noinspection PyBroadException try: with open(local_filename, 'wt') as f: f.write(artifact_object) except Exception: # cleanup and raise exception os.unlink(local_filename) raise elif artifact_object is None or (isinstance(artifact_object, str) and artifact_object == ""): artifact_type = '' store_as_pickle = False elif auto_pickle: # revert to pickling the object store_as_pickle = True else: raise ValueError("Artifact type {} not supported".format( type(artifact_object))) # revert to serializing the object with pickle if store_as_pickle: # if we are here it means we do not know what to do with the object, so we serialize it with pickle. artifact_type = 'pickle' artifact_type_data.content_type = 'application/pickle' # noinspection PyBroadException try: artifact_type_data.preview = preview or str( artifact_object.__repr__())[:self.max_preview_size_bytes] except Exception: artifact_type_data.preview = preview or '' delete_after_upload = True override_filename_ext_in_uri = '.pkl' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) # noinspection PyBroadException try: with open(local_filename, 'wb') as f: pickle.dump(artifact_object, f) except Exception: # cleanup and raise exception os.unlink(local_filename) raise # verify preview not out of scope: if artifact_type_data.preview and len(artifact_type_data.preview) > ( self.max_preview_size_bytes + 1024): artifact_type_data.preview = '# full preview too large to store, storing first {}kb\n{}'.format( self.max_preview_size_bytes // 1024, artifact_type_data.preview[:self.max_preview_size_bytes]) # remove from existing list, if exists for artifact in self._task_artifact_list: if artifact.key == name: if artifact.type == self._pd_artifact_type: raise ValueError( "Artifact of name {} already registered, " "use register_artifact instead".format(name)) self._task_artifact_list.remove(artifact) break if not local_filename: file_size = None file_hash = None else: # check that the file to upload exists local_filename = Path(local_filename).absolute() if not local_filename.exists() or not local_filename.is_file(): LoggerRoot.get_base_logger().warning( 'Artifact upload failed, cannot find file {}'.format( local_filename.as_posix())) return False file_hash, _ = sha256sum(local_filename.as_posix(), block_size=Artifacts._hash_block_size) file_size = local_filename.stat().st_size uri = self._upload_local_file( local_filename, name, delete_after_upload=delete_after_upload, override_filename=override_filename_in_uri, override_filename_ext=override_filename_ext_in_uri, wait_on_upload=wait_on_upload) timestamp = int(time()) artifact = tasks.Artifact( key=name, type=artifact_type, uri=uri, content_size=file_size, hash=file_hash, timestamp=timestamp, type_data=artifact_type_data, display_data=[(str(k), str(v)) for k, v in metadata.items()] if metadata else None) # update task artifacts self._add_artifact(artifact) return True def flush(self): # type: () -> () # start the thread if it hasn't already: self._start() # flush the current state of all artifacts self._flush_event.set() def stop(self, wait=True): # type: (bool) -> () # stop the daemon thread and quit # wait until thread exists self._exit_flag = True self._flush_event.set() if wait: if self._thread: self._thread.join() # remove all temp folders for f in self._temp_folder: # noinspection PyBroadException try: Path(f).rmdir() except Exception: pass def _start(self): # type: () -> () """ Start daemon thread if any artifacts are registered and thread is not up yet """ if not self._thread and self._artifacts_container: # start the daemon thread self._flush_event.clear() self._thread = Thread(target=self._daemon) self._thread.daemon = True self._thread.start() def _daemon(self): # type: () -> () while not self._exit_flag: self._flush_event.wait(self._flush_frequency_sec) self._flush_event.clear() artifact_keys = list(self._artifacts_container.keys()) for name in artifact_keys: try: self._upload_data_audit_artifacts(name) except Exception as e: LoggerRoot.get_base_logger().warning(str(e)) # create summary self._summary = self._get_statistics() def _add_artifact(self, artifact): if not self._task: raise ValueError("Task object not set") with self._task_edit_lock: if artifact not in self._task_artifact_list: self._task_artifact_list.append(artifact) # noinspection PyProtectedMember self._task._add_artifacts(self._task_artifact_list) def _upload_data_audit_artifacts(self, name): # type: (str) -> () logger = self._task.get_logger() pd_artifact = self._artifacts_container.get(name) pd_metadata = self._artifacts_container.get_metadata(name) # remove from artifacts watch list if name in self._unregister_request: try: self._unregister_request.remove(name) except KeyError: pass self._artifacts_container.unregister_artifact(name) if pd_artifact is None: return override_filename_ext_in_uri = self._save_format override_filename_in_uri = name fd, local_csv = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) local_csv = Path(local_csv) pd_artifact.to_csv(local_csv.as_posix(), index=False, compression=self._compression) current_sha2, file_sha2 = sha256sum( local_csv.as_posix(), skip_header=32, block_size=Artifacts._hash_block_size) if name in self._last_artifacts_upload: previous_sha2 = self._last_artifacts_upload[name] if previous_sha2 == current_sha2: # nothing to do, we can skip the upload # noinspection PyBroadException try: local_csv.unlink() except Exception: pass return self._last_artifacts_upload[name] = current_sha2 # If old clearml-server, upload as debug image if not Session.check_min_api_version('2.3'): logger.report_image(title='artifacts', series=name, local_path=local_csv.as_posix(), delete_after_upload=True, iteration=self._task.get_last_iteration(), max_image_history=2) return # Find our artifact artifact = None for an_artifact in self._task_artifact_list: if an_artifact.key == name: artifact = an_artifact break file_size = local_csv.stat().st_size # upload file uri = self._upload_local_file( local_csv, name, delete_after_upload=True, override_filename=override_filename_in_uri, override_filename_ext=override_filename_ext_in_uri) # update task artifacts with self._task_edit_lock: if not artifact: artifact = tasks.Artifact(key=name, type=self._pd_artifact_type) artifact_type_data = tasks.ArtifactTypeData() artifact_type_data.data_hash = current_sha2 artifact_type_data.content_type = "text/csv" artifact_type_data.preview = str( pd_artifact.__repr__()) + '\n\n' + self._get_statistics( {name: pd_artifact}) artifact.type_data = artifact_type_data artifact.uri = uri artifact.content_size = file_size artifact.hash = file_sha2 artifact.timestamp = int(time()) artifact.display_data = [ (str(k), str(v)) for k, v in pd_metadata.items() ] if pd_metadata else None self._add_artifact(artifact) def _upload_local_file(self, local_file, name, delete_after_upload=False, override_filename=None, override_filename_ext=None, wait_on_upload=False): # type: (str, str, bool, Optional[str], Optional[str], bool) -> str """ Upload local file and return uri of the uploaded file (uploading in the background) """ from clearml.storage import StorageManager upload_uri = self._task.output_uri or self._task.get_logger( ).get_default_upload_destination() if not isinstance(local_file, Path): local_file = Path(local_file) ev = UploadEvent( metric='artifacts', variant=name, image_data=None, upload_uri=upload_uri, local_image_path=local_file.as_posix(), delete_after_upload=delete_after_upload, override_filename=override_filename, override_filename_ext=override_filename_ext, override_storage_key_prefix=self._get_storage_uri_prefix()) _, uri = ev.get_target_full_upload_uri(upload_uri, quote_uri=False) # send for upload # noinspection PyProtectedMember if wait_on_upload: StorageManager.upload_file(local_file.as_posix(), uri, wait_for_upload=True, retries=ev.retries) if delete_after_upload: try: os.unlink(local_file.as_posix()) except OSError: LoggerRoot.get_base_logger().warning( 'Failed removing temporary {}'.format(local_file)) else: self._task._reporter._report(ev) _, quoted_uri = ev.get_target_full_upload_uri(upload_uri) return quoted_uri def _get_statistics(self, artifacts_dict=None): # type: (Optional[Dict[str, Artifact]]) -> str summary = '' artifacts_dict = artifacts_dict or self._artifacts_container thread_pool = ThreadPool() try: # build hash row sets artifacts_summary = [] for a_name, a_df in artifacts_dict.items(): hash_cols = self._artifacts_container.get_hash_columns(a_name) if not pd or not isinstance(a_df, pd.DataFrame): continue if hash_cols is True: hash_col_drop = [] else: hash_cols = set(hash_cols) missing_cols = hash_cols.difference(a_df.columns) if missing_cols == hash_cols: LoggerRoot.get_base_logger().warning( 'Uniqueness columns {} not found in artifact {}. ' 'Skipping uniqueness check for artifact.'.format( list(missing_cols), a_name)) continue elif missing_cols: # missing_cols must be a subset of hash_cols hash_cols.difference_update(missing_cols) LoggerRoot.get_base_logger().warning( 'Uniqueness columns {} not found in artifact {}. Using {}.' .format(list(missing_cols), a_name, list(hash_cols))) hash_col_drop = [ col for col in a_df.columns if col not in hash_cols ] a_unique_hash = set() def hash_row(r): a_unique_hash.add(hash(bytes(r))) a_shape = a_df.shape # parallelize a_hash_cols = a_df.drop(columns=hash_col_drop) thread_pool.map(hash_row, a_hash_cols.values) # add result artifacts_summary.append(( a_name, a_shape, a_unique_hash, )) # build intersection summary for i, (name, shape, unique_hash) in enumerate(artifacts_summary): summary += '[{name}]: shape={shape}, {unique} unique rows, {percentage:.1f}% uniqueness\n'.format( name=name, shape=shape, unique=len(unique_hash), percentage=100 * len(unique_hash) / float(shape[0])) for name2, shape2, unique_hash2 in artifacts_summary[i + 1:]: intersection = len(unique_hash & unique_hash2) summary += '\tIntersection with [{name2}] {intersection} rows: {percentage:.1f}%\n'.format( name2=name2, intersection=intersection, percentage=100 * intersection / float(len(unique_hash2))) except Exception as e: LoggerRoot.get_base_logger().warning(str(e)) finally: thread_pool.close() thread_pool.terminate() return summary def _get_temp_folder(self, force_new=False): # type: (bool) -> str if force_new or not self._temp_folder: new_temp = mkdtemp(prefix='artifacts_') self._temp_folder.append(new_temp) return new_temp return self._temp_folder[0] def _get_storage_uri_prefix(self): # type: () -> str if not self._storage_prefix: # noinspection PyProtectedMember self._storage_prefix = self._task._get_output_destination_suffix() return self._storage_prefix
class Coordinator(object): """A coordinator for processes. This class implements a simple mechanism to coordinate the termination of a set of processes. ```python with coord.stop_on_exception(): while not coord.should_stop(): ...do some work... ``` """ def __init__(self): """Create a new Coordinator. """ # Protects all attributes. self._lock = Lock() # Event set when processes must stop. self._stop_event = Event() # Python exc_info to report. # If not None, it should hold the returned value of sys.exc_info(), which is # a tuple containing exception (type, value, traceback). self._exc_info_to_raise = None # True if we have called join() already. self._joined = Value(ctypes.c_int, 0, lock=False) # Set of processes registered for joining when join() is called. These # will be joined in addition to the processes passed to the join() # call. It's ok if processes are both registered and passed to the join() # call. self._registered_processes = set() def request_stop(self, ex=None): """Request that the processes stop. After this is called, calls to `should_stop()` will return `True`. Note: If an exception is being passed in, in must be in the context of handling the exception (i.e. `try: ... except Exception as ex: ...`) and not a newly created one. Args: ex (Exception or exc_info tuple): Optional `Exception`, or Python `exc_info` tuple as returned by `sys.exc_info()`. If this is the first call to `request_stop()` the corresponding exception is recorded and re-raised from `join()`. """ with self._lock: # If we have already joined the coordinator the exception will not have a # chance to be reported, so just raise it normally. This can happen if # you continue to use a session have having stopped and joined the # coordinator process. if self.joined: if isinstance(ex, tuple): six.reraise(*ex) elif ex is not None: # NOTE(touts): This is bogus if request_stop() is not called # from the exception handler that raised ex. six.reraise(*sys.exc_info()) if not self._stop_event.is_set(): if ex and self._exc_info_to_raise is None: if isinstance(ex, tuple): logging.info("Error reported to Coordinator: %s", str(ex[1]), exc_info=ex) self._exc_info_to_raise = ex else: logging.info("Error reported to Coordinator: %s, %s", type(ex), str(ex)) self._exc_info_to_raise = sys.exc_info() # self._exc_info_to_raise should contain a tuple containing exception # (type, value, traceback) if (len(self._exc_info_to_raise) != 3 or not self._exc_info_to_raise[0] or not self._exc_info_to_raise[1]): # Raise, catch and record the exception here so that error happens # where expected. try: raise ValueError( "ex must be a tuple or sys.exc_info must " "return the current exception: %s" % self._exc_info_to_raise) except ValueError: # Record this error so it kills the coordinator properly. # NOTE(touts): As above, this is bogus if request_stop() is not # called from the exception handler that raised ex. self._exc_info_to_raise = sys.exc_info() self._stop_event.set() def clear_stop(self): """Clears the stop flag. After this is called, calls to `should_stop()` will return `False`. """ with self._lock: self._joined.value = 0 self._exc_info_to_raise = None if self._stop_event.is_set(): self._stop_event.clear() def should_stop(self): """Check if stop was requested. Returns: True if a stop was requested. """ return self._stop_event.is_set() @contextlib.contextmanager def stop_on_exception(self): """Context manager to request stop when an Exception is raised. Code that uses a coordinator must catch exceptions and pass them to the `request_stop()` method to stop the other processes managed by the coordinator. This context handler simplifies the exception handling. Use it as follows: ```python with coord.stop_on_exception(): # Any exception raised in the body of the with # clause is reported to the coordinator before terminating # the execution of the body. ...body... ``` This is completely equivalent to the slightly longer code: ```python try: ...body... except: coord.request_stop(sys.exc_info()) ``` Yields: nothing. """ try: yield except: # pylint: disable=bare-except self.request_stop(ex=sys.exc_info()) def wait_for_stop(self, timeout=None): """Wait till the Coordinator is told to stop. Args: timeout: Float. Sleep for up to that many seconds waiting for should_stop() to become True. Returns: True if the Coordinator is told stop, False if the timeout expired. """ return self._stop_event.wait(timeout) def register_process(self, process): """Register a process to join. Args: process: A python.multiprocessing.Process to join. """ with self._lock: self._registered_processes.add(process) def join(self, processes=None, stop_grace_period_secs=120, ignore_live_processes=False): """Wait for processes to terminate. This call blocks until a set of processes have terminated. The set of process is the union of the processes passed in the `processes` argument and the list of processes that registered with the coordinator by calling `Coordinator.register_process()`. After the processes stop, if an `exc_info` was passed to `request_stop`, that exception is re-raised. Grace period handling: When `request_stop()` is called, processes are given 'stop_grace_period_secs' seconds to terminate. If any of them is still alive after that period expires, a `RuntimeError` is raised. Note that if an `exc_info` was passed to `request_stop()` then it is raised instead of that `RuntimeError`. Args: processes (list of `Processes`): The started processes to join in addition to the registered processes. stop_grace_period_secs: Number of seconds given to processes to stop after `request_stop()` has been called. ignore_live_processes: If `False`, raises an error if any of the processes are still alive after `stop_grace_period_secs`. Raises: RuntimeError: If any process is still alive after `request_stop()` is called and the grace period expires. """ # processes registered after this call will not be joined. with self._lock: if processes is None: processes = self._registered_processes else: processes = self._registered_processes.union(set(processes)) # Copy the set into a list to avoid race conditions where a new process # is added while we are waiting. processes = list(processes) # Wait for all processes to stop or for request_stop() to be called. while any(t.is_alive() for t in processes) and not self.wait_for_stop(1.0): pass # If any process is still alive, wait for the grace period to expire. # By the time this check is executed, processes may still be shutting down, # so we add a sleep of increasing duration to give them a chance to shut # down without losing too many cycles. # The sleep duration is limited to the remaining grace duration. stop_wait_secs = 0.001 while any(t.is_alive() for t in processes) and stop_grace_period_secs >= 0.0: time.sleep(stop_wait_secs) stop_grace_period_secs -= stop_wait_secs stop_wait_secs = 2 * stop_wait_secs # Keep the waiting period within sane bounds. # The minimum value is to avoid decreasing stop_wait_secs to a value # that could cause stop_grace_period_secs to remain unchanged. stop_wait_secs = max(min(stop_wait_secs, stop_grace_period_secs), 0.001) # List the processes still alive after the grace period. stragglers = [t.name for t in processes if t.is_alive()] # Terminate with an exception if appropriate. with self._lock: self._joined.value = 1 self._registered_processes = set() if self._exc_info_to_raise: six.reraise(*self._exc_info_to_raise) elif stragglers: if ignore_live_processes: logging.info( "Coordinator stopped with processes still running: %s", " ".join(stragglers)) else: raise RuntimeError( "Coordinator stopped with processes still running: %s" % " ".join(stragglers)) @property def joined(self): return self._joined.value def raise_requested_exception(self): """If an exception has been passed to `request_stop`, this raises it.""" with self._lock: if self._exc_info_to_raise: six.reraise(*self._exc_info_to_raise)
class Plant(object): def __init__(self, params=None, x0=None, S0=None, dt=0.01, noise=None, name='Plant', angle_dims=[]): self.name = name self.params = params self.x0 = x0 self.S0 = S0 self.x = np.array(x0, dtype=np.float64).flatten() self.u = None self.t = 0 self.dt = dt self.noise = noise self.running = Event() self.done = False self.plant_thread = None self.angle_dims = angle_dims def apply_control(self, u): self.u = np.array(u, dtype=np.float64) if len(self.u.shape) < 2: self.u = self.u[:, None] def get_plant_state(self): if self.angle_dims is None: return self.x.flatten(), self.t else: return gTrig_np(self.x, self.angle_dims).flatten(), self.t def run(self): start_time = time() print('Starting plant loop', self.name) while self.running.is_set(): exec_time = time() self.step(self.dt) exec_time = time() - exec_time sleep(max(self.dt - exec_time, 0)) def start(self): if self.plant_thread is not None and self.plant_thread.is_alive(): while self.plant_thread.is_alive(): sleep(1.0) self.plant_thread = Thread(target=self.run) self.plant_thread.daemon = True self.running.set() self.plant_thread.start() def stop(self): self.running.clear() if self.plant_thread is not None and self.plant_thread.is_alive(): # wait until thread stops self.plant_thread.join(10) # create new thread object, since python threads can only be started once self.plant_thread = Thread(target=self.run) self.plant_thread.daemon = True def step(self): raise NotImplementedError( "You need to implement the step method in your Plant subclass.") def reset_state(self): raise NotImplementedError( "You need to implement the reset_state method in your Plant subclass." )
class PlantDraw(object): def __init__(self, plant, refresh_period=(1.0 / 24), name='PlantDraw'): super(PlantDraw, self).__init__() self.name = name self.plant = plant self.drawing_thread = None self.polling_thread = None self.dt = refresh_period self.scale = 150 # pixels per meter self.center_x = 0 self.center_y = 0 self.running = Event() self.polling_pipe, self.drawing_pipe = Pipe() def init_ui(self): self.fig = plt.figure(self.name) #,figsize=(16,10)) plt.xlim([-1.5, 1.5]) plt.ylim([-1.5, 1.5]) self.ax = plt.gca() self.ax.set_aspect('equal', 'datalim') self.ax.grid(True) self.bg = self.fig.canvas.copy_from_bbox(self.ax.bbox) self.init_artists() self.fig.canvas.draw() self.cursor = Cursor(self.ax, useblit=True, color='red', linewidth=2) plt.ion() plt.show() def drawing_loop(self, drawing_pipe): # start the matplotlib plotting self.init_ui() while self.running.is_set(): exec_time = time() # get any data from the polling loop updts = None while drawing_pipe.poll(): data_from_plant = drawing_pipe.recv() if data_from_plant is None: self.running.clear() break # get the visuzlization updates from the latest state state, t = data_from_plant updts = self.update(state, t) if updts is not None: # update the drawing from the plant state self.fig.canvas.restore_region(self.bg) for artist in updts: self.ax.draw_artist(artist) self.fig.canvas.blit(self.ax.bbox) # sleep to guarantee the desired frame rate exec_time = time() - exec_time sleep(max(self.dt - exec_time, 0)) # close the matplotlib windows, clean up plt.ioff() plt.close(self.fig) def polling_loop(self, polling_pipe): current_t = -1 while self.running.is_set(): exec_time = time() state, t = self.plant.get_plant_state() if t != current_t: polling_pipe.send((state, t)) # sleep to guarantee the desired frame rate exec_time = time() - exec_time sleep(max(self.dt - exec_time, 0)) def start(self): self.drawing_thread = Process(target=self.drawing_loop, args=(self.drawing_pipe, )) self.drawing_thread.daemon = True self.polling_thread = Thread(target=self.polling_loop, args=(self.polling_pipe, )) self.polling_thread.daemon = True #self.drawing_thread = Process(target=self.run) self.running.set() self.polling_thread.start() self.drawing_thread.start() def stop(self): self.running.clear() if self.drawing_thread is not None and self.drawing_thread.is_alive(): # wait until thread stops self.drawing_thread.join(10) if self.polling_thread is not None and self.polling_thread.is_alive(): # wait until thread stops self.polling_thread.join(10) def update(self): raise NotImplementedError( "You need to implement the self.update() method in your PlantDraw class." ) def init_artists(self): raise NotImplementedError( "You need to implement the self.init_artists() method in your PlantDraw class." )
class ValkkaProcess(Process): """ Semantics: Frontend: the part of the forked process that keeps running in the current, user virtual memory space Backend : the part of the forked process that runs in its own virtual memory space (e.g. "in the background") This class has both backend and frontend methods: - Backend methods should only be called from backend. They are designated with "_". - Frontend methods should only be called from frontend To avoid confusion, backend methods are designated with "_", except for the "run()" method, that's always in the backend Frontend methods use a pipe to send a signal to backend that then handles the signal with a backend method having the same name (but with "_" in the end) Backend methods can, in a similar fashion, send signals to the frontend using a pipe. In frontend, a listening thread is needed. That thread can then call the handleSignal method that chooses the correct frontend method to call TODO: add the possibility to bind the process to a certain processor """ # incoming signals : from frontend to backend incoming_signal_defs = { # each key corresponds to a front- and backend methods "test_": {"test_int": int, "test_str": str}, "stop_": [] } # outgoing signals : from back to frontend. Don't use same names as for # incoming signals .. outgoing_signal_defs = { "test_o": {"test_int": int, "test_str": str}, } def __init__(self, name, affinity=-1, **kwargs): super().__init__() self.pre = self.__class__.__name__ + " : " + name + \ " : " # auxiliary string for debugging output self.name = name self.affinity = affinity self.signal_in = Event() self.signal_out = Event() # communications pipe. Frontend uses self.pipe, backend self.childpipe self.pipe, self.childpipe = Pipe() self.signal_in.clear() self.signal_out.clear() # print(self.pre, "init") def getPipe(self): """Returns communication pipe for front-end """ return self.pipe def preRun_(self): """After the fork, but before starting the process loop """ if (self.affinity > -1): os.system("taskset -p -c %d %d" % (self.affinity, os.getpid())) def postRun_(self): """Just before process exit """ print(self.pre, "post: bye!") def cycle_(self): # Do whatever your process should be doing, remember timeout every now # and then time.sleep(5) print(self.pre, "hello!") def startAsThread(self): from threading import Thread t = Thread(target=self.run) t.start() def run(self): # No "_" in the name, but nevertheless, running in the backed """After the fork. Now the process starts running """ # print(self.pre," ==> run") self.preRun_() self.running = True while(self.running): self.cycle_() self.handleSignal_() self.postRun_() def handleSignal_(self): """Signals handling in the backend """ if (self.signal_in.is_set()): signal_dic = self.childpipe.recv() method_name = signal_dic.pop("name") method = getattr(self, method_name) method(**signal_dic) self.signal_in.clear() self.signal_out.set() def sendSignal(self, **kwargs): # sendSignal(name="test",test_int=1,test_str="kokkelis") """Incoming signals: this is used by frontend methods to send signals to the backend """ try: name = kwargs.pop("name") except KeyError: raise(AttributeError("Signal name missing")) # a dictionary: {"parameter_name" : parameter_type} model = self.incoming_signal_defs[name] for key in kwargs: # raises error if user is using undefined signal model_type = model[key] parameter_type = kwargs[key].__class__ if (model_type == parameter_type): pass else: raise(AttributeError("Wrong type for parameter " + str(key))) kwargs["name"] = name self.pipe.send(kwargs) self.signal_out.clear() self.signal_in. set() # indicate that there is a signal self.signal_out.wait() # wait for the backend to clear the signal def handleSignal(self, signal_dic): """Signal handling in the frontend """ method_name = signal_dic.pop("name") method = getattr(self, method_name) method(**signal_dic) def sendSignal_(self, **kwargs): # sendSignal_(name="test_out",..) """Outgoing signals: signals from backend to frontend """ try: name = kwargs.pop("name") except KeyError: raise(AttributeError("Signal name missing")) # a dictionary: {"parameter_name" : parameter_type} model = self.outgoing_signal_defs[name] for key in kwargs: # raises error if user is using undefined signal try: model_type = model[key] except KeyError: print("your outgoing_signal_defs for",name,"is:", model) print("you requested key:", key) raise parameter_type = kwargs[key].__class__ if (model_type == parameter_type): pass else: raise(AttributeError("Wrong type for parameter " + str(key))) kwargs["name"] = name self.childpipe.send(kwargs) # *** backend methods corresponding to each incoming signals *** def stop_(self): self.running = False def test_(self, test_int=0, test_str="nada"): print(self.pre, "test_ signal received with", test_int, test_str) # ** frontend methods corresponding to each incoming signal: these communicate with the backend via pipes ** def stop(self): self.sendSignal(name="stop_") def test(self, **kwargs): dictionaryCheck(self.incoming_signal_defs["test_"], kwargs) kwargs["name"] = "test_" self.sendSignal(**kwargs) # ** frontend methods corresponding to each outgoing signal ** # typically, there is a QThread in the frontend-side reading the process pipe # the QThread reads kwargs dictionary from the pipe, say # {"name":"test_o", "test_str":"eka", "test_int":1} # And calls handleSignal(kwargs) def test_o(self, **kwargs): pass
class MultiProcessConsumer(Consumer): """ A consumer implementation that consumes partitions for a topic in parallel using multiple processes client: a connected KafkaClient group: a name for this consumer, used for offset storage and must be unique topic: the topic to consume auto_commit: default True. Whether or not to auto commit the offsets auto_commit_every_n: default 100. How many messages to consume before a commit auto_commit_every_t: default 5000. How much time (in milliseconds) to wait before commit num_procs: Number of processes to start for consuming messages. The available partitions will be divided among these processes partitions_per_proc: Number of partitions to be allocated per process (overrides num_procs) Auto commit details: If both auto_commit_every_n and auto_commit_every_t are set, they will reset one another when one is triggered. These triggers simply call the commit method on this class. A manual call to commit will also reset these triggers """ def __init__(self, client, group, topic, auto_commit=True, auto_commit_every_n=AUTO_COMMIT_MSG_COUNT, auto_commit_every_t=AUTO_COMMIT_INTERVAL, num_procs=1, partitions_per_proc=0): # Initiate the base consumer class super(MultiProcessConsumer, self).__init__( client, group, topic, partitions=None, auto_commit=auto_commit, auto_commit_every_n=auto_commit_every_n, auto_commit_every_t=auto_commit_every_t) # Variables for managing and controlling the data flow from # consumer child process to master self.queue = MPQueue(1024) # Child consumers dump messages into this self.start = Event() # Indicates the consumers to start fetch self.exit = Event() # Requests the consumers to shutdown self.pause = Event() # Requests the consumers to pause fetch self.size = Value('i', 0) # Indicator of number of messages to fetch partitions = self.offsets.keys() # If unspecified, start one consumer per partition # The logic below ensures that # * we do not cross the num_procs limit # * we have an even distribution of partitions among processes if not partitions_per_proc: partitions_per_proc = round(len(partitions) * 1.0 / num_procs) if partitions_per_proc < num_procs * 0.5: partitions_per_proc += 1 # The final set of chunks chunker = lambda *x: [] + list(x) chunks = map(chunker, *[iter(partitions)] * int(partitions_per_proc)) self.procs = [] for chunk in chunks: chunk = filter(lambda x: x is not None, chunk) args = (client.copy(), group, topic, chunk, self.queue, self.start, self.exit, self.pause, self.size) proc = Process(target=_mp_consume, args=args) proc.daemon = True proc.start() self.procs.append(proc) def __repr__(self): return '<MultiProcessConsumer group=%s, topic=%s, consumers=%d>' % \ (self.group, self.topic, len(self.procs)) def stop(self): # Set exit and start off all waiting consumers self.exit.set() self.pause.set() self.start.set() for proc in self.procs: proc.join() proc.terminate() super(MultiProcessConsumer, self).stop() def __iter__(self): """ Iterator to consume the messages available on this consumer """ # Trigger the consumer procs to start off. # We will iterate till there are no more messages available self.size.value = 0 self.pause.set() while True: self.start.set() try: # We will block for a small while so that the consumers get # a chance to run and put some messages in the queue # TODO: This is a hack and will make the consumer block for # at least one second. Need to find a better way of doing this partition, message = self.queue.get(block=True, timeout=1) except Empty: break # Count, check and commit messages if necessary self.offsets[partition] = message.offset + 1 self.start.clear() self.count_since_commit += 1 self._auto_commit() yield message self.start.clear() def get_messages(self, count=1, block=True, timeout=10): """ Fetch the specified number of messages count: Indicates the maximum number of messages to be fetched block: If True, the API will block till some messages are fetched. timeout: If block is True, the function will block for the specified time (in seconds) until count messages is fetched. If None, it will block forever. """ messages = [] # Give a size hint to the consumers. Each consumer process will fetch # a maximum of "count" messages. This will fetch more messages than # necessary, but these will not be committed to kafka. Also, the extra # messages can be provided in subsequent runs self.size.value = count self.pause.clear() if timeout is not None: max_time = time.time() + timeout new_offsets = {} while count > 0 and (timeout is None or timeout > 0): # Trigger consumption only if the queue is empty # By doing this, we will ensure that consumers do not # go into overdrive and keep consuming thousands of # messages when the user might need only a few if self.queue.empty(): self.start.set() try: partition, message = self.queue.get(block, timeout) except Empty: break messages.append(message) new_offsets[partition] = message.offset + 1 count -= 1 if timeout is not None: timeout = max_time - time.time() self.size.value = 0 self.start.clear() self.pause.set() # Update and commit offsets if necessary self.offsets.update(new_offsets) self.count_since_commit += len(messages) self._auto_commit() return messages
from multiprocessing import Event #生成事件对象 e = Event() #检测事件对象,如果被设置则返回True否则返回false print(e.is_set()) #设置事件对象 e.set() #提供事件的阻塞 e.wait() print("wait.....") #清除对事件的设置 e.clear() e.wait() print("wait...wait.....")
class Que(Thread): def __init__(self, name, frequency, que): super(Que, self).__init__() self.__commands = dict() self.running = False self.__frequency = frequency self.__que = que self.__ready = False self.__paused = True self.pauseEvent = Event() self.resumeEvent = Event() self.__readyEvent = Event() self.daemon = True self.name = name self.deleteAfterPoll = False logger.debug('Que {} declared'.format(self.name)) def run(self): self.running = True logger.info('Que {} starting'.format(self.name)) while self.running: try: if not self.__ready: logger.debug('Not Ready on thread {}'.format(self.name)) self.__readyEvent.wait() self.__readyEvent.clear() logger.debug('Ready on thread {}'.format(self.name)) logger.debug('Que {} paused state = {}'.format( self.name, self.__paused)) if not self.__paused: for s in self.__commands: lastPolled = time() logger.debug( 'Que {} adding command {} to output queue'.format( self.name, s)) self.__que.put(s) if self.deleteAfterPoll: self.removeCommand(s) break time_elapsed = time() - lastPolled sleeptime = (1.0 / self.__frequency / len(self.__commands)) - time_elapsed sleep(sleeptime) else: logger.debug('Pausing thread {}'.format(self.name)) self.pauseEvent.set() self.resumeEvent.wait() self.resumeEvent.clear() logger.debug('Resuming thread {}'.format(self.name)) except (KeyboardInterrupt, SystemExit): self.running = False continue except (RuntimeError): continue except: logger.critical('Exception caught in Queue Thread {}:'.format( self.name), exc_info=True, stack_info=True) continue logger.info('Que {} stopped'.format(self.name)) def setFrequency(self, frequency): self.__frequency = frequency def addCommand(self, command, override=False): logger.info('Appending command {} to que {}'.format( command, self.name)) pausedState = self.__paused if not self.__paused: self.__paused = True # Pause the que when making changes if self.__ready: logger.debug('Waiting for main loop to stop on que {}'.format( self.name)) self.pauseEvent.wait( ) # Wait for pauseWait event to ensure dict will not be accessed self.pauseEvent.clear() logger.debug('Main loop stopped on que {}'.format(self.name)) self.__commands[command] = override if not self.__ready: self.__readyEvent.set() self.__ready = True if not pausedState: self.__paused = False # Resume que after update self.resumeEvent.set() logger.debug('Resuming main loop on que {}'.format(self.name)) def getCommands(self): l = [] for c in self.__commands: l.append(c) return l def removeCommand(self, command): logger.debug('Removing sensor {} from que {}'.format( command, self.name)) pausedState = self.__paused if not self.__paused: self.__paused = True if self.__ready: logger.debug('Waiting for main loop to stop on que {}'.format( self.name)) self.pauseEvent.wait() self.pauseEvent.clear() logger.debug('Main loop stopped on que {}'.format(self.name)) if command in self.__commands: del self.__commands[command] if len(self.__commands) == 0: self.__ready = False logger.info('Que {} not ready due to zero length'.format( self.name)) if not pausedState: self.__paused = False self.resumeEvent.set() logger.debug('Resuming main loop on que {}'.format(self.name)) def status(self): #returns a dict of que status d = dict() d['Name'] = self.name d['Frequency'] = self.__frequency d['Running'] = self.running d['Ready'] = self.__ready d['Paused'] = self.__paused d['Length'] = len(self.__commands) return d def pause(self): self.__paused = True def resume(self): self.__paused = False self.resumeEvent.set()
class UDPConnectionProcess(Process): """UDPConnectionProcess polls and writes to a data queue. Example:: # Server that prints each input and echos it to the client # that is currently connected from udp_connection import UDPConnectionProcess, Queue receive_queue = Queue() udp_p = UDPConnectionProcess(receive_queue=receive_queue) udp_p.start() udp_p.event_polling.set() # start polling while True: data = receive_queue.get() print(data) if data is not None: udp_p.send_queue.put(data.string) Example:: # connecting to a server """ # DOC def __init__(self, event_trigger=(), event_ignore_tag=None): """Initialize UDPConnectionProcess Parameters ---------- receive_queue: multiprocessing.Queue the queue to which the received data should be put peer_ip : string the IP of the peer to which the connection should be established sync_clock : Clock the internal clock for timestamps will synchronized with this clock event_trigger: multiprocessing.Event() (or list of..) event trigger(s) to be set. If Udp event is received and it is not a command to set this event (typical of sensor recording processes). event_ignore_tag: udp data that start with this tag will be ignored for event triggering """ # DOC super(UDPConnectionProcess, self).__init__() self.receive_queue = Queue() self.send_queue = Queue() self.event_is_connected = Event() self._event_quit_request = Event() self._event_is_polling = Event() self._event_ignore_tag = event_ignore_tag if isinstance(event_trigger, type(Event)): event_trigger = (event_trigger) try: self._event_trigger = tuple(event_trigger) except: self._event_trigger = () atexit.register(self.quit) @property def my_ip(self): return UDPConnection.MY_IP def quit(self): self._event_quit_request.set() if self.is_alive(): self.join() def pause(self): self._event_is_polling.clear() def start_polling(self): self._event_is_polling.set() def run(self): udp_connection = UDPConnection(udp_port=5005) self.start_polling() ptp = PollingTimeProfile() prev_event_polling = None while not self._event_quit_request.is_set(): if prev_event_polling != self._event_is_polling.is_set(): # event pooling changed prev_event_polling = self._event_is_polling.is_set() if prev_event_polling: logging.warning("UDP start, pid {}, priority {}".format( self.pid, get_priority(self.pid))) else: logging.warning("UDP stop") ptp.stop() if not self._event_is_polling.is_set(): self._event_is_polling.wait(timeout=0.1) else: data = udp_connection.poll() t = udp_connection.timer.time ptp.update(t) if data is not None: d = UDPData(string=data, time=t) self.receive_queue.put(d) if self._event_ignore_tag is not None and \ not d.startswith(self._event_ignore_tag): for ev in self._event_trigger: # set all connected trigger ev.set() try: udp_connection.send(self.send_queue.get_nowait()) except: pass # has connection changed? if self.event_is_connected.is_set( ) != udp_connection.is_connected: if udp_connection.is_connected: self.event_is_connected.set() else: self.event_is_connected.clear() if not udp_connection.is_connected: udp_connection.timer.wait(200) udp_connection.unconnect_peer() logging.warning("UDP quit, {}".format(ptp.get_profile_str()))
class JobExecutor2: """ Execute jobs on multiple processes. At a high level, we have worker executing on multiple processes. Each worker will be fed by an input queue and results of the processing pushed to an output queue. Pushing data on a queue is very fast BUT retrieving it from a different process takes time. Even if PyTorch claims to have memory shared arrays, retrieving a large array from a queue has a linear runtime complexity. To limit this copy penalty, we can use threads that copy from the worker process to the main process (`pinning` threads. Here, sharing data between threads is almost free). Notes: This class was designed for maximum speed and not reproducibility in mind. The processed of jobs will not keep their ordering. """ def __init__(self, nb_workers: int, function_to_run: Callable[[Batch], Batch], max_queue_size_per_worker: int = 2, max_queue_size_pin_thread_per_worker: int = 3, nb_pin_threads: Optional[int] = None, wait_time: float = 0.02, wait_until_processes_start: bool = True): """ Args: nb_workers: the number of workers (processes) to execute the jobs function_to_run: the job to be executed max_queue_size_per_worker: define the maximum number of job results that can be stored before a process is blocked (i.e., larger queue size will improve performance but will require more memory to store the results). the pin_thread need to process the result before the blocked process can continue its processing. max_queue_size_pin_thread_per_worker: define the maximum number of results available on the main process (i.e., larger queue size will improve performance but will require more memory to store the results). nb_pin_threads: the number of threads dedicated to collect the jobs processed by different processes. Data copy from the worker process to main process takes time, in particular for large data. It is advantageous to have multiple threads that copy these results to the main process. If `None`, nb_workers // 2 threads will be used wait_time: the default wait time for a process or thread to sleep if no job is available wait_until_processes_start: if True, the main process will wait until the worker processes and pin threads are fully running """ print(f'JobExecutor2 started on process={os.getpid()}') self.wait_until_processes_start = wait_until_processes_start self.wait_time = wait_time self.max_queue_size_pin_thread_per_worker = max_queue_size_pin_thread_per_worker self.max_queue_size_per_worker = max_queue_size_per_worker self.function_to_run = function_to_run self.nb_workers = nb_workers self.abort_event = Event() self.main_process = os.getpid() if nb_pin_threads is None: self.nb_pin_threads = max(1, nb_workers // 2) else: self.nb_pin_threads = nb_pin_threads assert self.nb_pin_threads >= 1, 'must have at least one thread to collect the processed jobs!' self.worker_control = 0 self.worker_input_queues = [] self.worker_output_queues = [] self.processes = [] self.jobs_processed = Value('i', 0) self.jobs_queued = 0 self.pin_memory_threads = [] self.pin_memory_queue = None # we can't cancel jobs, so instead record a session ID. If session of # the worker and current session ID do not match # it means the results of these tasks should be discarded self.job_session_id = Value('i', 0) self.start() def start(self, timeout: float = 10.0) -> None: """ Start the processes and queues. Args: timeout: Returns: """ if self.pin_memory_queue is None: self.pin_memory_queue = ThreadQueue( self.max_queue_size_pin_thread_per_worker * self.nb_workers) if self.nb_workers == 0: # nothing to do, this will be executed synchronously on # the main process return if len(self.processes) != self.nb_workers: logging.debug( f'Starting jobExecutor={self}, on process={os.getpid()} nb_workers={self.nb_workers}' ) self.close() self.abort_event.clear() with threadpool_limits(limits=1, user_api='blas'): for i in range(self.nb_workers): #maxsize = 0 self.worker_input_queues.append( Queue(maxsize=self.max_queue_size_per_worker)) self.worker_output_queues.append( Queue(self.max_queue_size_per_worker)) p = Process(target=worker, name=f'JobExecutorWorker-{i}', args=(self.worker_input_queues[i], self.worker_output_queues[i], self.function_to_run, self.abort_event, self.wait_time, i)) p.daemon = True p.start() self.processes.append(p) logging.debug( f'Child process={p.pid} for jobExecutor={self}') self.pin_memory_threads = [] for i in range(self.nb_pin_threads): pin_memory_thread = threading.Thread( name=f'JobExecutorThreadResultCollector-{i}', target=collect_results_to_main_process, args=(self.job_session_id, self.jobs_processed, self.worker_output_queues, self.pin_memory_queue, self.abort_event, self.wait_time)) self.pin_memory_threads.append(pin_memory_thread) pin_memory_thread.daemon = True pin_memory_thread.start() self.worker_control = 0 if self.wait_until_processes_start: # wait until all the processes and threads are alive waiting_started = perf_counter() while True: wait_more = False for p in self.processes: if not p.is_alive(): wait_more = True continue for t in self.pin_memory_threads: if not t.is_alive(): wait_more = True continue if wait_more: waiting_time = perf_counter() - waiting_started if waiting_time < timeout: sleep(self.wait_time) else: logging.error( 'the worker processes/pin threads were too slow to start!' ) break logging.debug(f'jobExecutor ready={self}') def close(self, timeout: float = 10) -> None: """ Stops the processes and threads. Args: timeout: time allowed for the threads and processes to shutdown cleanly before using `terminate()` """ # notify all the threads and processes to be shut down print('Setting `abort_event` to interrupt Processes and threads!') self.abort_event.set() if os.getpid() != self.main_process: logging.error( f'attempting to close the executor from a ' f'process={os.getpid()} that did not create it! ({self.main_process})' ) return # give some time to the threads/processes to shutdown normally shutdown_time_start = perf_counter() while True: wait_more = False if len(self.processes) != 0: for p in self.processes: if p.is_alive(): wait_more = True break if len(self.pin_memory_threads): for t in self.pin_memory_threads: if t.is_alive(): wait_more = True break shutdown_time = perf_counter() - shutdown_time_start if wait_more: if shutdown_time < timeout: sleep(0.1) continue else: logging.error( 'a job did not respond to the shutdown request in the allotted time. ' 'It could be that it needs a longer timeout or a deadlock. The processes' 'will now be forced to shutdown!') # done normal shutdown or timeout break if len(self.processes) != 0: #logging.debug(f'JobExecutor={self}: shutting down workers...') [i.terminate() for i in self.processes] for i, p in enumerate(self.processes): self.worker_input_queues[i].close() self.worker_input_queues[i].join_thread() self.worker_output_queues[i].close() self.worker_output_queues[i].join_thread() self.worker_input_queues = [] self.worker_output_queues = [] self.processes = [] if len(self.pin_memory_threads) > 0: for thread in self.pin_memory_threads: thread.join() del thread self.pin_memory_threads = [] del self.pin_memory_queue self.pin_memory_queue = None def is_full(self) -> bool: """ Check if the worker input queues are full. Returns: True if full, False otherwise """ if self.nb_workers == 0: return False for i in range(self.nb_workers): queue = self.worker_input_queues[self.worker_control] if not queue.full(): return False self.worker_control = (self.worker_control + 1) % self.nb_workers return True def put(self, data: Batch) -> bool: """ Queue a batch of data to be processed. Warnings: if the queues are full, the batch will NOT be appended Args: data: a batch of data to process Returns: True if the batch was successfully appended, False otherwise. """ if self.nb_workers == 0: batch_in = copy.deepcopy(data) batch_out = self.function_to_run(batch_in) self.pin_memory_queue.put(batch_out) self.jobs_queued += 1 return True else: for i in range(self.nb_workers): queue = self.worker_input_queues[self.worker_control] if not queue.full(): queue.put((self.job_session_id.value, data)) self.worker_control = (self.worker_control + 1) % self.nb_workers self.jobs_queued += 1 return True # all queues are full, we have to wait return False def is_idle(self) -> bool: """ Returns: True if the executor is not currently processing jobs """ with self.jobs_processed.get_lock(): return self.jobs_processed.value == self.jobs_queued def reset(self): """ Reset the input and output queues as well as job session IDs. The results of the jobs that have not yet been calculated will be discarded """ # here we could clear the queues for a faster implementation. # Unfortunately, this is not an easy task to properly # counts all the jobs processed or discarded due to the # multi-threading. Instead, all tasks queued are executed # and we use a `job_session_id` to figure out the jobs to be # discarded """ # empty the various queues try: for input_queue in self.worker_input_queues: while not input_queue.empty(): input_queue.get() except EOFError: # in case the other process was already terminated pass try: for output_queue in self.worker_output_queues: while not output_queue.empty(): output_queue.get() except EOFError: # in case the other process was already terminated pass with self.jobs_processed.get_lock(): self.jobs_processed.value = 0 self.jobs_queued = 0 """ # empty the current queue results, they are not valid anymore! try: while not self.pin_memory_queue.empty(): self.pin_memory_queue.get() except EOFError: # in case the other process was already terminated pass # discard the results of the jobs that will not have the # current `job_session_id` with self.job_session_id.get_lock(): self.job_session_id.value += 1 def __del__(self): #logging.debug(f'JobExecutor={self}: destructor called') self.close()
class AmpDecorator(Amplifier): """This class 'decorates' the Low-Level Amplifier classes with Network-Marker and Save-To-File functionality. You use it by decorating (not as in Python-Decorator, but in the GoF sense) the low level amplifier class you want to use:: import libmushu from libmushu.ampdecorator import AmpDecorator from libmushu.driver.randomamp import RandomAmp amp = Ampdecorator(RandomAmp) Waring: The network marker timings on Windows have a resolution of 10ms-15ms. On Linux the resolution is 1us. This is due to limitations of Python's time.time method, or rather a Windows specific issue. There exists currently no precise timer, providing times which are comparable between two running processes on Windows. The performance counter provided on Windows, has a much better resolution but is relative to the processes start time and it drifts (1s per 100s), so it is only precise for a relatively short amount of time. If a higher precision is needed one has to replace the time.time calls with something which provides a better precision. For example one could create a third process which provides times or regularly synchronize both processes with the clock synchronization algorithm as described here: http://en.wikipedia.org/wiki/Network_Time_Protocol Alternatively one could use `timeGetTime` from Windows' Multi Media library, which is tunable via `timeBeginPeriod` and provides a precision of 1-2ms. Apparently this is the way Chrome and many others do it.:: from __future__ import division from ctypes import windll import time timeBeginPeriod = windll.winmm.timeBeginPeriod timeEndPeriod = windll.winmm.timeEndPeriod timeGetTime = windll.winmm.timeGetTime if __name__ == '__main__': # wrap the code that needs high precision in timeBegin- and # timeEndPeriod with the same parameter. The parameter is # the interval in ms you want as precision. Usually the # minimum value allowed is 1 (best). timeBeginPeriod(1) times = [] t_start = time.time() while time.time() < (time.time() + 1): times.append(timeGetTime()) times = sorted(list(set(times))) print(1000 / len(times)) timeEndPeriod(1) """ def __init__(self, ampcls): self.amp = ampcls() self.write_to_file = False @property def presets(self): return self.amp.presets def start(self, filename=None): # prepare files for writing self.write_to_file = False if filename is not None: self.write_to_file = True filename_marker = filename + '.marker' filename_eeg = filename + '.eeg' filename_meta = filename + '.meta' for filename in filename_marker, filename_eeg, filename_meta: if os.path.exists(filename): logger.error('A file "%s" already exists, aborting.' % filename) raise Exception self.fh_eeg = open(filename_eeg, 'wb') self.fh_marker = open(filename_marker, 'w') self.fh_meta = open(filename_meta, 'w') # write meta data meta = { 'Channels': self.amp.get_channels(), 'Sampling Frequency': self.amp.get_sampling_frequency(), 'Amp': str(self.amp) } json.dump(meta, self.fh_meta, indent=4) # start the marker server self.marker_queue = Queue() self.tcp_reader_running = Event() self.tcp_reader_running.set() tcp_reader_ready = Event() self.tcp_reader = Process(target=marker_reader, args=(self.marker_queue, self.tcp_reader_running, tcp_reader_ready)) self.tcp_reader.start() logger.debug('Waiting for marker server to become ready...') tcp_reader_ready.wait() logger.debug('Marker server is ready.') # zero the sample counter self.received_samples = 0 # start the amp self.amp.start() def stop(self): # stop the amp self.amp.stop() # stop the marker server self.tcp_reader_running.clear() logger.debug('Waiting for marker server process to stop...') self.tcp_reader.join() logger.debug('Marker server process stopped.') # close the files if self.write_to_file: logger.debug('Closing files.') for fh in self.fh_eeg, self.fh_marker, self.fh_meta: fh.close() def configure(self, **kwargs): self.amp.configure(**kwargs) def get_data(self): """Get data from the amplifier. This method is supposed to get called as fast as possible (i.e hundreds of times per seconds) and returns the data and the markers. Returns ------- data : 2darray a numpy array (time, channels) of the EEG data markers : list of (float, str) a list of markers. Each element is a tuple of timestamp and string. The timestamp is the time in ms relative to the onset of the block of data. Note that negative values are *allowed* as well as values bigger than the length of the block of data returned. That is to be interpreted as a marker from the last block and a marker for a future block respectively. """ # get data and marker from underlying amp data, marker = self.amp.get_data() t = time.time() # length in sec of the new block according to #samples and fs block_duration = len(data) / self.amp.get_sampling_frequency() # abs time of start of the block t0 = t - block_duration # duration of all blocks in ms except the current one duration = 1000 * self.received_samples / self.amp.get_sampling_frequency( ) # merge markers tcp_marker = [] while not self.marker_queue.empty(): m = self.marker_queue.get() m[0] = (m[0] - t0) * 1000 tcp_marker.append(m) marker = sorted(marker + tcp_marker) # save data to files if self.write_to_file: for m in marker: self.fh_marker.write("%f %s\n" % (duration + m[0], m[1])) self.fh_eeg.write(struct.pack("f" * data.size, *data.flatten())) self.received_samples += len(data) if len(data) == 0 and len(marker) > 0: logger.error( 'Received marker but no data. This is an error, the amp should block on get_data until data is available. Marker timestamps will be unreliable.' ) return data, marker def get_channels(self): return self.amp.get_channels() def get_sampling_frequency(self): return self.amp.get_sampling_frequency()
class EyeTracker: """Generic class that is intended to act as a parent class for other implementations. The general concept of processing images to find eyes in them is the same whether the input is a webcam stream, a video, or a series of images. """ def __init__(self, logfile=u'default', facedetect=True, eyedetect=True, \ pupthreshold=50, glintthreshold=200, glintdetect=True, \ pupsizemode=u'diameter', minfacesize=(30,30), Lexpect=(0.7,0.4), \ Rexpect=(0.3,0.4), maxpupdist=0.2, maxpupsize=0.3, maxcpu=6, \ **kwargs): """Initialises an EyeTracker class. Keyword Arguments logfile - A string that indicates the path to the log file. An extension will be added automatically. Default = 'default'. facedetect - A Boolean that indicates whether face detection should be attempted before further processing (eye detection, and pupil/glint detection). Set this to False if you will be using the EyeTracker from close to an eye, in which cases faces need and could not be detected. Default = True. pupthreshold - An integer that indicates what the highest luminance value is that is still considered to be part of the pupil. This value needs to be between 0 and 255. Default = 50. glintthreshold - An integer that indicates what the lowest luminance value is that is still considered to be part of the glint. This value needs to be between 0 and 255. Default = 200. glintdetect - A Boolean that indicates whether the glint (the corneal reflection) should also be detected. Default = True. pupsizemode - A string that indicates how the pupil size should be reported. 'diameter' reports the width of the rect in which the thresholded pupil fits. 'surface' reports the number of thresholded pixels that are assumed to be the pupil. minfacesize - A (w,h) tuple that indicates what size a face should minimally be. Default = (30,30) Lexpect - A (x,y) tuple that indicates where the left eye is expected to be. Note that the coordinates are in relative space, where (0,0) is the top-left of the image, (0,1) is the bottom-left, and (1,1) is the bottom-right. Also note that the left eye is likely to be on the right side of the image, and the right eye is likely to be in the left part of the image. Default = (0.7,0.4) Rexpect - A (x,y) tuple that indicates where the right eye is expected to be. Note that the coordinates are in relative space, where (0,0) is the top-left of the image, (0,1) is the bottom-left, and (1,1) is the bottom-right. Also note that the left eye is likely to be on the right side of the image, and the right eye is likely to be in the left part of the image. Default = (0.3,0.4) maxpupdist - A float that indicates what the maximal allowable distance is between the expected eye position, and the position of detected potential eye. The maximal distance is defined as a proportion of the image height. It can also be set to None. Default = (0.2) maxpupsize - A float that indicates what the maximal allowable width is of the detected eyes. The maximal size is defined as a proportion of the image width. It can also be set to None. Default = (0.3) maxcpu - Integer indicating the maximum amount of parallel processes that will be doing all of the image processing. This happens in parallel to speed things up; the processing time on one CPU can't keep up with the camera's sampling rate. Default = 6. """ # DEBUG message. _message(u'debug', u'generic.EyeTracker.__init__', \ u"Initialising a new EyeTracker.") # GENERAL SETTINGS # Face detection yes/no, and from what size. self._facedetect = facedetect self._minfacesize = minfacesize # Face eye yes/no. self._eyedetect = eyedetect # Eye detection settings. These are relative positions of where # each eye is expected to be in a frame, how far away detected eyes # are allowed to be from the expected locations, and how large the # detected eyes are allowed to be. (All defined as proportions of # the frame's width and height.) self._Lexpect = Lexpect self._Rexpect = Rexpect self._maxpupdist = maxpupdist self._maxpupsize = maxpupsize # Pupil detection thresholds (dark for pupil, bright for glint), # and additional options that determine whether glints should be # detected, and how the pupil size should be reported. self._pupt = pupthreshold self._glit = glintthreshold self._glintdetect = glintdetect self._pupsizemode = pupsizemode # ALIVE EVENT # This event signals whether the tracker is still alive. It should # only be cleared when closing the connection to the tracker! self._alive = Event() self._alive.set() # FRAME OBTAINING THREAD # Boolean that turns to True when a connection with the source of # frames has been established. self._connected = False # We need a Queue for frames that are generated in the obtainer # Thread. The Queue is read out by the parallel processes. self._framequeue = Queue() # We need a lock to prevent potential simultaneous attempts to # access the image source at the same time. This shouldn't actually # be possible in the current implementation, but may be added in # the future. self._sourcelock = Lock() # Start the frame obtaining Thread _message(u'debug', u'generic.EyeTracker.__init__', \ u"Starting a Thread to obtain frames.") self._frame_obtainer_thread = Thread(target=self._frame_obtainer, \ args=[self._alive, self._framequeue]) self._frame_obtainer_thread.name = u'frame_obtainer' self._frame_obtainer_thread.daemon = True self._frame_obtainer_thread.start() # PARALLEL PROCESSING # We need a Queue for samples that are generated in the parallel # processes that are simultaneously processing new frames. self._samplequeue = Queue() # Check how many CPUs we can use. cpus = cpu_count() if cpus > maxcpu: cpus = maxcpu # Start parallel processes to do image processing. _message(u'debug', u'generic.EyeTracker.__init__', \ u"Starting %d parallel processes to process frames into samples." \ % (cpus-1)) self._frame_processes = [] for i in range(1, cpus): p = Process(target=_frame_processer, \ args=[self._alive, self._framequeue, self._samplequeue, \ self._pupt, self._glit, self._facedetect, self._eyedetect, \ self._minfacesize, self._Lexpect, self._Rexpect, \ self._maxpupdist, self._maxpupsize, self._glintdetect, \ self._pupsizemode]) p.name = u'frame_processor_%d' % (i) p.daemon = True p.start() self._frame_processes.append(p) # SAMPLE WRITING # Variable that keeps track of the latest sample. self._latest_sample = [0, numpy.zeros((2, 5)) * numpy.NaN] # Boolean that signals whether the recording Thread should be # active or not. self._recording = False # Lock to prevent simultaneous access to the log file. self._loglock = Lock() # The log file is an open text file. It will be opened when # self._start_recording is called, and it will be closed when # self._stop_recording is called. Between calling those two # functions, samples will be appended to the log. To prevent # samples from being appended to an existing log file, here we # open a new logfile with in 'w' mode, thereby erasing any existing # content of a previous log file. This means users need to be # careful when naming their files, to prevent overwriting. self._logfilename = u'%s.tsv' % (logfile) _message(u'debug', u'generic.EyeTracker.__init__', \ u"Creating new logfile '%s'." \ % (self._logfilename)) # Create a header for the log file. l = [u'time'] l.extend([u'Lpx', u'Lpy', u'Lps', u'Lgx', u'Lgy']) l.extend([u'Rpx', u'Rpy', u'Rps', u'Rgx', u'Rgy']) line = u'\t'.join(map(unicode, l)) + u'\n' # Create a new log file. self._loglock.acquire(True) self._logfile = open(self._logfilename, u'w') self._logfile.write(line) self._logfile.close() self._loglock.release() # Start the sample logging Thread _message(u'debug', u'generic.EyeTracker.__init__', \ u"Starting a Thread to log samples to file '%s'." \ % (self._logfilename)) self._sample_logging_thread = Thread(target=self._sample_logger, \ args=[self._alive, self._samplequeue]) self._sample_logging_thread.name = u'sample_logger' self._sample_logging_thread.daemon = True self._sample_logging_thread.start() # CUSTOM INITIALISATION # Run the custom initialisation procedure. self.connect(**kwargs) def close(self, **kwargs): """Use this function to implement the ending of a specific type of eye tracking. """ # Only close the connection EyeTracker if it isn't closed yet. if self._connected: # Signal the _frame_obtainer that the tracker isn't connected # anymore. self._connected = False # Signal all Threads and Processes that they should stop. self._alive.clear() # Wait for all frame-processing Processes to join. for p in self._frame_processes: p.join() # Wait for the frame-obtaining Thread to join. self._frame_obtainer_thread.join() # Wait for the logging Thread to join. self._sample_logging_thread.join() # Call the custom closing implementation. self._close(**kwargs) def connect(self, **kwargs): """Use this function to implement the initialisation of a specific type of eye tracking. """ # Only intialise the EyeTracker if it isn't initialised yet. if not self._connected: # Signal the frame obtainer that we are connected! self._connected = True # CUSTOM IMPLEMENTATION HERE def is_connected(self): """Tells you whether the EyeTracker is still connected with its source of images. Returns connected - A Boolean that indicates whether the tracker is still connected. """ return self._connected def sample(self): """Returns the newest sample. Returns t, [L, R] - t is a timestamp associated with the latest sample. [L, R] is a NumPy array that contains the following: L = numpy.array([px, py, ps, gx, gy]) R = numpy.array([px, py, ps, gx, gy]) px is an integer indicating the pupil's likely horizontal position within the image. py is an integer indicating the pupil's likely vertical position within the image. ps is an integer indicating the pupil's size in pixels. The size reflects the width of the rect in which the pupil falls when mode=='diameter', or the total amount of thresholded pixels when mode'surface'. gx is an integer indicating the pupil's likely horizontal position within the image. gy is an integer indicating the pupil's likely vertical position within the image. All of these values can be None if they could not be obtained. """ # Return the latest sample. # TODO: Potential issue here is that this sample is at least five # samples old. Maybe we could peek ahead in the Queue, and actually # get the latest? return self._latest_sample def log(self, msg): """Logs a message to the log file. Arguments msg - A string that should be added to the log file. """ # Make sure that the message is in a unicode format. msg = msg.decode(u'utf-8') # A message is a time-stamped sample, with a slightly different # content than normal samples. In essence, the log message sample # is pretending to be a real sample, with a list for the 'left' # and the 'right' eye. It seems a bit ridiculous, but the nested # lists are there to prevent the strings from being pulled apart # when self._log_sample tries to turn them into lists. sample = [time.time(), [[u'MSG'], [msg]]] # Put the sample in the Queue. self._samplequeue.put(sample) def start_recording(self): """Starts the writing of samples to the log file. """ # Only start recording if it isn't currently active. if not self._recording: _message(u'debug', u'generic.EyeTracker.__init__', \ u"Starting recording, and re-opening logfile '%s'." \ % (self._logfilename)) # Signal the recording thread to start. self._recording = True # Re-open the logfile. self._loglock.acquire(True) self._logfile = open(self._logfilename, u'a') self._loglock.release() def stop_recording(self): """Pauses the writing of samples to the log file. """ # Only pause recording if recording is currently active. if self._recording: # Signal the recording Thread to stop what it's doing. self._recording = False # Wait for a bit, to allow the emptying of the local queue. time.sleep(0.2) # Close the logfile. self._loglock.acquire(True) self._logfile.close() self._loglock.release() _message(u'debug', u'generic.EyeTracker.__init__', \ u"Stopped recording, and closed logfile '%s'" \ % (self._logfilename)) def _get_frame(self): """Use this function to implement how the EyeTracker should obtain a frame. A frame is supposed to be a two-dimensional image, usually an image from a webcam or a video that was converted to greyscale. Note that a frame should be a numpy.ndarray with shape=(w,h), and dtype='uint8'. In addition to the frame, a success Boolean should be returned by this function. It tells the functions that call _get_frame whether a new frame was available. (See below what the returned values should be exactly.) IMPORTANT: This function should not have any keyword arguments. Any settings should be handled through properties of self. Returns success, frame - success is a Boolean that indicates whether a frame could be obtained. frame is a numpy.ndarray with unsigned, 8-bit integers that reflect the greyscale values of the image. If no frame could be obtained, None will be returned. """ # Obtain a frame. _message(u'message', u'generic.EyeTracker._get_frame', \ u"Implement your own _get_frame functionality") return False, None def _close(self, **kwargs): """Use this function to implement the specifics of closing a connection in your eye-tracking implementation. You could, for example, use it to close the connection to a webcam. This function is automatically called when the close() method is passed, and this setup allows you to pass your own keyword arguments to close (which will then be passed on to _close). """ # CUSTOM IMPLEMENTATION HERE _message(u'message', u'generic.EyeTracker._close', \ u"Implement your own _close functionality") # # # # # # # HELPERS # # # # # # # def _log_sample(self, sample): """Writes a sample to the log file. Arguments sample - A (t, [L, R]) tuple/list. Explanation below: t, [L, R] - t is a timestamp associated with the sample. [L, R] is a NumPy array that contains the following: L = numpy.array([px, py, ps, gx, gy]) R = numpy.array([px, py, ps, gx, gy]) px is an integer indicating the pupil's likely horizontal position within the image. py is an integer indicating the pupil's likely vertical position within the image. ps is an integer indicating the pupil's size in pixels. The size reflects the width of the rect in which the pupil falls when mode=='diameter', or the total amount of thresholded pixels when mode'surface'. gx is an integer indicating the pupil's likely horizontal position within the image. gy is an integer indicating the pupil's likely vertical position within the image. All of these values can be None if they could not be obtained. """ # Compose a string that can be written to the log file. This # contains the timestamp in milliseconds (hence *1000), the left # eye's values, and the right eye's values, all separated by tabs. l = [int(sample[0] * 1000)] l.extend(list(sample[1][0])) l.extend(list(sample[1][1])) line = u'\t'.join(map(unicode, l)) + u'\n' # Write to the log file. Acquire the log lock first, and release it # after writing, to prevent simultaneous writing to the same log. self._loglock.acquire(True) self._logfile.write(line) self._loglock.release() # # # # # # # # # # # # RUNNING PROCESSES # # # # # # # # # # # # def _frame_obtainer(self, event, framequeue): """Continuously tries to get new frames using self._get_frame, and puts the obtained frames in the framequeue. """ # Continuously run. while event.is_set(): # Only try to get frames if a connection is established. if self._connected: # Obtain a new frame, and an associated timestamp. # Make sure to lock the source while obtaining the # frame, to prevent simultaneous access. (This # shouldn't actually happen for the moment, but might # become a feature in the future.) self._sourcelock.acquire(True) success, frame = self._get_frame() t = time.time() self._sourcelock.release() # Put the obtained timestamp and frame in the queue. framequeue.put([t, frame]) # If the tracker is not connected, wait for a bit to avoid # wasting processing resources on continuously checking # whether the tracker is already connected. else: # Pause for 10 milliseconds. time.sleep(0.01) def _sample_logger(self, event, samplequeue): """Continuously monitors the Queue, and writes samples to the log file whenever over five are still in the Queue. """ # Create a list to keep track of samples. timestamps = [] samplelist = [] # Continuously run. while event.is_set(): # Only process samples if the tracker is recording. if self._recording: # Obtain a new sample if the Queue isn't empty, and lock # the Queue while we're using it. if not samplequeue.empty(): # if samplequeue.qsize() > 0: # Get the oldest sample in the Queue. sample = samplequeue.get() # Add the sample to the list. timestamps.append(sample[0]) samplelist.append(sample[1]) # Store the sample locally, but only if it's not # a message. if sample[1][0][0] != 'MSG': self._latest_sample = sample # Write the oldest samples from the list, but make sure # there are always at least five samples still in the # list. We do this, because the sampling happens # asynchronously in several parallel processes. This # might result in newer samples being processed and # becoming available before older samples. Obviously, # we want to log the samples in chronological order of # obtaining them, not of them becoming available. So # we keep a buffer of five samples, which should # hopefully be enough to allow slightly older samples # to come in. while len(timestamps) > 5: # Find the oldest timestamp. i = numpy.argmin(timestamps) t = timestamps.pop(i) LR = samplelist.pop(i) # Log the sample. self._log_sample([t, LR]) # If we're not recording anymore, but there are samples left # in the list, then we need to process those first. elif not self._recording and len(timestamps) > 0: # Empty out the sample buffer. while len(timestamps) > 0: # Find the oldest timestamp. i = numpy.argmin(timestamps) t = timestamps.pop(i) LR = samplelist.pop(i) # Log the sample. self._log_sample([t, LR]) # If the tracker is not recording, wait for a bit to avoid # wasting processing resources on continuously checking # whether the tracker is recording. else: # Pause for 10 milliseconds. time.sleep(0.01)
class AthSpectralScanner(object): def __init__(self, interface): # Set interface, phy, driver and debugfs directory self.interface = interface self.phy = None with open('/sys/class/net/%s/phy80211/name' % interface) as f: self.phy = f.read().strip() self.driver = None self.debugfs_dir = None for dirname, subd, files in os.walk('/sys/kernel/debug/ieee80211'): if 'spectral_scan_ctl' in files: phy = dirname.split(os.path.sep)[-2] if phy == self.phy: self.driver = dirname.split(os.path.sep)[-1] self.debugfs_dir = dirname with open( os.path.sep.join(dirname.split(os.path.sep)[:-1]) + os.path.sep + 'ht40allow_map') as f: self.ht40allow_map_raw = f.read() f.close() break if self.debugfs_dir is None: raise Exception( "Unable to access 'spectral_scan_ctl' file for interface '%s'. " "Maybe you need to adjust the access rights of /sys/kernel/debug/ieee80211 ?" % interface) # Fixme: sudo chmod -R 777 /sys/kernel/debug logger.debug("interface '%s' is on '%s' via %s. debugfs found at %s" % (self.interface, self.phy, self.driver, self.debugfs_dir)) # hardware capabilities: channels to tune to self.channels = [] self._get_supported_channels() logger.debug("interface '%s' supports the channels: %s" % (self.interface, self.channels)) logger.debug("ht40allow map: %s" % self.ht40allow_map_raw.replace('\n', ",")) self.ht40allow_map = dict() for line in self.ht40allow_map_raw.split( '\n'): # 2412 HT40 + \n2432 HT40 -+ \n2484 Disabled freq_mode_allowmap = line.split() if len(freq_mode_allowmap) == 3: self.ht40allow_map[ freq_mode_allowmap[0]] = freq_mode_allowmap[2] # chanscan mode triggers on changed channels. Use Process to run "iw scan" to tune to all channels self.chanscan_process = None self.chanscan_process_exit = Event() # Store current state of the config files self.cfg_files = {} self.cfg_filenames = ("spectral_count", "spectral_fft_period", "spectral_period", "spectral_scan_ctl", "spectral_short_repeat") self._store_former_config() self.mode = self.cfg_files['spectral_scan_ctl']['former_value'] self.need_tear_down = True # fixme self.running = False self.current_freq = 0 # Fixme: read from interface self.current_ht_mode = "HT20" # Fixme: read from interface self.set_channel(1) # Fixme: read from interface def __del__(self): #self.stop() # FIXME pass # suger for set_mode(mode) def set_mode_chanscan(self): self.set_mode("chanscan") def set_mode_background(self): self.set_mode("background") def set_mode_manual(self): self.set_mode("manual") def set_mode_disable(self): self.set_mode("disable") def set_mode(self, mode, skip_interface_config=False): if mode not in ["chanscan", "background", "manual", "disable"]: raise Exception("Unknown mode requested: '%s'" % mode) if mode is "chanscan" and self.mode is not "chanscan": self.mode = mode if not skip_interface_config: logger.debug( "enter 'chanscan' mode: set dev type to 'managed'") os.system("sudo ifconfig %s down" % self.interface) os.system("sudo iw dev %s set type managed" % self.interface) os.system( "sudo ifconfig %s up" % self.interface) # FIXME: does the interface need to be up? self._set_spectral_cfg('spectral_scan_ctl', "chanscan") #self._start_scan_process() -> start() self.need_tear_down = True return self._stop_scan_process( ) # all other modes: kill external "iw scan" (if any) if mode is "background" and self.mode is not "background": self.mode = mode if not skip_interface_config: logger.debug( "enter 'background' mode: set dev type to 'monitor'") os.system("sudo ifconfig %s down" % self.interface) os.system("sudo iw dev %s set monitor fcsfail" % self.interface ) # fcsfail = also report frames with corrupt FCS os.system("sudo ifconfig %s up" % self.interface) # need to be up self._set_spectral_cfg('spectral_scan_ctl', "background") #self._set_spectral_cfg('spectral_scan_ctl', "trigger") -> start() self.need_tear_down = True return if mode is "manual" and self.mode is not "manual": self.mode = mode if not skip_interface_config: logger.debug("enter 'manual' mode: set dev type to 'monitor'") os.system("sudo ifconfig %s down" % self.interface) os.system("sudo iw dev %s set monitor fcsfail" % self.interface ) # fcsfail = also report frames with corrupt FCS os.system("sudo ifconfig %s up" % self.interface) # need to be up self._set_spectral_cfg('spectral_scan_ctl', "manual") self.need_tear_down = True return if mode is "disable" and self.mode is not "disable": self.mode = mode if not skip_interface_config: os.system("sudo ifconfig %s down" % self.interface) os.system("sudo iw dev %s set type managed" % self.interface) self._set_spectral_cfg('spectral_scan_ctl', "disable") # need to trigger() here? ? -> not needed. ath9k_cmn_spectral_scan_config() calls # ath9k_hw_ops(ah)->spectral_scan_config() which unset the AR_PHY_SPECTRAL_SCAN_ENABLE flag if needed self.need_tear_down = True return def trigger(self): self._set_spectral_cfg('spectral_scan_ctl', "trigger") def retrigger(self): if self.mode == "background": self._set_spectral_cfg('spectral_scan_ctl', "trigger") def get_mode(self): return self.mode def get_debugfs_directory(self): return self.debugfs_dir def get_data_filename(self): return self.debugfs_dir + os.path.sep + "spectral_scan0" def get_config(self): cfg = {} for fn in self.cfg_filenames: path = self.debugfs_dir + os.path.sep + fn with open(path, 'r') as f: cfg[fn] = f.read().strip() f.close() cfg['driver'] = self.driver cfg['frequency'] = self.current_freq return cfg def set_channel(self, channel): self._tune(channel=channel) def set_frequency(self, frequency): self._tune(frequency=frequency) def get_supported_freqchan(self): return self.channels def set_HT_mode(self, ht_mode): if ht_mode == "HT20": self.current_ht_mode = ht_mode elif "HT40" in ht_mode: self._set_ht40_mode_for_freq(self.current_freq) else: raise Exception( "unknown value for HT mode: '%s'. valid: HT20, HT40" % count) self.set_channel(self.current_chan) # set new HT mode def _set_ht40_mode_for_freq(self, freq): if not str(freq) in self.ht40allow_map.keys(): self.current_ht_mode = "HT20" return if "+" in self.ht40allow_map[str(freq)]: self.current_ht_mode = "HT40+" else: self.current_ht_mode = "HT40-" # Source of min/max values for parameters: ath9k/spectral-common.c def set_spectral_count(self, count): if count > 255 or count < 0: logger.error( "invalid value for 'spectral_count' of %d. valid: 0-255" % count) return self._set_spectral_cfg('spectral_count', count) def get_spectral_count(self): return int(self._get_spectral_cfg('spectral_count')) def set_spectral_fft_period(self, period): if period > 15 or period < 0: logger.error( "invalid value for 'spectral_fft_period' of %d. valid: 0-15" % period) return self._set_spectral_cfg('spectral_fft_period', period) def get_spectral_fft_period(self): return int(self._get_spectral_cfg('spectral_fft_period')) def set_spectral_period(self, period): if period > 255 or period < 0: logger.error( "invalid value for 'spectral_period' of %d. valid: 0-255" % period) return self._set_spectral_cfg('spectral_period', period) def get_spectral_period(self): return int(self._get_spectral_cfg('spectral_period')) def set_spectral_short_repeat(self, repeat): if repeat > 1 or repeat < 0: logger.error( "invalid value for 'spectral_short_repeat' of %d. valid: 0-1" % repeat) return self._set_spectral_cfg('spectral_short_repeat', repeat) def get_spectral_short_repeat(self): return int(self._get_spectral_cfg('spectral_short_repeat')) def _set_spectral_cfg(self, filenname, value): logger.debug("set '%s' to '%s'" % (filenname, value)) with open(self.cfg_files[filenname]['path'], 'w') as f: f.write("%s" % value) f.close() def _get_spectral_cfg(self, filenname): with open(self.cfg_files[filenname]['path']) as f: cfg = f.read() f.close() return cfg def start(self): self.running = True if self.mode is "chanscan": self._start_scan_process() else: self._set_spectral_cfg('spectral_scan_ctl', "trigger") def stop(self): self.running = False self.set_mode("disable") if self.need_tear_down: self._restore_former_config() self.need_tear_down = False self._stop_scan_process() def _tune(self, channel=None, frequency=None): if channel is None and frequency is None: raise Exception("need channel or frequency") if self.mode is "chanscan": logger.warning( "Manual set of channel/frequency gets probably overwritten in chanscan mode." ) for i in range(0, len(self.channels)): (freq, chan) = self.channels[i] if chan == channel or freq == frequency: self.current_freq = freq self.current_chan = chan if self.current_ht_mode != "HT20": self._set_ht40_mode_for_freq(freq) logger.debug("set freq to %d in mode %s" % (freq, self.current_ht_mode)) os.system("sudo iw dev %s set freq %d %s" % (self.interface, freq, self.current_ht_mode)) if self.running: self._set_spectral_cfg( 'spectral_scan_ctl', "trigger" ) # need to trigger again after switch channel return logger.warning( "can not tune to unsupported channel %d / frequency %d. " "Supported channels: %s" % channel, frequency, self.channels) # FIXME: add interface config, use with open def _store_former_config(self): for fn in self.cfg_filenames: path = self.debugfs_dir + os.path.sep + fn f = open(path, 'r') val = f.read().strip() f.close() self.cfg_files[fn] = {'path': path, 'former_value': val} logger.debug("read config for '%s': '%s'" % (path, val)) def _restore_former_config(self): #for fn, stored_cfg in self.cfg_files.iteritems(): python2 # FIXME remove for fn, stored_cfg in self.cfg_files.items(): val = stored_cfg['former_value'] path = stored_cfg['path'] f = open(path, 'w') f.write(val) f.close() logger.debug("restore '%s' to: '%s'" % (path, val)) def _get_supported_channels(self): # parses the supported channels as (channel, freq) from 'iw phy' iw_phy_output = subprocess.check_output(["iw", "phy"]).decode('UTF-8') found_device = False for line in iw_phy_output.split('\n'): line = line.strip() if "Wiphy" in line: if self.phy in line: found_device = True else: found_device = False continue if found_device: if "*" in line and "MHz" in line and "[" in line and "]" in line: try: freq = int(line.split("MHz")[0].split("*")[1].strip()) chan = int(line.split("[")[1].split("]")[0].strip()) self.channels.append((freq, chan)) except Exception as e: raise Exception( "Cant parse freq/channel from line '%s': '%s'" % (line, e)) def _start_scan_process(self): if self.chanscan_process is None: logger.debug("start process chanscan") self.chanscan_process_exit.clear() self.chanscan_process = Process(target=self._scan, args=()) self.chanscan_process.start() def _stop_scan_process(self): if self.chanscan_process is not None: logger.debug("stop process chanscan") #self.chanscan_process.terminate() # dont work self.chanscan_process_exit.set() self.chanscan_process.join() self.chanscan_process = None def _scan(self): while not self.chanscan_process_exit.is_set(): cmd = 'iw dev %s scan' % self.interface os.system('%s >/dev/null 2>/dev/null' % cmd) # call blocks
class TestRestFirmware(TestAcceptanceBase): def setUp(self): super().setUp() self.analysis_finished_event = Event() self.elements_finished_analyzing = Value('i', 0) self.db_backend_service = BackEndDbInterface(config=self.config) self._start_backend(post_analysis=self._analysis_callback) self.test_container_uid = '418a54d78550e8584291c96e5d6168133621f352bfc1d43cf84e81187fef4962_787' time.sleep(2) # wait for systems to start def tearDown(self): self._stop_backend() self.db_backend_service.shutdown() super().tearDown() def _analysis_callback(self, fo): self.db_backend_service.add_analysis(fo) self.elements_finished_analyzing.value += 1 if self.elements_finished_analyzing.value == 4 * 3: # container including 3 files times 3 plugins self.analysis_finished_event.set() def _rest_upload_firmware(self): data = get_firmware_for_rest_upload_test() rv = self.test_client.put('/rest/firmware', data=json.dumps(data), follow_redirects=True) self.assertIn(b'"status": 0', rv.data, 'rest upload not successful') self.assertIn(self.test_container_uid.encode(), rv.data, 'uid not found in rest upload reply') def _rest_get_analysis_result(self): rv = self.test_client.get('/rest/firmware/{}'.format( self.test_container_uid), follow_redirects=True) self.assertIn(b'analysis_date', rv.data, 'rest analysis download not successful') self.assertIn(b'software_components', rv.data, 'rest analysis not successful') self.assertIn(b'"device_part": "test_part"', rv.data, 'device part not present') def _rest_search(self): rv = self.test_client.get('/rest/firmware?query={}'.format( urllib.parse.quote('{"device_class": "test_class"}')), follow_redirects=True) self.assertIn(self.test_container_uid.encode(), rv.data, 'test firmware not found in rest search') def _rest_search_fw_only(self): query = json.dumps({'sha256': self.test_container_uid.split('_')[0]}) rv = self.test_client.get('/rest/firmware?query={}'.format( urllib.parse.quote(query)), follow_redirects=True) self.assertIn(self.test_container_uid.encode(), rv.data, 'test firmware not found in rest search') def _rest_update_analysis_bad_analysis(self): rv = self.test_client.put('/rest/firmware/{}?update={}'.format( self.test_container_uid, urllib.parse.quote('["unknown_system"]')), follow_redirects=True) self.assertIn( 'Unknown analysis system'.encode(), rv.data, "rest analysis update should break on request of non existing system" ) def _rest_update_analysis_success(self): rv = self.test_client.put('/rest/firmware/{}?update={}'.format( self.test_container_uid, urllib.parse.quote(json.dumps(['crypto_material']))), follow_redirects=True) self.assertNotIn(b'error_message', rv.data, 'Error on update request') def _rest_check_new_analysis_exists(self): rv = self.test_client.get('/rest/firmware/{}'.format( self.test_container_uid), follow_redirects=True) response_data = json.loads(rv.data.decode()) assert response_data['firmware']['analysis']['crypto_material'] assert response_data['firmware']['analysis']['crypto_material'][ 'analysis_date'] > response_data['firmware']['analysis'][ 'software_components']['analysis_date'] def test_run_from_upload_to_show_analysis_and_search(self): self._rest_upload_firmware() self.analysis_finished_event.wait(timeout=15) self.elements_finished_analyzing.value = 4 * 2 # only one plugin to update so we offset with 4 times 2 plugins self.analysis_finished_event.clear() self._rest_get_analysis_result() self._rest_search() self._rest_search_fw_only() self._rest_update_analysis_bad_analysis() self._rest_update_analysis_success() self.analysis_finished_event.wait(timeout=10) self._rest_check_new_analysis_exists()
def main(): args = parse_args() timeout = args.timeout ## start crawler # crawler_noticer = Event() # crawler_noticer.clear() # result_noticer = Event() # result_noticer.clear() # qreader, qwriter = Pipe() # stdreader, stdwriter = Pipe() # crawler = multiprocessing.Process( # target=crawler_daemon, # args=(crawler_noticer, qreader, result_noticer, stdwriter) # ) # crawler.daemon = True # crawler.start() adb_bin = get_adb_tool() if use_monitor: os.system("{0} connect 127.0.0.1:62001".format(adb_bin)) check_screenshot(filename="screenshot.png", directory=data_directory) if enable_chrome: closer = Event() noticer = Event() closer.clear() noticer.clear() reader, writer = Pipe() browser_daemon = multiprocessing.Process( target=run_browser, args=(closer, noticer, reader,)) browser_daemon.daemon = True browser_daemon.start() def __inner_job(): start = time.time() text_binary = analyze_current_screen_text( directory=data_directory, compress_level=image_compress_level[0], crop_area=crop_areas[game_type], use_monitor=use_monitor ) keywords = get_text_from_image( image_data=text_binary, ) if not keywords: print("text not recognize") return true_flag, real_question, question, answers = parse_question_and_answer( keywords) ## notice crawler to work # qwriter.send(real_question.strip("?")) # crawler_noticer.set() print('-' * 72) print(real_question) print('-' * 72) print("\n".join(answers)) # notice browser if enable_chrome: writer.send(question) noticer.set() search_question = pre_process_question(question) summary = baidu_count(search_question, answers, timeout=timeout) summary_li = sorted( summary.items(), key=operator.itemgetter(1), reverse=True) data = [("选项", "同比")] for a, w in summary_li: data.append((a, w)) table = AsciiTable(data) print(table.table) print("*" * 72) if true_flag: print("肯定回答(**): ", summary_li[0][0]) print("否定回答( ): ", summary_li[-1][0]) else: print("肯定回答( ): ", summary_li[0][0]) print("否定回答(**): ", summary_li[-1][0]) print("*" * 72) # try crawler # retry = 4 # while retry: # if result_noticer.is_set(): # print("~" * 60) # print(stdreader.recv()) # print("~" * 60) # break # retry -= 1 # time.sleep(1) # result_noticer.clear() print("~" * 60) print(kwquery(real_question.strip("?"))) print("~" * 60) end = time.time() print("use {0} 秒".format(end - start)) save_screen( directory=data_directory ) print(""" 请选择答题节目: 1. 百万英雄 2. 冲顶大会 3. 芝士超人 """) game_type = input("输入节目序号: ") if game_type == "1": game_type = '百万英雄' elif game_type == "2": game_type = '冲顶大会' elif game_type == "3": game_type = "芝士超人" else: game_type = '百万英雄' while True: print(""" 请在答题开始前就运行程序, 答题开始的时候按Enter预测答案 """) print("当前选择答题游戏: {}\n".format(game_type)) enter = input("按Enter键开始,按ESC键退出...") if enter == chr(27): break try: __inner_job() except Exception as e: print(str(e)) print("欢迎下次使用") if enable_chrome: reader.close() writer.close() closer.set() time.sleep(3)
class IWebCamStreamProcessing: """ Webcam stream processing interface. Takes images from webcam, process it via NDL API and visualize results :param service: NDL API service """ def __init__(self, service): self.service = service # Assign streaming function to service. This function have to yield images and will be called from NDL API core self.service.set_streaming_function(self.iterate_webcam_images, 'image') # Create images and result sources for communication between several processes self.images_queue, self.result_queue = Queue(), Queue() # Create event for stopping the system self.stop_event = Event() self.stop_event.clear() def iterate_webcam_images(self): """ Iterates over images stream. Takes images from input source (i.e. queue) and send it to NDL API Raises Empty if there are no images to send for at least 5 seconds """ try: # Do until somebody stop event while not self.stop_event.is_set(): # Get image from input source image = self.images_queue.get(timeout=5) # Yield image for NDL API yield image except Empty: print('Webcam timeout exceeded. Aborting.') raise except: print('Exception in iterating webcam images. Aborting') raise finally: print('Webcam iterating was successfully stopped') def iterate_api_responses(self): """ Iterates over NDL API responses Takes response from NDL API, translates it to the correct format and puts to the target source (i.e. queue) Raises BaseException in case of postprocessing error """ try: # Do until service stop processing for response in self.service.process_stream(): # Check response status if response[1] is None: print(response[2]) continue # Get result and translate it to the correct format result = { i: json.loads(image_res.result) for i, image_res in enumerate(response[1]) } processed_result = self.service._postprocess_result(result) # Put result to target source self.result_queue.put(processed_result) # Check system is not stopped if self.stop_event.is_set(): break except: print("Exception while iterating API response") raise finally: self.stop_event.set() def start_streaming(self): """ Start webcam streaming. Open webcam stream and process images from it Creates additional process to iterate over NDL API responses :return: """ # Create and start parallel process for iterating over NDL API response response_iterating_process = Process(target=self.iterate_api_responses) response_iterating_process.daemon = True response_iterating_process.start() # Last processed result last_result, wait_for_result = None, False # Open webcam stream cap = cv2.VideoCapture(0) # Get first frame from webcam stream ret, frame = cap.read() # Check that webcam works normally and nobody stopped the system while ret and not self.stop_event.is_set(): # Check if last putted images already processed if self.images_queue.empty() and not wait_for_result: self.images_queue.put(frame) wait_for_result = True # Check if there is new processing result if not self.result_queue.empty(): last_result = self.result_queue.get() wait_for_result = False # Visualize result on frame vis_frame = self.visualize_result(frame, last_result) # Show frame cv2.imshow('webcam', vis_frame) # Wait 50ms for any key pressed. Stops system if 'q' was pressed. if cv2.waitKey(50) & 0xff == ord('q'): break # Read next frame ret, frame = cap.read() # Stop system self.stop_event.set() # Release webcam cap.release() print('Webcam capture released') def visualize_result(self, image, res=None): """ Visualizes data on image :param image: image from webcam stream :param res: result from NDL API :return: image with visualized data """ raise NotImplementedError
class Core(Process): __SESSIONFILENAME = 'session.ini' __CONFIGFILENAME = 'config.ini' DEFAULT_LANGUAGE = 'english' DEFAULT_USERNAME = '******' DEFAULT_PASSWORD = '******' DEFAULT_STORAGEDIRNAME = 'downloads' DEFAULT_LOGDIRNAME = 'logs' DEFAULT_LOGFILENAME = 'log.txt' def _init_consolelogger(self): if False and self.config.get('log', 'color_console') and ismodule('colorlog'): fmt = "%(label)s %(levelname)-8s %(reset)s %(log_color)s%(asctime)s %(message)s" datefmt = "%Y-%m-%d %H:%M:%S" primary_colors = { 'DEBUG': "bold,cyan", 'WARNING': "bold,yellow", 'ERROR': "bold,red", 'CRITICAL': "bold,purple", } secondary_colors = { 'label': { 'DEBUG': "bold,white,bg_cyan", 'INFO': "bold,white,bg_green", 'WARNING': "bold,white,bg_yellow", 'ERROR': "bold,white,bg_red", 'CRITICAL': "bold,white,bg_purple", } } consoleform = colorlog.ColoredFormatter( fmt, datefmt, log_colors=primary_colors, secondary_log_colors=secondary_colors) else: fmt = "%(asctime)s %(levelname)-8s %(message)s" datefmt = "%Y-%m-%d %H:%M:%S" consoleform = logging.Formatter(fmt, datefmt) consolehdlr = logging.StreamHandler(sys.stdout) consolehdlr.setFormatter(consoleform) self.log.addHandler(consolehdlr) def _init_syslogger(self): # try to mimic to normal syslog messages fmt = "%(asctime)s %(name)s: %(message)s" datefmt = "%b %e %H:%M:%S" syslogform = logging.Formatter(fmt, datefmt) syslogaddr = None syslog = self.config.get('log', 'syslog') if syslog == 'remote': syslog_host = self.config.get('log', 'syslog_host') syslog_port = self.config.get('log', 'syslog_port') syslogaddr = (syslog_host, syslog_port) else: syslog_folder = self.config.get('log', 'syslog_folder') if syslogaddr: syslogaddr = syslog_folder elif sys.platform == 'darwin': syslogaddr = '/var/run/syslog' elif os.name != 'nt': syslogaddr = '/dev/log' sysloghdlr = logging.handlers.SysLogHandler(syslogaddr) sysloghdlr.setFormatter(syslogform) self.log.addHandler(sysloghdlr) def _init_filelogger(self): fmt = "%(asctime)s %(levelname)-8s %(message)s" datefmt = "%Y-%m-%d %H:%M:%S" fileform = logging.Formatter(fmt, datefmt) logfile_folder = self.config.get('log', 'logfile_folder') if not logfile_folder: logfile_folder = self.DEFAULT_LOGDIRNAME makedirs(logfile_folder, exist_ok=True) logfile_name = self.config.get('log', 'logfile_name') if not logfile_name: logfile_name = self.DEFAULT_LOGFILENAME logfile = os.path.join(logfile_folder, logfile_name) if self.config.get('log', 'rotate'): logfile_size = self.config.get('log', 'logfile_size') << 10 max_logfiles = self.config.get('log', 'max_logfiles') filehdlr = logging.handlers.RotatingFileHandler( logfile, maxBytes=logfile_size, backupCount=max_logfiles, encoding=locale.getpreferredencoding(do_setlocale=False)) else: filehdlr = logging.FileHandler( logfile, encoding=locale.getpreferredencoding(do_setlocale=False)) filehdlr.setFormatter(fileform) self.log.addHandler(filehdlr) # TODO: Extend `logging.Logger` like `..plugin.Log` def _init_logger(self): level = logging.DEBUG if self.debug else logging.INFO # Init logger self.log = logging.getLogger() self.log.setLevel(level) # Set console handler self._init_consolelogger() # Set syslog handler if self.config.get('log', 'syslog') != 'no': self._init_syslogger() # Set file handler if self.config.get('log', 'logfile'): self._init_filelogger() def _setup_permissions(self): if os.name == 'nt': return None change_group = self.config.get('permission', 'change_group') change_user = self.config.get('permission', 'change_user') if change_group: try: group = self.config.get('permission', 'group') set_process_group(group) except Exception as e: self.log.error(self._("Unable to change gid: %s"), str(e)) if change_user: try: user = self.config.get('permission', 'user') set_process_user(user) except Exception as e: self.log.error(self._("Unable to change uid: %s"), str(e)) def set_language(self, lang): localedir = resource_filename(__package__, 'locale') lc = locale.locale_alias[lang.lower()].split('_', 1)[0] trans = get_translation('core', localedir, (lc, )) try: self._ = trans.ugettext except AttributeError: self._ = trans.gettext def _setup_language(self): self.log.debug("Loading language ...") lang = self.config.get('general', 'language') default = self.DEFAULT_LANGUAGE if not lang: code = locale.getlocale()[0] or locale.getdefaultlocale()[0] lang = default if code is None else code.lower().split('_', 1)[0] try: self.set_language(lang) except Exception as e: if lang == default: raise self.log.warning( self._( "Unable to load `%(lang)s` language, using default " "`%(default)s`: %(error)s", ), { 'lang': lang, 'default': default, 'error': str(e) }, ) self.set_language(default) def _setup_debug(self): if self.__debug is None: debug_log = self.config.get('log', 'debug') verbose_log = self.config.get('log', 'verbose') self.__debug = 0 if not debug_log else 2 if verbose_log else 1 def _start_interface(self, webui=None, rpc=None): # if webui is None: # webui = self.__webui # if rpc is None: # rpc = self.__rpc # TODO: Parse `remote` # if rpc or self.config.get('rpc', 'activated'): # self.log.debug("Activating RPC interface ...") # self.rem.start() # elif not webui: if not webui: webui = True # TODO: Parse remote host:port if isinstance(webui, str): host, port = map(str.strip, webui.rsplit(':', 1)) webui = True else: host, port = (None, None) kwgs = { 'server': self.config.get('webui', 'server'), 'host': host or self.config.get('webui', 'host'), 'port': port or self.config.get('webui', 'port'), 'key': self.config.get('ssl', 'key'), 'cert': self.config.get('ssl', 'cert'), 'ssl': self.config.get('ssl', 'activated') } if webui or self.config.get('webui', 'activated'): from pyload.webui.webserver import WebServer # from .thread.webserver import WebServer self.webserver = WebServer(self) self.webserver.start() time.sleep(1000) # self.svm.add('webui', **kwgs) # self.svm.start() def _init_api(self): from .api import Api self.api = Api(self) def _init_database(self): from .database import DatabaseBackend from .datatype import Permission, Role # TODO: Move inside DatabaseBackend newdb = not os.path.isfile(DatabaseBackend.DB_FILE) self.db = DatabaseBackend(self) self.db.setup() if self.__restore or newdb: self.db.add_user(self.DEFAULT_USERNAME, self.DEFAULT_PASSWORD, Role.Admin, Permission.All) if self.__restore: self.log.warning( self._("Restored default login credentials `admin|pyload`")) def _init_managers(self): from .manager import (AccountManager, AddonManager, EventManager, ExchangeManager, FileManager, InfoManager, PluginManager, TransferManager, RemoteManager) self.scheduler = sched.scheduler(time.time, time.sleep) self.filemanager = self.files = FileManager(self) self.pluginmanager = self.pgm = PluginManager(self) self.exchangemanager = self.exm = ExchangeManager(self) self.eventmanager = self.evm = EventManager(self) self.accountmanager = self.acm = AccountManager(self) self.infomanager = self.iom = InfoManager(self) self.transfermanager = self.tsm = TransferManager(self) self.addonmanager = self.adm = AddonManager(self) self.remotemanager = self.rem = RemoteManager(self) # self.servermanager = self.svm = ServerManager(self) self.db.manager = self.files # ugly? def _init_requests(self): from .network.factory import RequestFactory self.request = self.req = RequestFactory(self) def _init_config(self): session = ConfigParser(self.__SESSIONFILENAME, session_defaults) flags = portalocker.LOCK_EX | portalocker.LOCK_NB #with session.fp: portalocker.lock(session.fp, flags) profiledir = os.path.join(self.configdir, self.profile) psp = psutil.Process() session.set('current', 'id', time.time()) session.set('current', 'profile', 'path', profiledir) session.set('current', 'profile', 'pid', psp.pid) session.set('current', 'profile', 'ctime', psp.create_time()) self.config = ConfigParser(self.__CONFIGFILENAME, config_defaults) self.session = session def _init_cache(self): # Re-use cache tempdir = self.__tempdir if tempdir is None: tempdir = self.session.get('previous', 'cache', 'path') if tempdir is None or not os.path.isdir(tempdir): pydir = os.path.join(TMPDIR, __namespace__) makedirs(pydir, exist_ok=True) tempdir = tempfile.mkdtemp(dir=pydir) self.session.set('current', 'cache', 'path', tempdir) self.cachedir = tempdir # if tempdir not in sys.path: # sys.path.append(tempdir) def _register_signals(self): shutfn = lambda s, f: self.shutdown() quitfn = lambda s, f: self.terminate() try: if os.name == 'nt': # signal.signal(signal.CTRL_C_EVENT, shutfn) signal.signal(signal.CTRL_BREAK_EVENT, shutfn) else: signal.signal(signal.SIGTERM, shutfn) # signal.signal(signal.SIGINT, shutfn) signal.signal(signal.SIGQUIT, quitfn) # signal.signal(signal.SIGTSTP, lambda s, f: self.stop()) # signal.signal(signal.SIGCONT, lambda s, f: self.run()) except Exception: pass def __init__(self, profiledir=None, tempdir=None, debug=None, restore=None): self.__running = Event() self.__do_restart = False self.__do_shutdown = False self.__debug = debug if debug is None else int(debug) self.__restore = bool(restore) self.__tempdir = tempdir self._ = lambda x: x self._init_profile(profiledir) # if refresh: # cleanpy(PACKDIR) Process.__init__(self) @property def version(self): return __version__ @property def version_info(self): return __version_info__ @property def running(self): return self.__running.is_set() @property def debug(self): return self.__debug def _init_profile(self, profiledir): profiledir = fullpath(profiledir) os.chdir(profiledir) self.configdir, self.profile = os.path.split(profiledir) def _setup_process(self): try: set_process_name('pyLoad') except AttributeError: pass niceness = self.config.get('general', 'niceness') renice(niceness=niceness) ioniceness = int(self.config.get('general', 'ioniceness')) ionice(niceness=ioniceness) def _setup_storage(self): storage_folder = self.config.get('general', 'storage_folder') if not storage_folder: storage_folder = os.path.join(USERDIR, self.DEFAULT_STORAGEDIRNAME) self.config.set('general', 'storage_folder', storage_folder) self.log.debug("Storage: {0}".format(storage_folder)) makedirs(storage_folder, exist_ok=True) avail_space = format.size(availspace(storage_folder)) self.log.info( self._("Available storage space: {0}").format(avail_space)) def _workloop(self): self.__running.set() self.tsm.pause = False # NOTE: Recheck... while True: time.sleep(1) # temporary fix, hopefully self.__running.wait() self.tsm.work() self.iom.work() self.exm.work() if self.__do_restart: raise Restart if self.__do_shutdown: raise Shutdown self.scheduler.run() def _start_plugins(self): # TODO: Move to accountmanager self.log.info(self._("Activating accounts ...")) # self.acm.get_account_infos() # self.scheduler.enter(0, 0, self.acm.get_account_infos) self.adm.activate_addons() def _show_info(self): self.log.info(self._("Welcome to pyLoad v{0}").format(self.version)) self.log.info(self._("Profile: {0}").format(self.profile)) self.log.info(self._("Config directory: {0}").format(self.configdir)) self.log.debug("Cache directory: {0}".format(self.cachedir)) def run(self): self._init_config() self._init_cache() self._setup_debug() self._init_logger() try: self.log.debug("Running pyLoad ...") self._setup_language() self._setup_permissions() self._init_database() self._init_managers() self._init_requests() self._init_api() self._show_info() self._setup_storage() self._start_plugins() self._setup_process() self._start_interface() self.log.info(self._("pyLoad is up and running")) self.evm.fire('pyload:started') # # some memory stats # from guppy import hpy # hp=hpy() # print(hp.heap()) # import objgraph # objgraph.show_most_common_types(limit=30) # import memdebug # memdebug.start(8002) # from meliae import scanner # scanner.dump_all_objects(os.path.join(PACKDIR, 'objs.json')) self._workloop() except Restart: self.restart() except Shutdown: self.shutdown() except (KeyboardInterrupt, SystemExit): self.shutdown() except Exception as e: self.log.critical(str(e)) self.terminate() raise else: self.shutdown() def _remove_loggers(self): for handler in self.log.handlers: with closing(handler) as hdlr: self.log.removeHandler(hdlr) def restart(self): self.stop() self.log.info(self._("Restarting pyLoad ...")) self.evm.fire('pyload:restarting') self.start() def _register_instance(self): profiledir = os.path.join(self.configdir, self.profile) if profiledir in _pmap: raise RuntimeError("A pyLoad instance using profile `{0}` " "is already running".format(profiledir)) _pmap[profiledir] = self def _unregister_instance(self): profiledir = os.path.join(self.configdir, self.profile) _pmap.pop(profiledir, None) def _close_session(self): id = str( self.session.get('previous', 'id') ) # TODO: added str(), is previous still needed at all? maybe delete this and the next line self.session[id] = self.session['previous'] self.session['previous'] = self.session['current'] self.session['current'].reset() self.session.close() def terminate(self): try: self.log.debug("Killing pyLoad ...") self._unregister_instance() self._close_session() finally: # TODO: remove try catch; used to remove debug output as _popen is None try: Process.terminate(self) except Exception: pass def shutdown(self): try: self.stop() self.log.info(self._("Exiting pyLoad ...")) self.tsm.shutdown() self.db.shutdown() # NOTE: Why here? self.config.close() self._remove_loggers() # if cleanup: # self.log.info(self._("Deleting temp files ...")) # remove(self.tempdir, ignore_errors=True) finally: self.terminate() def start(self): if not self.is_alive(): self._register_instance() self._register_signals() Process.start(self) elif not self.running: self.log.info(self._("Starting pyLoad ...")) self.evm.fire('pyload:starting') self.__running.set() def stop(self): if not self.running: return None try: self.log.info(self._("Stopping pyLoad ...")) self.evm.fire('pyload:stopping') self.adm.deactivate_addons() self.api.stop_all_downloads() finally: self.files.sync_save() self.__running.clear() self.evm.fire('pyload:stopped')
q.put(w) if __name__ == "__main__": buffer_size = int(1e+6) env_dict = {"obs": {"shape": 4}, "act": {}, "rew": {}, "next_obs": {"shape": 4}, "done": {}} n_explorer = 4 global_rb = MPPrioritizedReplayBuffer(buffer_size,env_dict) is_training_done = Event() is_training_done.clear() qs = [SimpleQueue() for _ in range(n_explorer)] ps = [Process(target=explorer, args=[global_rb,env_dict,is_training_done,q]) for q in qs] for p in ps: p.start() learner(global_rb,qs) is_training_done.set() for p in ps: p.join()
class ExposureMonitoring(Process): def __init__(self): super().__init__() self.pool = futures.ThreadPoolExecutor(max_workers=1) self.process = None self.exit = Event() self.running = Event() self.ics_last_exposure = {} self.ics = get_qlf_interface() self.process_id = Value('i', 0) def run(self): """ """ while not self.exit.is_set(): time.sleep(1.5) if not self.process or not self.process.running(): self.running.clear() last_exposure = self.ics.last_exposure() try: exposure = last_exposure['exposure'] fibermap = last_exposure['fibermap'] except: logger.debug('No exposure available') continue fibermap = last_exposure.get('fibermap', None) ics_last_expid = self.ics_last_exposure.get('exposure_id', None) if exposure.get('exposure_id') == ics_last_expid: logger.debug('Exposure {} has already been processed'.format( ics_last_expid)) continue self.ics_last_exposure = exposure logger.debug('Exposure {} obtained'.format( exposure.get('exposure_id'))) # records exposure in database exposure_obj = QLFModels().insert_exposure(**exposure) if exposure_obj: fibermap['exposure'] = exposure_obj QLFModels().insert_fibermap(**fibermap) if self.process and self.process.running(): logger.debug('Process {} is running.'.format( str(self.process_id.value))) continue if isinstance(exposure.get('time'), str): exposure['time'] = datetime.datetime.strptime( exposure.get('time'), "%Y-%m-%dT%H:%M:%S.%f") delay = datetime.datetime.utcnow() - exposure.get('time') delay = delay.total_seconds() if delay > allowed_delay: logger.debug( ('The delay in the acquisition of the exposure ' 'went from {} seconds'.format(str(allowed_delay)))) continue logger.info('Exposure {} ({} {}) available.'.format( exposure.get('exposure_id'), exposure.get('program').capitalize(), exposure.get('flavor'))) del self.process gc.collect() self.process = self.pool.submit(run_process, exposure, self.process_id) self.process.add_done_callback(future_callback) self.running.set() logger.debug("Bye!") def shutdown(self): """ Turn off monitoring """ self.exit.set() self.running.clear() self.pool.shutdown() del self.process gc.collect() self.process = None
class H5FileWatcher: def __init__(self, h5_filepath, row_handler, contraints_dict=None): self.handle_event = Event() self.h5_filepath = h5_filepath self.handle_count = 0 self.job_frames = {} self.last_handle_count = None self.row_handler = row_handler self.contraints_clause = '' if contraints_dict is None else ' '.join( 'and {}={}'.format(k, v) for k, v in contraints_dict.items()) Log.d('cc: {}', self.contraints_clause) assert row_handler def handle_change(self): self.handle_count += 1 Log.d('file change event {}', self.handle_count) self.handle_event.set() def ensure_strictly_increasing_index(self, df): p = None for i, r in df.iterrows(): assert p is None or p < i, '{} < {}'.format(p, i) p = i def process_h5(self): with pd.HDFStore(self.h5_filepath, mode='r') as h5: for jobuid in h5: is_first_encounter = jobuid not in self.job_frames if is_first_encounter == True: self.job_frames[jobuid] = pd.read_hdf( h5, jobuid, start=0, stop=1) # fetch first row to get the first index/epoch job_df = self.job_frames[jobuid] latest_epoch = job_df.index.values[ -1] # will ensure we don't 'miss' any rows in case the handle count jumps more than once where_clause = 'index > {} {}'.format(latest_epoch, self.contraints_clause) new_df = pd.read_hdf(h5, jobuid, where=where_clause) if new_df.empty: Log.w('dataset was empty for key {} and index > {}', jobuid, latest_epoch) else: assert new_df.index.values[0] > latest_epoch new_first_index = 0 if is_first_encounter == True else len( job_df) joined = pd.concat([job_df, new_df]) self.job_frames[jobuid] = joined if len(joined) > 100000: Log.w( 'holding a dataset of significant length ({} rows, {:.1f}mb): {}', len(joined), joined.memory_usage().sum() / 1_000_000, jobuid) assert joined.shape[0] == len(job_df) + len(new_df) self.ensure_strictly_increasing_index( joined) # TODO: remove once this is confirmed self.row_handler(jobuid, joined, new_first_index) def trigger(self): self.handle_event.set() def __run(self): Log.d('Watching file: {}', self.h5_filepath) thread = FileWatcher(self.h5_filepath, modified=self.handle_change).run_async() try: while self.handle_event.wait(): if self.last_handle_count is not None: jump_count = self.handle_count - self.last_handle_count if jump_count > 1: Log.w( 'handle count has jumped {} times than once since the last processing', jump_count) self.last_handle_count = self.handle_count self.process_h5() self.handle_event.clear() finally: Log.w('run loop broken, unwatching file: {}', self.h5_filepath) thread.stop() thread.join() def run_async(self): thread = Thread(target=self.__run) thread.start() return thread
class SpectrumIdentificationWorkerBase(Process, SpectrumEvaluatorBase): verbose = False def __init__(self, input_queue, output_queue, producer_done_event, consumer_done_event, scorer_type, evaluation_args, spectrum_map, mass_shift_map, log_handler, solution_packer): Process.__init__(self) if evaluation_args is None: evaluation_args = dict() self.daemon = True self.input_queue = input_queue self.output_queue = output_queue self.producer_done_event = producer_done_event self.consumer_done_event = consumer_done_event self.scorer_type = scorer_type self.evaluation_args = evaluation_args self.solution_packer = solution_packer self.spectrum_map = spectrum_map self.mass_shift_map = mass_shift_map self.local_scan_map = LRUMapping(1000) self.local_mass_shift_map = dict({Unmodified.name: Unmodified}) self.solution_map = dict() self._work_complete = Event() self._work_complete.clear() self.log_handler = log_handler self.token = uid() self.items_handled = 0 self.result_buffer = [] self.buffer_size = 1000 self.last_sent_result = time.time() def log(self, message): """Send a normal logging message via :attr:`log_handler` Parameters ---------- message : str The message to log """ if self.log_handler is not None: self.log_handler(message) def debug(self, message): """Send a debugging message via :attr:`log_handler` Parameters ---------- message : str The message to log """ if self.verbose: self.log_handler("DEBUG::%s" % message) def fetch_scan(self, key): try: return self.local_scan_map[key] except KeyError: serialized_scan = self.spectrum_map[key] scan = pickle.loads(serialized_scan) self.local_scan_map[key] = scan return scan def fetch_mass_shift(self, key): try: return self.local_mass_shift_map[key] except KeyError: mass_shift = self.mass_shift_map[key] self.local_mass_shift_map[key] = mass_shift return mass_shift def all_work_done(self): """A helper method to encapsulate :attr:`_work_complete`'s ``is_set`` method. Returns ------- bool """ return self._work_complete.is_set() def _append_to_result_buffer(self, payload): self.result_buffer.append(payload) if len(self.result_buffer) > self.buffer_size: self._flush_result_buffer() def _flush_result_buffer(self): if self.result_buffer: self.output_queue.put(self.result_buffer) self.result_buffer = [] self.last_sent_result = time.time() def pack_output(self, target): """Transmit the completed identifications for a given target structure. Rather than returning these values directly, this implementation uses the output queue to send the target and its scores along an IPC queue. Parameters ---------- target : object The structure being identified. """ if self.solution_map: target_id = getattr(target, 'id', target) self._append_to_result_buffer( (target_id, self.solution_map, self.token)) self.solution_map = dict() def evaluate(self, scan, structure, evaluation_context=None, *args, **kwargs): raise NotImplementedError() def cleanup(self): """Send signals indicating the worker process is finished and do any final shared resource cleanup needed on the worker's side. This will set the :attr:`_work_complete` event and join :attr:`output_queue` """ self.debug( "... Process %s Setting Work Complete Flag. Processed %d structures" % (self.name, self.items_handled)) try: self._work_complete.set() except (RemoteError, KeyError): self.log("An error occurred while cleaning up worker %r" % (self, )) self._flush_result_buffer() self.output_queue.put(SentinelToken(self.token)) self.consumer_done_event.wait() # joining the queue may not be necessary if we depend upon consumer_event_done self.debug("... Process %s Queue Joining" % (self.name, )) self.output_queue.join() self.debug("... Process %s Finished" % (self.name, )) def before_task(self): '''A method to be overriden by subclasses that want to do something before starting the task loop. ''' pass def task(self): """The worker process's main loop where it will poll for new work items, process incoming work items and send them back to the master process. """ has_work = True self.items_handled = 0 strikes = 0 while has_work: try: payload = self.input_queue.get(True, 5) self.input_queue.task_done() strikes = 0 except QueueEmptyException: if self.producer_done_event.is_set(): has_work = False break else: strikes += 1 self._flush_result_buffer() if strikes % 1000 == 0: self.log("... %d iterations without work for %r" % (strikes, self)) continue if self.items_handled % 100 == 0: if time.time() - self.last_sent_result > 60: self._flush_result_buffer() # Handling a group of work items if isinstance(payload, dict): work_order = payload self.items_handled += 1 try: self.handle_group(work_order) except Exception: message = "An error occurred while processing %r on %r:\n%s" % ( work_order, self, traceback.format_exc()) self.log(message) break else: # Handling a single work item structure, scan_ids = payload self.items_handled += 1 try: self.handle_item(structure, scan_ids) except Exception: message = "An error occurred while processing %r on %r:\n%s" % ( structure, self, traceback.format_exc()) self.log(message) break self.cleanup() def run(self): new_name = getattr(self, 'process_name', None) if new_name is not None: TaskBase().try_set_process_name(new_name) try: self.before_task() except Exception: self.log("An exception occurred during before_task for %r.\n%s" % (self, traceback.format_exc())) try: self.task() except Exception: self.log("An exception occurred while executing %r.\n%s" % (self, traceback.format_exc())) self.cleanup()