def main(): file_changes_q = Queue() # 启动监听线程。 p = Process(target=start_watch_file_changes, args=(file_changes_q,".",)) p.start() while 1: print file_changes_q.get()
def test(): NUMBER_OF_PROCESSES = 4 TASKS1 = [(mul, (i, 7)) for i in range(20)] TASKS2 = [(plus, (i, 8)) for i in range(10)] # Create queues task_queue = Queue() done_queue = Queue() # Submit tasks for task in TASKS1: task_queue.put(task) # Start worker processes for i in range(NUMBER_OF_PROCESSES): Process(target=worker, args=(task_queue, done_queue)).start() # Get and print results print 'Unordered results:' for i in range(len(TASKS1)): print '\t', done_queue.get() # Add more tasks using `put()` for task in TASKS2: task_queue.put(task) # Get and print some more results for i in range(len(TASKS2)): print '\t', done_queue.get() # Tell child processes to stop for i in range(NUMBER_OF_PROCESSES): task_queue.put('STOP')
def test_transfert_queue(): t1 = "testTopic" topic = Topics() q = Queue() topic.process(t1,123) topic.process(t1,456) topic.process(t1,789) assert q.empty() topic.transfer(t1,q) assert q.qsize() > 0 assert q.get() == [0, 123] assert q.get() == [1, 456] assert q.get() == [2, 789] topic.process(t1,111) topic.process(t1,222) assert q.qsize() > 0 assert q.get() == [3, 111] assert q.get() == [4, 222]
def calculate_quality_list(self, voi, gantry, couch, calculate_from=0, stepsize=1.0, avoid=[], gradient=True): """ TODO: Documentation """ q = Queue(32767) process = [] d = voi.get_voi_cube() d.cube = np.array(d.cube, dtype=np.float32) voi_cube = DensityProjections(d) result = [] for gantry_angle in gantry: p = Process( target=self.calculate_angle_quality_thread, args=(voi, gantry_angle, couch, calculate_from, stepsize, q, avoid, voi_cube, gradient)) p.start() p.deamon = True process.append(p) if len(process) > 2: tmp = q.get() result.append(tmp) for p in process: if not p.is_alive(): process.remove(p) while not len(result) == len(gantry) * len(couch): tmp = q.get() result.append(tmp) return result
def genPairs(PNGMaps, compareMaps): pairA = [] pairB = [] # Maximum possible fitness totalFitness = len(compareMaps) threadsA = [] threadsB = [] # Thread safe way to get parent PNGMaps queueA = Queue() queueB = Queue() # Create a list of threads to get a PNGMap for listMap in PNGMaps: threadA = Process(target=randPair , args=(PNGMaps, compareMaps, totalFitness, queueA)) threadB = Process(target=randPair , args=(PNGMaps, compareMaps, totalFitness, queueB)) threadA.start() threadB.start() threadsA.append(threadA) threadsB.append(threadB) # Get the parents from the queues while not len(pairA) == len(PNGMaps): pairA.append(queueA.get()) while not len(pairB) == len(PNGMaps): pairB.append(queueB.get()) # Join the threads with the current one for thread in threadsA: thread.join() for thread in threadsB: thread.join() # Return the pair of PNGMaps return pairA, pairB
def likelihood_mp_simple(seqlens,fss,uon,bon,theta,seqnum,K,ufnum,bfnum,regtype,sigma): global _gradient grad = numpy.array(fss,copy=True) # data distribuition likelihood = numpy.dot(fss,theta) que1 = Queue() # for the likihood output que2 = Queue() # for the gradient output np = 0 subprocesses = [] corenum=multiprocessing.cpu_count() #corenum=1 if corenum>1: chunk=seqnum/corenum+1 else: chunk=seqnum starti=0 while starti < (seqnum): endi=starti+chunk if endi>seqnum: endi=seqnum p = Process(target=likelihoodthread_simple, args=(seqlens[starti:endi],uon[starti:endi],bon[starti:endi],theta,K,ufnum,bfnum,que1,que2)) p.start() np+=1 #print 'delegated %s:%s to subprocess %s' % (starti, endi, np) subprocesses.append(p) starti += chunk for i in range(np): likelihood += que1.get() for i in range(np): grad += que2.get() while subprocesses: subprocesses.pop().join() grad -= regularity_deriv(theta,regtype,sigma) _gradient = grad return likelihood - regularity(theta,regtype,sigma)
def parallel_precompute(global_conf_file=None): # Define queues queueIn = Queue(nb_workers+2) queueOut = Queue(nb_workers+8) queueProducer = Queue() queueFinalizer = Queue() queueConsumer = Queue(nb_workers) # Start finalizer t = Process(target=finalizer, args=(global_conf_file, queueOut, queueFinalizer)) t.daemon = True t.start() # Start consumers for i in range(nb_workers): t = Process(target=consumer, args=(global_conf_file, queueIn, queueOut, queueConsumer)) t.daemon = True t.start() # Start producer t = Process(target=producer, args=(global_conf_file, queueIn, queueProducer)) t.daemon = True t.start() # Wait for everything to be started properly producerOK = queueProducer.get() finalizerOK = queueFinalizer.get() for i in range(nb_workers): consumerOK = queueConsumer.get() print "[parallel_precompute: log] All workers are ready." sys.stdout.flush() # Wait for everything to be finished finalizerEnded = queueFinalizer.get() print "[parallel_precompute: log] Done at {}".format(get_now()) return
class CommunicationQueues(object): """Queues to handle communication between the threads. On the bc side, this is also a logging handler sending log messages to the node side.""" def __init__(self): self.bc_to_node = Queue() self.node_to_bc = Queue() def set(self, bc_to_node=None, node_to_bc=None, queues=None): if bc_to_node: self.bc_to_node = bc_to_node return if node_to_bc: self.node_to_bc = node_to_bc return assert queues.bc_to_node self.bc_to_node = queues.bc_to_node assert queues.node_to_bc self.node_to_bc = queues.node_to_bc def empty_queues(self): print "Emptying queues:" while not self.bc_to_node.empty(): print "BC to node:", self.bc_to_node.get() while not self.node_to_bc.empty(): print "Node to BC:", self.node_to_bc.get() print "Emptying queues done." def get_handler(): return _CommQueueHandler(self.bc_to_node)
def runBacktracking(self, clauseList, partialAssignment, literalList, queue): result = self.partialInterp(clauseList, partialAssignment) if result == 'true': queue.put(True) return if result == 'false': queue.put(False) return chosenLiteral = self.chooseLiteral(partialAssignment, literalList) posDict = dict(partialAssignment) posDict[chosenLiteral] = 'true' negDict = dict(partialAssignment) negDict[chosenLiteral] = 'false' q1 = Queue() q2 = Queue() thread1 = Thread(target=self.runBacktracking, args=(copy.deepcopy(clauseList), posDict, list(literalList), q1)) thread1.start() thread2 = Thread(target=self.runBacktracking, args=(copy.deepcopy(clauseList), negDict, list(literalList), q2)) thread2.start() thread1.join() thread2.join() if(q1.get()): queue.put(True) return if(q2.get()): queue.put(True) return queue.put(False) return
def main(): if __name__ == '__main__': queue = Queue() p1 = Process(target=sum, args=(queue, 0, 2500000)) p2 = Process(target=sum, args=(queue, 2500000, 5000000)) p3 = Process(target=sum, args=(queue, 5000000, 7500000)) p4 = Process(target=sum, args=(queue, 7500000, 10000001)) now_1 = time.time() p1.start() now_2 = time.time() now_3 = time.time() p2.start() now_4 = time.time() now_5 = time.time() p3.start() now_6 = time.time() now_7 = time.time() p4.start() now_8 = time.time() p1.join() p2.join() p3.join() p4.join() total_result = queue.get() + queue.get() print(total_result) print(now_2 - now_1) print(now_4 - now_3) print(now_6 - now_5) print(now_8 - now_7)
def process_labelit(self, inp=False): """ Initiate Labelit runs. """ if self.verbose: self.logger.debug("LabelitPP::process_labelit") try: queue = Queue() params = {} params["test"] = self.test params["cluster"] = False # self.cluster_use # TODO params["verbose"] = self.verbose args1 = {} if inp: args1["input"] = inp else: args1["input"] = self.images args1["output"] = queue args1["params"] = params args1["logger"] = self.logger # Import the RunLabelit class agent = load_module(seek_module="rapd_agent_index+strategy", directories=self.agent_directories, logger=self.logger) Process(target=agent.RunLabelit, kwargs=args1).start() self.labelit_results = queue.get() self.labelit_log = queue.get() except: self.logger.exception("**Error in LabelitPP.process_labelit**")
class DataLoaderOnTheFly(): def __init__(self, config): default_config = Config(proc_count = 4, limit_batch_count = None) self.config = default_config(**config) self.exit = Event() self.batch_queue = Queue(maxsize = 10) if self.config.limit_batch_count is None: self.limited = False else: self.limited = True self.batch_list = [] self.index = -1 self.workers = [] for _ in range(self.config.proc_count): self.workers.append(Process(target = config.worker, args = (self,))) for w in self.workers: w.daemon = True w.start() def next_batch(self): if self.limited: if len(self.batch_list) < self.config.limit_batch_count: self.batch_list.append(Config(self.batch_queue.get())) self.index = (self.index + 1) % self.config.limit_batch_count return Config(self.batch_list[self.index]) else: return Config(self.batch_queue.get()) def __del__(self): self.exit.set() for w in self.workers: w.join()
def split_messages(d, n, nPrime, r, bit,data): """ Splits a data set based on the subtraction in montgomery exponentiation.""" mlist = data q_t = Queue() q_f = Queue() processes = [] start = 0 numProcs = 8 NP = 0 chunk = len(mlist)//numProcs while start < len(mlist): p = Process(target=do_sim, args=(q_t, q_f, mlist[start:start+chunk], d, n, nPrime, r, bit)) NP += 1 p.start() start += chunk processes.append(p) m_true = [] m_false = [] for i in range(NP): m_true += q_t.get() m_false += q_f.get() while processes: processes.pop().join() return (m_true, m_false)
def testBackTracking(self): testData = [[-1, -2], [1, -2], [-1, -3]] testLiterals = [1, 2, 3] queue0 = Queue() self.dpllTester.runBacktracking(testData, {}, testLiterals, queue0) if queue0.get() == False: print "Error" else: print "Passed Test 4" queue1 = Queue() testData = [[-1, -2], [-1, 2], [1, -2], [2, -3], [1, 3]] self.dpllTester.runBacktracking(testData, {}, testLiterals, queue1) if queue1.get() == True: print "Error" else: print "Passed Test 5" queue2 = Queue() testData = [[2, 1], [-1], [-2, -3], [3, 1]] self.dpllTester.runBacktracking(testData, {}, testLiterals, queue2) if queue2.get() == True: print "Error" else: print "Passed Test 6"
def parallel_return_function(function, args_list, threads): """ Run function using multiple threads """ values = [] processes = [] queue = Queue() for arguments in args_list: # run function for each set of args in args_list arguments['queue'] = queue # append queue to list of args p = Process(target=function, kwargs=arguments) processes.append(p) processes[-1].start() while len(processes) >= threads: # control number of active processes indexes = [] for index, process in enumerate(processes): if process.is_alive(): # keep processes that are still alive indexes.append(index) else: # return values from processes that are finished values.append(queue.get()) processes = [processes[i] for i in indexes] # update list of processes # wait until there are no active processes while len(processes) > 0: indexes = [] for index, process in enumerate(processes): if process.is_alive(): # keep processes that are still alive indexes.append(index) else: # return values from processes that are finished values.append(queue.get()) processes = [processes[i] for i in indexes] # update list of processes return values
def createMatrix(messageList,vocabList):#输入短信列表和词库 返回短信-短信向量矩阵 t1 = time.time() print "开始构建向量矩阵..." num = len(messageList) n = num/2 def Pro(N1,N2,getReturn=None,isP=False): returnMatrix = [] if isP: for i in range(num)[N1:N2]: print " 构建中... %i%%"%(i/float(n)*100), sys.stdout.write("\r") returnMatrix.append(createVecInVocabList(messageList[i],vocabList=vocabList)) else: for i in range(num)[N1:N2]: returnMatrix.append(createVecInVocabList(messageList[i],vocabList=vocabList)) getReturn.put(returnMatrix) # return returnMatrix getReturn1 = Queue();getReturn2 = Queue() P1 = Process(target=Pro,args=(0,n,getReturn1,True)) P2 = Process(target=Pro,args=(n,2*n,getReturn2)) # P3 = Process(target=Pro,args=(2*n,3*n,getReturn3)) # P4 = Process(target=Pro,args=(3*n,4*n,getReturn4)) P1.start() P2.start() matrix = getReturn1.get() + getReturn2.get() # matrix = Pro(0,num,None,True) t2 = time.time() print "矩阵构建完毕,花费时间:%is\n"%(t2-t1) return matrix
class chan: def __init__(self): self._queue = Queue() self._closed = False def __iter__(self): return self def __next__(self): try: return self.get() except queue.Empty: raise StopIteration def put(self, x): self._queue.put(x) def get(self): if self._closed: return self._queue.get(False) else: return self._queue.get() def close(self): self._closed = True
def main(): l=Lock() dat = Queue() pool=Pool(processes=2) # pool.apply_async(produce_number, [l,dat]) p1 = Process(target=produce_number, args=(l,dat,)) p2 = Process(target=produce_number, args=(l,dat,)) p3 = Process(target=produce_number, args=(l,dat,)) p4 = Process(target=produce_number, args=(l,dat,)) p5 = Process(target=produce_number, args=(l,dat,)) p1.start() p2.start() p3.start() p4.start() p5.start() # p1.join() # p1.join() # p1.join() # p1.join() while True: if dat.empty(): pass else: print dat.get() p1.join() p1.join() p1.join() p1.join()
class CommunicationChannels(object): '''Bi directional communication channel ''' def __init__(self): self.qin = Queue() self.qout = Queue() def set_child(self): q = self.qin self.qin = self.qout self.qout = q def close(self): self.qin.close() self.qout.close() def dump(self, obj): self.qout.put(obj, block=True) confirm = self.qin.get() assert confirm def load(self, conn=None): res = self.qin.get() self.qout.put(True) return res
def test_worker(self): tournament = axelrod.Tournament( name=self.test_name, players=self.players, game=self.game, turns=200, repetitions=self.test_repetitions) work_queue = Queue() chunks = tournament.match_generator.build_match_chunks() count = 0 for chunk in chunks: work_queue.put(chunk) count += 1 work_queue.put('STOP') done_queue = Queue() tournament._worker(work_queue, done_queue) for r in range(count): new_matches = done_queue.get() for index_pair, matches in new_matches.items(): self.assertIsInstance(index_pair, tuple) self.assertEqual(len(matches), self.test_repetitions) queue_stop = done_queue.get() self.assertEqual(queue_stop, 'STOP')
def multi_threaded_generator(generator, num_cached=10, num_threads=4): queue = MPQueue(maxsize=num_cached) # define producer (putting items into queue) def producer(): for item in generator: queue.put(item) # pretend we are doing some calculations # sleep(0.5) queue.put("end") # start producer (in a background thread) threads = [] for _ in xrange(num_threads): np.random.seed() threads.append(Process(target=producer)) threads[-1].daemon = True threads[-1].start() # run as consumer (read items from queue, in current thread) # print "starting while" item = queue.get() while item != "end": # print len(item) yield item item = queue.get() queue.close()
def make_work(callback, tasks, limit, ignore_exceptions=True, taskq_size=50): """ Run up to "limit" processes, do tasks and yield results. :param callback: the function that will process single task :param tasks: the sequence or iterator or queue of tasks, each task in turn is sequence of arguments, if task is just signle argument it should be wrapped into list or tuple :param limit: the maximum number of processes """ # If tasks is number convert it to the list of number if isinstance(tasks, int): tasks = xrange(tasks) # Ensure that tasks sequence is iterator tasks = iter(tasks) taskq = Queue(taskq_size) # Here results of task processing will be saved resultq = Queue() # Prepare and run up to "limit" processes processes = [] for x in xrange(limit): process = Worker(callback, taskq, resultq, ignore_exceptions) process.daemon = True process.start() processes.append(process) # Put tasks from tasks iterator to taskq queue # until tasks iterator ends # Do it in separate process def task_processor(task_iter, task_queue, limit): try: for task in task_iter: task_queue.put(task) finally: for x in xrange(limit): task_queue.put(STOP) processor = Process(target=task_processor, args=[tasks, taskq, limit]) processor.daemon = True processor.start() while True: try: yield resultq.get(True, 0.2) except Empty: pass if not any(x.is_alive() for x in processes): break while True: try: yield resultq.get(False) except Empty: break
def test_warc_writer_locking(tmpdir): """Test if WarcWriter is locking WARC files. When we don't have the .open suffix, WarcWriter locks the file and the external process trying to ``lock_file`` fails (result=0). """ recorder = ProxyingRecorder(None, None, 'sha1', url='http://example.com') recorded_url = RecordedUrl( url='http://example.com', content_type='text/plain', status=200, client_ip='127.0.0.2', request_data=b'abc', response_recorder=recorder, remote_ip='127.0.0.3', timestamp=datetime.utcnow(), payload_digest=hashlib.sha1()) dirname = os.path.dirname(str(tmpdir.mkdir('test-warc-writer'))) wwriter = WarcWriter(Options( directory=dirname, no_warc_open_suffix=True)) wwriter.write_records(recorded_url) warcs = [fn for fn in os.listdir(dirname) if fn.endswith('.warc')] assert warcs target_warc = os.path.join(dirname, warcs[0]) # launch another process and try to lock WARC file q = Queue() p = Process(target=lock_file, args=(q, target_warc)) p.start() p.join() assert q.get() == 'FAILED TO OBTAIN LOCK' wwriter.close() # locking must succeed after writer has closed the WARC file. p = Process(target=lock_file, args=(q, target_warc)) p.start() p.join() assert q.get() == 'OBTAINED LOCK'
class TestSetTemp(OpengbTestCase): def setUp(self): self.to_printer = Queue() self.message_handler = server.MessageHandler(to_printer=self.to_printer) def test_pass_set_temps_method_to_printer(self): """Valid set temperatures result in a 'set_temp' message on the to_printer queue.""" mh = self.message_handler.set_temp(bed=100, nozzle1=200, nozzle2=200) self.assertEqual(json.loads(self.to_printer.get())["method"], "set_temp") def test_valid_set_temps_passed_to_printer(self): """Valid set temperatures are added as a message on the to_printer queue.""" mh = self.message_handler.set_temp(bed=100, nozzle1=200, nozzle2=200) self.assertDictEqual(json.loads(self.to_printer.get()), { "method": "set_temp", "params": {"bed": 100, "nozzle2": 200, "nozzle1": 200}}) def test_set_bed_temp_defaults_to_none(self): """Unspecified bed_temperature is passed to_the printer as None.""" mh = self.message_handler.set_temp(nozzle1=200, nozzle2=200) self.assertEqual( json.loads(self.to_printer.get())["params"]["bed"], None) def test_set_nozzle1_temp_defaults_to_none(self): """Unspecified nozzle1_temperature is passed to_the printer as None.""" mh = self.message_handler.set_temp(bed=100, nozzle2=200) self.assertEqual( json.loads(self.to_printer.get())["params"]["nozzle1"], None) def test_set_nozzle2_temp_defaults_to_none(self): """Unspecified nozzle2_temperature is passed to_the printer as None.""" mh = self.message_handler.set_temp(bed=100, nozzle1=200) self.assertEqual( json.loads(self.to_printer.get())["params"]["nozzle2"], None)
def test(): NUMBER_OF_PROCESSES = 4 TASK1 = [(mul, (i, 7)) for i in range(20)] TASK2 = [(plus, (i, 8)) for i in range(10)] task_queue = Queue() done_queue = Queue() for task in TASK1: task_queue.put(task) for i in range(NUMBER_OF_PROCESSES): Process(target=worker, args=(task_queue, done_queue)).start() print 'Unordered results:' for i in range(len(TASK1)): print '\t', done_queue.get() for task in TASK2: task_queue.put(task) for i in range(len(TASK2)): print '\t', done_queue.get() for i in range(NUMBER_OF_PROCESSES): task_queue.put('STOP')
def test_worker_fills_internal_queue_only_until_maximum_queue_size(): """ Test read workers fill internal queue only to maximum size """ conn = boto3.client('sqs', region_name='us-east-1') # Set visibility timeout low to improve test speed queue_url = conn.create_queue( QueueName="tester", Attributes={'VisibilityTimeout': '1'})['QueueUrl'] message = json.dumps({ 'task': 'tests.tasks.index_incrementer', 'args': [], 'kwargs': { 'message': 'Test message', }, }) for i in range(3): conn.send_message(QueueUrl=queue_url, MessageBody=message) internal_queue = Queue(maxsize=2) worker = ReadWorker(queue_url, internal_queue, BATCHSIZE) worker.read_message() # The internal queue should only have two messages on it internal_queue.get(timeout=1) internal_queue.get(timeout=1) try: internal_queue.get(timeout=1) except Empty: pass else: raise AssertionError("The internal queue should be empty")
def driver(): for ioengine in "sync libaio".split(): for readwrite in "read write randread randwrite rw randrw".split(): for sync in "0 1".split(): for direct in "0 1".split(): name = "/home/sharath.g/fio2/{}-{}-o_sync{}-o_direct{}".format(ioengine, readwrite, sync, direct) bs = 32*1024 time = 60 experiments = [hostname] q1 = Queue() q2 = Queue() # cc('> {}'.format(name)) while bs <= 1024*1024*16: pass if direct=="1" and bs % 4096 != 0: bs *= 2 continue fio_cmd = "sudo fio --name=global --bs={bs} --ioengine={ioengine} --iodepth=10 --runtime={runtime} --time_based --size=3G --group_reporting --disable_lat=1 --disable_clat=1 --disable_slat=1 --clat_percentiles=0 --filename=/mnt/vdb1/fio/myfile --name={name} --rw={readwrite} --sync={sync} --direct={direct} --minimal".format(bs=bs, ioengine=ioengine, runtime=time, name=name, readwrite=readwrite, sync=sync, direct=direct) iostat_cmd = "/usr/bin/iostat -k -x -d 1 {} /dev/vdb".format(time) t1 = Process(target=fio, args=(fio_cmd, q1)) t2 = Process(target=iostat, args=(iostat_cmd, q2)) t1.start() t2.start() t1.join() t2.join() experiments.append([bs, fio_cmd, q1.get(), iostat_cmd, q2.get()]) l(json.dumps(experiments, indent=2)) bs*=2 with open(name, "w") as output_file: json.dump(experiments, output_file, indent=2) log.debug("======================") log.debug(json.dumps(experiments, indent=2))
def processLabelit(self,inp=False): """ Initiate Labelit runs. """ if self.verbose: self.logger.debug('LabelitPP::processLabelit') try: queue = Queue() params = {} params['test'] = self.test params['cluster'] = self.cluster_use params['verbose'] = self.verbose args1 = {} if inp: args1['input'] = inp else: args1['input'] = self.images args1['output'] = queue args1['params'] = params args1['logger'] = self.logger Process(target=RunLabelit,kwargs=args1).start() self.labelit_results = queue.get() self.labelit_log = queue.get() except: self.logger.exception('**Error in LabelitPP.processLabelit**')
def create_lists(self, copy_list, new_loc): #Used to create the call lists to_q= Queue() resp_q = Queue() p = Process(target=self.render_thing, args=(to_q,resp_q,self.initalize,new_loc,self.OFFSET, self.FACTOR, self.use_old)) p.start() to_q.put(self.diamonds, False) to_q.put(copy_list, False) to_q.put(self.OFFSET, False) to_q.put(self.rw.convert, False) print "process....." print "get pos_list" self.pos_list = resp_q.get() self.diamonds = resp_q.get() print self.pos_list new_dic = resp_q.get() dic = self.combine(self.pos_list, new_dic, self.trans.location_var) self.trans.location_var = dic p.join() print "updating...." self.rw.need_lists = True self.initalize = False
def test_processor(): pathq = Queue() pathq.put(io.StringIO(SAMPLE_XML)) outputq = Queue() def process_dump(dump, path): for page in dump: yield page.id processor = Processor(pathq, outputq, process_dump) processor.start() error, item = outputq.get() assert not error eq_(item, 1) error, item = outputq.get() assert not error eq_(item, 2) error, item = outputq.get() assert not error eq_(item, DONE)
producers = [] no_of_producers = min(2, len(dirs)) per_prod = len(dirs) // no_of_producers for i in range(no_of_producers): p = Process(target=producer, args=(dirs[i * per_prod:i * per_prod + per_prod], shared_q, count_q)) p.start() producers.append(p) for p in producers: p.join() count = 0 for i in range(len(processes)): count += count_q.get() else: count = producer(dirs, shared_q) # wait for all processes to exit print("Waiting for processes to terminate...") for i in range(len(processes)): shared_q.put(None) tcount = 0 for i in range(len(processes)): tcount += res_q.get() for p in processes: p.join() print("total files which had only 1 face:", count) print("total_written", tcount)
class DataSet(object): def __init__(self, images, labels): assert images.shape[0] == labels.shape[0], ( 'images.shape: %s labels.shape: %s' % (images.shape, labels.shape)) self._num_examples = images.shape[0] self._images = images self._labels = labels self._epochs_completed = 0 self._index_in_epoch = 0 def image_worker(self, worker_num, q, e, work_queue): while True: try: image, label = work_queue.get(timeout=0.1) image = random_crop(image) image = random_HLS(image) image = random_flip(image) q.put([image, label]) except: if e.is_set(): return def loop_fill(self, epoch_num, batch_size, q, e): self.workers = [] self.pipequeue = Queue() for k in range(NUM_PROCESSES): worker = Process(target=self.image_worker, args=(k, q, e, self.pipequeue)) worker.daemon = True worker.start() self.workers += [worker] for i in xrange(epoch_num * (self.num_examples / batch_size)): #for i in range(1): #print 'processing batch#', i while (self.pipequeue.qsize() > MAX_QUEUE) or (q.qsize() > MAX_QUEUE): time.sleep(0.2) self.batch_images, self.batch_labels = self.next_batch(batch_size) for image, labels in zip(self.batch_images, self.batch_labels): self.pipequeue.put([image, labels]) for worker in self.workers: worker.join() def start_fill(self, epoch_num, batch_size): print 'start filling...' self._resource_pool = Queue() self._stop_event = Event() self._filler = Process(target=self.loop_fill, args=(epoch_num, batch_size, self._resource_pool, self._stop_event)) self._filler.start() print 'start_fill finish.' def stop_fill(self): print 'stop filling.' self._stop_event.set() self._filler.join() print 'filling stoped' def next_batch_from_queue(self, batch_size): ready_images = [] ready_labels = [] for i in range(batch_size): image, label = self._resource_pool.get() ready_images += [image] ready_labels += [label] return ready_images, ready_labels @property def images(self): return self._images @property def labels(self): return self._labels @property def num_examples(self): return self._num_examples @property def epochs_completed(self): return self._epochs_completed def next_batch(self, batch_size, distort=False): start = self._index_in_epoch self._index_in_epoch += batch_size if self._index_in_epoch > self._num_examples: # Finished epoch self._epochs_completed += 1 # Shuffle the data perm = np.arange(self._num_examples) np.random.shuffle(perm) self._images = self._images[perm] self._labels = self._labels[perm] # Start next epoch start = 0 self._index_in_epoch = batch_size assert batch_size <= self._num_examples end = self._index_in_epoch result_images = self._images[start:end] #if distort == True: # result_images = random_crop(result_images) # result_images = random_HLS(result_images) # result_images = random_flip(result_images) return result_images, self._labels[start:end]
class LidarParser: def __init__(self, server): assert isinstance(server, CharStream) self.lidarData = [ [] ] * 360 # A list of 360 elements Angle, Distance, quality self.dataArrs = Queue() self.init_level = 0 self.server = server self.rpm = 0 def __len__(self): return int(self.dataArrs.qsize()) def pop(self): return self.dataArrs.get() def savePacketQuarter(self, angle, data): """Save a sample. Takes the angle (an int, from 0 to 359) and the list of four bytes of data in the order they arrived. """ logging.debug("Saving quarter-packet of sum %s at angle %s." % (sum(data), angle)) # unpack data using the denomination used during the discussions x = data[0] x1 = data[1] x2 = data[2] x3 = data[3] dist_mm = x | ( (x1 & 0x3f) << 8) # distance is coded on 13 bits ? 14 bits ? quality = x2 | (x3 << 8) # quality is on 16 bits self.lidarData[angle] = [dist_mm, quality] def packageScan(self): try: ragged = False for angle in self.lidarData: if len(angle) != 2: ragged = True break if not ragged: dataArr = np.vstack(self.lidarData) if dataArr.size > 0: return dataArr except ValueError as e: # Data is likely ragged (not all filled). logging.debug("Got a ValueError.") raise e def parse(self): while True: time.sleep(0.00001) # do not hog the processor power nb_errors = 0 self.init_level if self.init_level == 0: # We're searching for a start byte. logging.debug("Looking for a start byte now.") char = self.server.getChar() if char is None: continue b = ord(char) # start byte if b == 0xFA: logging.debug("Got a start byte, 0xFA!") self.init_level = 1 else: self.init_level = 0 elif self.init_level == 1: # We found a start byte; now we're parsing. logging.debug("Looking for an index byte now.") # position index b = ord(self.server.getChar(1)) if b >= 0xA0 and b <= 0xF9: # The angle indices range from 160 to 249 if b == 0xA0: # We're at the first index; save the previous scan. ####################### #### YIELD/RETURN ##### ####################### yield self.packageScan(), self.rpm ####################### #### YIELD/RETURN ##### ####################### index = b - 0xA0 logging.debug("At index %d." % index) self.init_level = 2 elif b != 0xFA: # If the next byte after the start byte isn't in the index range, go back to searching for a start byte. self.init_level = 0 elif self.init_level == 2: logging.debug("Getting two speed bytes.") # speed b_speed = [ord(b) for b in self.server.getChar(2)] # data # # (Data description from # https://xv11hacking.wikispaces.com/LIDAR+Sensor .) # # Each packet is organized as follows: # <start> <index> <speed_L> <speed_H> # [Data 0] [Data 1] [Data 2] [Data 3] <checksum_L> <checksum_H> # # where: # # start is always 0xFA=250 # index is the index byte in the 90 packets, going from 0xA0 # (packet 0, readings 0 to 3) to 0xF9 (packet 89, readings 356 to 359). # speed is a two-byte information, little-endian. It represents # the speed, in 64th of RPM (aka value in RPM represented in # fixed point, with 6 bits used for the decimal part). # [Data 0] to [Data 3] are the 4 readings. Each one is 4 bytes # long, and organized as follows : # # `byte 0 : <distance 7:0>` # `byte 1 : <"invalid data" flag> <"strength warning" flag> <distance 13:8>` # `byte 2 : <signal strength 7:0>` # `byte 3 : <signal strength 15:8>` logging.debug("Getting four data0 bytes.") b_data0 = [ord(b) for b in self.server.getChar(4)] logging.debug("Getting four data1 bytes.") b_data1 = [ord(b) for b in self.server.getChar(4)] logging.debug("Getting four data2 bytes.") b_data2 = [ord(b) for b in self.server.getChar(4)] logging.debug("Getting four data3 bytes.") b_data3 = [ord(b) for b in self.server.getChar(4)] # for the checksum, we need all the data of the packet... # this could be collected in a more elegent fashion... all_data = [0xFA, index + 0xA0 ] + b_speed + b_data0 + b_data1 + b_data2 + b_data3 # checksum logging.debug("Getting two checksum bytes.") b_checksum = [ord(b) for b in self.server.getChar(2)] incoming_checksum = int( b_checksum[0]) + (int(b_checksum[1]) << 8) logging.debug("incoming_checksum is %s" % incoming_checksum) # verify that the received checksum is equal to the one computed from the data if checksum(all_data) == incoming_checksum: self.rpm = compute_speed(b_speed) self.savePacketQuarter(index * 4 + 0, b_data0) self.savePacketQuarter(index * 4 + 1, b_data1) self.savePacketQuarter(index * 4 + 2, b_data2) self.savePacketQuarter(index * 4 + 3, b_data3) else: # the checksum does not match, something went wrong... nb_errors += 1 logging.debug("Checksum errors!") # "saveScan" data in error state. self.savePacketQuarter(index * 4 + 0, [0, 0x80, 0, 0]) self.savePacketQuarter(index * 4 + 1, [0, 0x80, 0, 0]) self.savePacketQuarter(index * 4 + 2, [0, 0x80, 0, 0]) self.savePacketQuarter(index * 4 + 3, [0, 0x80, 0, 0]) self.init_level = 0 # reset and wait for the next packet else: # default, should never happen... raise RuntimeError("You shouldn't be here!") self.init_level = 0
class AgentCtrl: ''' agent子进程维护 ''' def __init__(self, agent_name, size_x, size_y, detector_num, fighter_num, gpu_num): self.agent_name = agent_name self.size_x = size_x self.size_y = size_y self.detector_num = detector_num self.fighter_num = fighter_num self.send_q = None self.recv_q = None self.agent = None self.gpu_num = gpu_num def agent_init(self): self.send_q = Queue(1) self.recv_q = Queue(1) self.agent = AgentProc(self.agent_name, self.size_x, self.size_y, self.detector_num, self.fighter_num, self.send_q, self.recv_q, self.gpu_num) self.agent.start() try: agent_msg = self.recv_q.get(True, AGENT_INIT_TIMEOUT) except: self.terminate() return False else: return True def terminate(self): if self.agent: if self.agent.is_alive(): self.send_q.put('done') self.agent.terminate() self.agent = None if self.send_q: self.send_q.close() self.send_q = None if self.recv_q: self.recv_q.close() self.recv_q = None def get_action(self, obs_raw_dict, step_cnt): ''' 获得动作 :param obs_raw_dict: raw obs :param step_cnt: step计数,从1开始 :return: action: 动作信息 :return: result: 0, 正常; 1, 崩溃; 2, 超时 ''' action = [] result = 0 self.send_q.put({'obs_raw_dict': obs_raw_dict, 'step_cnt': step_cnt}) try: action = self.recv_q.get(True, AGENT_RESP_TIMEOUT) except: if not self.agent.is_alive(): # 子进程不存在,崩溃 result = 1 else: # 子进程存在,卡死 result = 2 self.__agent_restart() return action, result def __agent_restart(self): ''' 重启agent。由于重启代表之前成功启动过,所以此处认为重启也会成功。但若重启不成功将导致程序卡死。若后续出现此类问题应重点排查此处。 :return: ''' self.terminate() self.send_q = Queue(1) self.recv_q = Queue(1) self.agent = AgentProc(self.agent_name, self.size_x, self.size_y, self.detector_num, self.fighter_num, self.send_q, self.recv_q, self.gpu_num) self.agent.start() self.recv_q.get()
class ProcessStats(Process): def __init__(self): super(ProcessStats, self).__init__() self.episode_log_q = Queue( maxsize=100 ) # Queue for saving episode log (multiprocessesing queue) self.episode_count = Value('i', 0) self.training_count = Value('i', 0) self.should_save_model = Value('i', 0) self.trainer_count = Value('i', 0) self.predictor_count = Value('i', 0) self.agent_count = Value('i', 0) self.total_frame_count = 0 def FPS(self): # average FPS from the beginning of the training (not current FPS) return np.ceil(self.total_frame_count / (time.time() - self.start_time)) def TPS(self): # average TPS from the beginning of the training (not current TPS) return np.ceil(self.training_count.value / (time.time() - self.start_time)) def run(self): with open(Config.RESULTS_FILENAME, 'a') as results_logger, open(Config.EPISDOES_LOG_FILENAME, 'a') as eval_logger: rolling_frame_count = 0 rolling_reward = 0 results_q = queueQueue(maxsize=Config.STAT_ROLLING_MEAN_WINDOW) episode_scores = [] self.start_time = time.time() first_time = datetime.now() while True: episode_time, reward, length = self.episode_log_q.get( ) # get episode log from queue results_logger.write( '%s, %d, %d\n' % (episode_time.strftime("%Y-%m-%d %H:%M:%S"), reward, length)) results_logger.flush() self.total_frame_count += length self.episode_count.value += 1 rolling_frame_count += length rolling_reward += reward if results_q.full(): old_episode_time, old_reward, old_length = results_q.get() rolling_frame_count -= old_length rolling_reward -= old_reward first_time = old_episode_time results_q.put((episode_time, reward, length)) episode_scores.append(reward) if len(episode_scores) >= 1000: eval_logger.write("{} {} {} {}\n".format( self.episode_count.value, np.max(episode_scores[-100:]), np.mean(episode_scores[-100:]), np.min(episode_scores[-100:]))) eval_logger.flush() episode_scores = [] if self.episode_count.value % Config.SAVE_FREQUENCY == 0: self.should_save_model.value = 1 if self.episode_count.value % Config.PRINT_STATS_FREQUENCY == 0: print('[Time: %8d] ' '[Episode: %8d Score: %10.4f] ' '[RScore: %10.4f RPPS: %5d] ' '[PPS: %5d TPS: %5d] ' '[NT: %2d NP: %2d NA: %2d]' % (int(time.time() - self.start_time), self.episode_count.value, reward, rolling_reward / results_q.qsize(), rolling_frame_count / (datetime.now() - first_time).total_seconds(), self.FPS(), self.TPS(), self.trainer_count.value, self.predictor_count.value, self.agent_count.value)) sys.stdout.flush()
def main(): global parent_pid parent_pid = os.getpid() args_info = ArgsProcessing(sys.argv[1:]) # 处理命令行参数 work_mode = args_info.current_mode # 工作模式 log_file_list = args_info.log_file_list # 所有的日志文件 signal.signal(signal.SIGINT, signal_handler) # SIGINT是ctrl+c发出的信号,值为2 start_time = time.time() # 模式1:分析单笔交易 模式2:分析所有交易 模式3:分析单个区块 模式4:分析所有区块 if work_mode == 1: tx_hash = args_info.tx_hash # 交易Hash # 获取全部的字典格式的交易数据 all_tx_dict_list = get_all_log_dict(log_file_list, 'transaction') overall_earliest_msg, overall_latest_msg = retrieve_earliest_latest_msg( all_tx_dict_list, tx_hash) if overall_earliest_msg and overall_latest_msg: print('最早: %s' % overall_earliest_msg) print('最晚: %s' % overall_latest_msg) interval_time = millisecond2time_format( calc_millisecond_interval(overall_latest_msg[0], overall_earliest_msg[0])) print('间隔: %s' % interval_time) else: print('The transaction %s was not found in log file!' % tx_hash) elif work_mode == 2: all_tx_dict_list = get_all_log_dict(log_file_list, 'transaction') all_tx_hash = [] # 所有的交易Hash for tx_dict in all_tx_dict_list: all_tx_hash.extend(list(tx_dict.keys())) # 去除重复元素 all_tx_hash = list(set(all_tx_hash)) broadcasting_time_queue = Queue() # 存储广播时间的Queue processes = [] # 获取cpu核心数 processor_num = cpu_count() # 将所有的交易Hash分割为与cpu核心数相等的尽量平均的份数 split_all_tx_hash = split_list(all_tx_hash, processor_num) for work_list in split_all_tx_hash: # 创建与交易Hash份数(cpu核心数)数量相同的子进程用于计算广播时间 p = Process(target=calc_broadcasting_time, args=(work_list, broadcasting_time_queue, all_tx_dict_list)) p.start() processes.append(p) for process in processes: # 等待所有子进程结束 process.join() broadcasting_time_list = [] while True: # 将子进程的分析结果存储到父进程中的列表 broadcasting_time_list.append(broadcasting_time_queue.get()) if broadcasting_time_queue.empty(): break # 使用最小堆对所有子进程生成的有序列表合并 broadcasting_time_list = list(heapq.merge(*broadcasting_time_list)) # 计算广播时间列表平均值和中位数 average, median = get_average_median(broadcasting_time_list) print('最短时间: %s' % millisecond2time_format(broadcasting_time_list[0])) print('最长时间: %s' % millisecond2time_format(broadcasting_time_list[-1])) print('平均值: %s' % millisecond2time_format(average)) print('中位数: %s' % millisecond2time_format(median)) elif work_mode == 3: height = args_info.height # 区块高度 all_block_dict_list = get_all_log_dict(log_file_list, 'block') overall_earliest_msg, overall_latest_msg = retrieve_earliest_latest_msg( all_block_dict_list, height) if overall_earliest_msg and overall_latest_msg: print('最早: %s' % overall_earliest_msg) print('最晚: %s' % overall_latest_msg) interval_time = millisecond2time_format( calc_millisecond_interval(overall_latest_msg[0], overall_earliest_msg[0])) print('间隔: %s' % interval_time) else: print('The block height %s was not found in log file!' % height) elif work_mode == 4: all_block_dict_list = get_all_log_dict(log_file_list, 'block') all_block_height = [] for block_dict in all_block_dict_list: all_block_height.extend(list(block_dict.keys())) all_block_height = list(set(all_block_height)) broadcasting_time_queue = Queue() processes = [] processor_num = cpu_count() split_all_block_height = split_list(all_block_height, processor_num) for work_list in split_all_block_height: p = Process(target=calc_broadcasting_time, args=(work_list, broadcasting_time_queue, all_block_dict_list)) p.start() processes.append(p) for process in processes: process.join() broadcasting_time_list = [] while True: broadcasting_time_list.append(broadcasting_time_queue.get()) if broadcasting_time_queue.empty(): break broadcasting_time_list = list(heapq.merge(*broadcasting_time_list)) average, median = get_average_median(broadcasting_time_list) print('最短时间: %s' % millisecond2time_format(broadcasting_time_list[0])) print('最长时间: %s' % millisecond2time_format(broadcasting_time_list[-1])) print('平均值: %s' % millisecond2time_format(average)) print('中位数: %s' % millisecond2time_format(median)) print('分析用时:', time.time() - start_time)
class Main(QMainWindow): def __init__(self): # Initialize the super class super().__init__() # self.setMinimumSize(QSize(640, 480)) self.setMinimumWidth(480) self.setWindowTitle("pyJulieScanner - A Kodi Database Scanner") centralWidget = QWidget(self) self.setCentralWidget(centralWidget) gridLayout = QGridLayout(self) centralWidget.setLayout(gridLayout) self.allOptions = {} # title.setAlignment(QtCore.Qt.AlignCenter) # gridLayout.addWidget(title, 0, 0) # Setup checkboxes for possible tasks checkboxLabel = QLabel("Perform the following actions:", self) option1 = QCheckBox( "Produce a file containing all movies Kodi does not see", self) option2 = QCheckBox("Produce a file containing all movies Kodi sees", self) option3 = QCheckBox("Check TV shows for missing seasons", self) self.fileOutputDir = "./" self.fileOutputDirText = "Output Files to: " self.fileOutputDirButton = QPushButton(self.fileOutputDirText, self) self.fileOutputDirLabel = QLabel(self.fileOutputDir, self) # Connect it to setting the outputdir self.fileOutputDirButton.clicked.connect(self.setOutputDir) self.allOptions["option1"] = option1 self.allOptions["option2"] = option2 self.allOptions["option3"] = option3 # The log area self.logText = QTextEdit("", self) self.logText.setReadOnly(True) # Action button! self.doButton = QPushButton("Do Checked Tasks", self) # Connect a function to the button self.doButton.clicked.connect(self.doAllPossibleActions) self.option1ProgressLabel = QLabel("", self) self.option1ProgressLabel.hide() gridLayout.addWidget(checkboxLabel, 0, 0, 1, 3) gridLayout.addWidget(option1, 1, 0, 1, 3) gridLayout.addWidget(option2, 2, 0, 1, 3) gridLayout.addWidget(option3, 3, 0, 1, 3) gridLayout.addWidget(self.fileOutputDirButton, 4, 0, 1, 1) gridLayout.addWidget(self.fileOutputDirLabel, 4, 1, 1, 3) # gridLayout.addWidget(self.fileOutputDirText, 4, 1) gridLayout.addWidget(QLabel("Log:"), 5, 0) gridLayout.addWidget(self.logText, 6, 0, 1, 4) gridLayout.addWidget(self.doButton, 5, 3) gridLayout.addWidget(self.option1ProgressLabel, 7, 0, 1, 4) # Find Kodi database # self.databases = self.getDatabases() self.getDatabases() self.activeProcess = None self.latest = None self.option1Timer = None # Find all media that Kodi doesn't see # Find all media that Kodi sees # Find missing TV Show seasons def doAllPossibleActions(self): if not self.fileOutputDir: # Get a file output location self.setOutputDir() if self.fileOutputDir is None or not self.fileOutputDir: msg = QMessageBox() msg.setWindowTitle("Error") msg.setText("No Output Directory Selected!") msg.exec() self.updateText("No actions selected!") return None p = 0 r = re.compile("[\\:\\w&.\\-\\/]+MyVideos\\d+.db$") videoDatabase = list(filter(r.match, self.databases))[0] if self.allOptions['option1'].isChecked(): # If a current job is running, kill it! if self.option1Timer is not None and self.option1Timer.isActive(): self.option1Timer.stop() if self.activeProcess is not None and self.activeProcess.is_alive( ): self.updateText("Killing active task...") self.activeProcess.terminate() self.updateText("Success! Starting new requested tasks...") # Establish a connection to the movie database connection = sqlite3.connect(videoDatabase) cursor = connection.cursor() # Get the movie name, movie path, movie filename cursor.execute("""SELECT DISTINCT strFilename,strPath FROM files INNER JOIN movie ON files.idFile = movie.idFile INNER JOIN path ON files.idPath = path.idPath;""") allKnownMovies = cursor.fetchall() self.updateText("Found {} files in Kodi...".format( len(allKnownMovies))) connection.close() p += 1 mediaDirectory = QFileDialog.getExistingDirectory( self, "Select Media Directory To Scan") if mediaDirectory == "": self.updateText( "Media directory cannot be ignored for option 1! Skipping..." ) else: # Definitely broken! # Start a thread safe queue self.queue = Queue() # Setup a timer self.option1Timer = QTimer(self) self.option1Timer.setInterval(1000) self.option1Timer.timeout.connect( self.updateMovieDirectoriesScanned) # Spin up a process self.activeProcess = VideoWorker( self.queue, mediaDirectory, allKnownMovies, self.fileOutputDir + "/Missing Movies.csv") # Start a timer for checking and updating the files found marker self.activeProcess.start() self.option1Timer.start() if self.allOptions['option2'].isChecked(): # Establish a connection to the movie database connection = sqlite3.connect(videoDatabase) cursor = connection.cursor() # Get the movie name, movie path, movie filename cursor.execute("SELECT DISTINCT c00 FROM movie;") allKnownMovies = sorted(cursor.fetchall()) allKnownMovies = list( map(lambda nameTup: '"{}"'.format(nameTup[0]), allKnownMovies)) # Point out possible duplicates? cursor.execute("""SELECT movie.c00,cnt,strPath FROM movie INNER JOIN files ON files.idFile = movie.idFile INNER JOIN path ON path.idPath = files.idPath LEFT JOIN ( SELECT c00, COUNT(c00) AS cnt FROM movie GROUP BY c00 ) AS temptable ON movie.c00 = temptable.c00 WHERE cnt > 1;""") duplicates = sorted(cursor.fetchall()) duplicates = list( map( lambda entry: ",".join( list(map(lambda item: '"{}"'.format(item), entry))), duplicates)) self.updateText("Writing out a CSV of movie names from Kodi...") with open("{}/All Movies.csv".format(self.fileOutputDir), "w") as outfile: outfile.write("Title\n") outfile.write("\n".join(allKnownMovies)) self.updateText("{}/All Movies.csv".format(self.fileOutputDir)) with open("{}/All Duplicate Movies.csv".format(self.fileOutputDir), "w") as outfile: outfile.write("Title,Total,Path\n") outfile.write("\n".join(duplicates)) self.updateText("{}/All Duplicate Movies.csv".format( self.fileOutputDir)) # Get the movie name, movie path, movie filename p += 1 # self.updateText("Found {} movies logged by Kodi...".format(numFound)) # Write out the file if self.allOptions['option3'].isChecked(): connection = sqlite3.connect(videoDatabase) cursor = connection.cursor() # cursor.execute("SELECT idMovie, idFile, c00 FROM movie") # allKnownShows = cursor.fetchall() connection.close() self.updateText("Option 3 is not yet available") p += 1 # self.updateText("Found {} missing seasons".format(numMissing)) # Write out the file if p == 0: msg = QMessageBox() msg.setWindowTitle("Error") msg.setText("No Actions Selected!") msg.exec() self.updateText("No actions selected!") return None # Possibly broken def updateMovieDirectoriesScanned(self): # Check if active qsize = self.queue.qsize() totalActive = len(active_children()) print("Active children: {}".format(totalActive)) # Check the queue for num in range(qsize): self.latest = self.queue.get() if totalActive != 0: if self.latest is not None: self.option1ProgressLabel.show() self.option1ProgressLabel.setText("Currently in {}...".format( str(self.latest))) else: self.updateText("Cleaning up...") if self.latest is not None: self.option1ProgressLabel.hide() self.updateText( "Safe finish: {} files not imported into Kodi".format( self.latest)) self.updateText("{}Missing Movies.csv".format( self.fileOutputDir)) self.option1Timer.stop() print("Tick!") def setOutputDir(self): self.fileOutputDir = QFileDialog.getExistingDirectory( self, "Select Media Location") self.fileOutputDirLabel.setText(self.fileOutputDir) self.updateText("Output directory selected as:\n{}".format( self.fileOutputDir)) return None def updateText(self, moreText): self.logText.setText("{}{}{}".format(self.logText.toPlainText(), moreText, "\n")) self.logText.verticalScrollBar().setValue( self.logText.verticalScrollBar().maximum()) return None def getDatabases(self, databaseRoot=None): # For windows only if databaseRoot is None: databaseRoot = "C:/Users/{}/AppData/Roaming/Kodi/userdata/Database".format( getuser()) databases = [] if exists(databaseRoot): allfiles = listdir(databaseRoot) for afile in allfiles: if afile[-3:] == ".db": databases.append(afile) if databases == []: self.updateText( "Directory exists but no databases found? Program will crash if ran further" ) else: databases = list( map( lambda database: "{}/{}".format( databaseRoot, database), databases)) self.updateText("Databases found: {}".format(databases)) self.databases = databases else: # Need a better work around if databaseRoot == "" or databaseRoot != "C:/Users/{}/AppData/Roaming/Kodi/userdata/Database".format( getuser()): exit() self.updateText("No databases found at {}".format(databaseRoot)) self.updateText( "A new location must be selected and must contain the userdata folder for kodi or the application will crash" ) # Allow the user to pick a directory but see files # databaseRoot = QFileDialog.getExistingDirectory(self, "Select Database Location",QFileDialog.) databaseRoot = QFileDialog.getExistingDirectory( self, "Select Database Location") self.getDatabases(databaseRoot=databaseRoot)
def search_single_record(self, rec, n_parallel_words=1, word_limit=None, process_timeout=None, maximum_matches=1000, filter=None): if n_parallel_words is None: n_parallel_words = cpu_count() if word_limit is None: word_limit = self.N initial_q = managerQueue.Queue() [ initial_q.put({field_name: rec[field_name]}) for field_name in self.index_names[:word_limit] ] # enqueue a sentinel value so we know we have reached the end of the queue initial_q.put('STOP') queue_empty = False # create an empty queue for results results_q = Queue() # create a set of unique results, using MongoDB _id field unique_results = set() l = list() while True: # build children processes, taking cursors from in_process queue first, then initial queue p = list() while len(p) < n_parallel_words: word_pair = initial_q.get() if word_pair == 'STOP': # if we reach the sentinel value, set the flag and stop queuing processes queue_empty = True break if not initial_q.empty(): p.append( Process(target=get_next_match, args=(results_q, word_pair, self.collection, np.array(rec['signature']), self.distance_cutoff, maximum_matches))) if len(p) > 0: for process in p: process.start() else: break # collect results, taking care not to return the same result twice num_processes = len(p) while num_processes: results = results_q.get() if results == 'STOP': num_processes -= 1 else: for key in results.keys(): if key not in unique_results: unique_results.add(key) l.append(results[key]) for process in p: process.join() # yield a set of results if queue_empty: break return l
class SketchDataHandler(DataHandler): def __init__(self, root_path, paths_file, batch_size, target_size): super(SketchDataHandler, self).__init__(batch_size, target_size) self._index = 0 self.root_path = root_path self._image_paths = self._get_image_paths( os.path.join(root_path, paths_file)) if (len(self._image_paths) < batch_size * 100): self._image_paths = self._image_paths * 100 self._shuffle_image_paths() self._total_num = len(self._image_paths) self.queue = Queue(40) self.msg_queue = Queue(4) self.procs = [] self.start_threads() def _get_image_paths(self, paths_file): with open(paths_file) as f: return [ os.path.join(self.root_path, line.rstrip('\n')) for line in f ] def _shuffle_image_paths(self): random.shuffle(self._image_paths) def _random_preprocessing(self, image, size): # rotate image rand_degree = np.random.randint(0, 180) rand_flip = np.random.randint(0, 2) image = scipy.ndimage.interpolation.rotate(image, rand_degree, cval=255) if rand_flip == 1: image = np.flip(image, 1) # Select cropping range between (target_size/2 ~ original_size) original_h, original_w = image.shape crop_width = np.random.randint(self.target_size / 2, original_w) crop_height = np.random.randint(self.target_size / 2, original_h) topleft_x = np.random.randint(0, original_w - crop_width) topleft_y = np.random.randint(0, original_h - crop_height) cropped_img = image[topleft_y:topleft_y + crop_height, topleft_x:topleft_x + crop_width] output = scipy.misc.imresize(cropped_img, [self.target_size, self.target_size]) output = (output - 128.0) / 128.0 return output def next(self): output = self.queue.get() return output def _enqueue_op(self, queue, msg_queue): while msg_queue.qsize() == 0: # randomly select index indexes = np.random.randint(0, self._total_num, self.batch_size) sz = self.target_size output = np.zeros([self.batch_size, sz, sz, 1]) for i in range(len(indexes)): index = indexes[i] output[i] = self._random_preprocessing( scipy.misc.imread(self._image_paths[index], mode='L').astype(np.float), self.target_size).reshape([sz, sz, 1]) while np.amin(output[i]) == np.amax( output[i]): # some data are strange.. output[i] = self._random_preprocessing( scipy.misc.imread(self._image_paths[index], mode='L').astype(np.float32), self.target_size).reshape([sz, sz, 1]) queue.put(output) def start_threads(self): print("start threads called") for i in range(2): proc = Process(target=self._enqueue_op, args=(self.queue, self.msg_queue)) self.procs.append(proc) proc.daemon = True proc.start() print("enqueue thread started!") def get_batch_shape(self): return (self.batch_size, self.target_size, self.target_size, 1) def kill(self): self.msg_queue.put("illkillyou") for proc in self.procs: proc.terminate() proc.join() print('sketch data killed')
def test_net(imdb, output_dir, target_test, thresh=0.05, no_cache=False, step=0): # Run demo if imdb is None: assert cfg.TEST.DEMO.ENABLE, "check your config and stderr!" return demo(target_test, thresh) # Initializing the timers logger.info('Evaluating {} on {}'.format(cfg.NAME, imdb.name)) run_inference = True if not no_cache: det_file = os.path.join(output_dir, 'detections.pkl') if os.path.exists(det_file): try: with open(det_file, 'r') as f: dets = cPickle.load(f) run_inference = False logger.info( 'Loading detections from cache: {}'.format(det_file)) except: logger.warning( 'Could not load the cached detections file, detecting from scratch!' ) # Perform inference on images if necessary if run_inference: pyramid = True if len(cfg.TEST.SCALES) > 1 else False if isinstance(cfg.TEST.GPU_ID, int): cfg.TEST.GPU_ID = [cfg.TEST.GPU_ID] assert len(cfg.TEST.GPU_ID) >= 1, "You must specify at least one GPU" if len(cfg.TEST.GPU_ID) == 1: dets = inference_worker(0, imdb, target_test, 0, len(imdb), thresh) else: result_queue = Queue() procs = [] len_per_gpu = int(np.ceil(1. * len(imdb) / len(cfg.TEST.GPU_ID))) for rank in range(len(cfg.TEST.GPU_ID)): p = Process(target=inference_worker, args=(rank, imdb, target_test, len_per_gpu * rank, min(len_per_gpu * (rank + 1), len(imdb)), thresh, result_queue)) p.daemon = True p.start() procs.append(p) dets = [result_queue.get() for _ in procs] for p in procs: p.join() dets = [det[1] for det in sorted(dets, key=lambda x: x[0])] dets = [[_ for det in dets for _ in det[i]] for i in range(imdb.num_classes)] assert len(dets[0]) == len(imdb), "Detection result compromised" det_file = os.path.join(output_dir, 'detections.pkl') if not no_cache: with open(det_file, 'wb') as f: cPickle.dump(dets, f, cPickle.HIGHEST_PROTOCOL) # Evaluate the detections logger.info('Evaluating detections') result = imdb.evaluate_detections(all_boxes=dets, output_dir=output_dir, method_name=cfg.NAME, step=step) logger.info(result) logger.info('All Done!')
# if np.any(bool_list): # time.sleep(.1) # else: # break # else: # print("Timed out, killing all processes") # for p in procs: # p.terminate() liveprocs = list(procs) results = [] while liveprocs: try: while 1: results.append(output.get(False)) except queue.Empty: pass time.sleep(0.5) if not output.empty(): continue liveprocs = [p for p in liveprocs if p.is_alive()] for proc in procs: proc.join() # results = [] # for proc in procs: # while proc.is_alive():
# coding:utf-8 from multiprocessing import Queue try: q = Queue() q.get(timeout=5) except BaseException as e: print('--' + str(e))
def await_solution(puzzle: np.ndarray, q: Queue) -> None: solved = q.get() print("\nOriginal:\n") display_grid(puzzle) print("\nSolved:\n") display_grid(solved)
# -*- coding: utf-8 -*- """ Created on Fri Apr 13 17:35:23 2018 @author: Administrator """ from multiprocessing import Process, Queue def f(q, n): q.put([42, n, 'hello']) if __name__ == '__main__': q = Queue() p_list = [] for i in range(3): p = Process(target=f, args=(q, i)) p_list.append(p) p.start() print(q.get()) print(q.get()) print(q.get()) for i in p_list: i.join()
with ThreadPoolExecutor() as executor: results = executor.map(long_running_search, url_list) q.put([x for x in results]) # Send results to the shared queue. print(f'Worker {name} done.') def long_running_search(url): try: r = requests.get(url, timeout=2, verify=False) return r.status_code except Exception as e: print(f'{e}') if __name__ == '__main__': jobs = [] for root_name in root_instances: p = Process(target=search_instances, args=(root_name, q, url_dict[root_name])) jobs.append(p) p.start() for job in jobs: job.join() # Ensure each process exits normally. while not q.empty(): print(q.get()) # Retrieve each item from the queue. print('Fin')
# Created by Elshad Karimov on 31/05/2020. # Copyright © 2020 AppMillers. All rights reserved. # How to use multiprocessing.Queue as a FIFO queue: from multiprocessing import Queue customQueue = Queue(maxsize=3) customQueue.put(1) print(customQueue.get())
def flush_worker(queue: Queue): while True: q_contents = queue.get() tracker_session.add(q_contents) tracker_session.flush()
class NoverScratch: # TODO list # 1. write txt # 2. multi threat to handle # 3. how to write txt. write once or more times. if one time, string would overflow # more times include more txt join on txt or append text in order # initial varaiable # @param {string} chromeDriver Local Path eg: C:\Program Files (x86)\Google\Chrome\Application\chromedriver def __init__(self, chromeDriverPath): # chrome browser # defaultChromeDriverPath = 'C:\Program Files (x86)\Google\Chrome\Application\chromedriver' # self.browser = webdriver.Chrome(defaultChromeDriverPath) # phantomjs defaultPhantomjsPath = './phantomjs' if platform.system() == 'Linux': # specfic path related to phantamjs.sh defaultPhantomjsPath = '/usr/local/src/phantomjs/bin/phantomjs' self.browser = webdriver.PhantomJS(service_args=[ '--ignore-ssl-errors=true', '--ssl-protocol=any' ]) else: self.browser = webdriver.PhantomJS( executable_path=defaultPhantomjsPath) # headless chrome # chrome_options = Options() # chrome_options.add_argument('--headless') # self.browser = webdriver.Chrome(chrome_options=chrome_options, executable_path=r'C:\Program Files (x86)\Google\Chrome\Application\chrome.exe') if chromeDriverPath: self.browser = webdriver.Chrome(chromeDriverPath) # web driver browser wait timeout self.wait = WebDriverWait(self.browser, 10) # multi thread lock self.threadLock = threading.Lock() # Queue self.queue = Queue() # get each chapter link by menu link # @param {string} url eg:http:// or https://www.biquyun.com/1_1559/ # @returns {chapterName} chapterName def getEachChapterLink(self, menuUrl): # check param validation if menuUrl.index('http') > -1 or menuUrl.index('https') > -1: # begin browser load try: self.browser.get(menuUrl) # add chapter link when content load chapterList = self.wait.until( EC.presence_of_element_located((By.CSS_SELECTOR, '#list'))) html = self.browser.page_source doc = pq(html) chapterName = doc('#info > h1').text() items = doc('#list > dl > dd > a').items() # add each link for item in items: link = item.attr('href') # eg /1_1559/952222.html # using re to join link TODO # current use split to join linkArr = link.split('/') linkSuffix = linkArr[len(linkArr) - 1] self.queue.put(menuUrl + linkSuffix) return chapterName except TimeoutException: # todo # print('error') return '' # get specific text by each chapter link # @param {string} chapter link eg:https://www.biquyun.com/1_1559/9986611.html # @param {chapterName} chapterName # @param {thread_id} thread id # @returns {string} novel text or other info (TODO) def getSpecificTextByChapterLink(self, chapterLink, chapterName): try: self.browser.get(chapterLink) chapterContent = self.wait.until( EC.presence_of_element_located((By.CSS_SELECTOR, '#content'))) html = self.browser.page_source doc = pq(html) chapterTitle = doc( '#wrapper > div.content_read > div > div.bookname > h1').text( ) chapterText = doc('#content').text() # chapterText may include chapterTitle 标题写了两遍 # parse chapterText because chapterText mat like this. eg: 第一五二章风雨初平<br> # write chaptertext to txt # get lock self.threadLock.acquire() self.writeTextToLocalTXT((' \r\n ' + chapterTitle + ' \r\n ' + chapterText).encode('utf-8'), chapterName) # release lock self.threadLock.release() except TimeoutException: return # parse chapterText eg: 第一五二章风雨初平<br> # @param {string} chapterText to be parsed # @param {chapterName} chapterName # @returns {array} content after parsing def parseChapterContent(self, chapterContent): # parse and <br/> by re parseContent = re.sub(r'(<br>+)\s*(<br>)*', '<br>\r\n<br>', chapterContent) p = re.compile(r'<br>') return p.split(parseContent) # write content to local txt # @params {array} chapterContent array # @param {thread_id} thread id # @returns {boolean} write success or false def writeTextToLocalTXT(self, chapterContent, chapterName): file = None try: # file = open('static/' + chapterName + '_' + str(thread_id) + '.txt', 'ab+') file = open('static/' + chapterName + '.txt', 'ab+') if isinstance(chapterContent, list): # '\r\n' represent \n normaly file.writelines(chapterContent) else: file.write(chapterContent) except IOError: file.close() finally: file.close() # join several txt file to one # @param {txtFileArr} # @param {savePath} saved txt path def joinTxtFile(self, txtFileArr, savePath): with open(savePath, 'w') as outfile: for fname in txtFileArr: with open(fname) as infile: outfile.write(infile.read()) # multi thread to write txt # @param {linkArr} linkArr to handle # @param {thread_num} thread num # @param {chapterName} chapterName def multiThreadGetChapter(self, thread_num, chapterName): efficientWorker = EfficientWorker(self.batchWriteChapterToTxt, [chapterName], 'thread', 4, 5) efficientWorker.start() # batch write chapter to txt def batchWriteChapterToTxt(self, chapterName): while self.queue.empty() != True: chanpterLink = self.queue.get() self.getSpecificTextByChapterLink(chanpterLink, chapterName) # write each chapter to txt # @param {link} eg.https://www.biquyun.com/1_1559/ def writeTotalChapterToTxt(self, link): chapterName = self.getEachChapterLink(link) thread_num = 5 try: self.multiThreadGetChapter(thread_num, chapterName) # thread_txt = ['static/' + chapterName + '_' + str(i) + '.txt' for i in range(thread_num)] # self.joinTxtFile(thread_txt, 'static/' + chapterName + '.txt') # for tfile in thread_txt: # if os.path.exists(tfile): # os.remove(tfile) return chapterName + '.txt' except IOError: return -1
pass for idx in range(num_worker): parent_conn, child_conn = Pipe() worker = MarioEnv(env_id, idx, child_conn, queue, n_step, is_render) worker.start() workers.append(worker) parent_conns.append(parent_conn) while model.g_episode < max_episode: while queue.empty(): # Wait for worker's state continue # Received some data idx, command, parameter = queue.get() if command == "OnStep": transition, reward, done = parameter action = model.get_action(transition, is_random=False) buffer_state[idx].append(np.array(transition)) buffer_action[idx].append(action) buffer_reward[idx].append(reward) # n-step을 위한 데이터들이 다 모였을 시 if len(buffer_state[idx]) > n_step: model.train(buffer_state[idx], buffer_action[idx], buffer_reward[idx], done) buffer_state[idx].pop()
video_capture = WebcamVideoStream(src=cam_url, width=640, height=480).start() #video_capture=cv2.VideoCapture(0) fps = FPS().start() t_start = time.time() out = None while True: # fps._numFrames < 120 if time.time() - t_start > 1.2: # 输出图像的延迟 t_start = time.time() frame = video_capture.read() frame = cv2.resize(frame, (int(640), int(480))) input_q.put(frame) t = time.time() out = mid_q.get() output_q.put(out) cv2.imshow('Video', out) fps.update() print('[INFO] elapsed time: {:.2f}'.format(time.time() - t)) else: frame = video_capture.read() if out is not None: cv2.imshow("Video", out) else: cv2.imshow("Video", frame) if cv2.waitKey(1) & 0xFF == ord('q'): break # time.sleep(0.3)
p.start() while not SHOULD_EXIT: # print("processing GUI events in CALLBACK") sleep(0.01) # check if encoders have changed pedal_hardware.process_input() pedal_hardware.EXIT_THREADS = True send_ui_message("exit", ("exit", )) p.terminate() p.join() while True: try: ui_messages.get(block=False) except queue.Empty: break while True: try: core_messages.get(block=False) except queue.Empty: break ui_messages.close() ui_messages.join_thread() core_messages.close() core_messages.join_thread() # print("exiting core") print("Normal exit")
print(m[i][j], end="") print(end='\n') print(end='\n\n') # multiprocessing for _ in range(args.processes): p = Process(target=next_step, args=(x, q)) processes.append(p) p.start() for process in processes: process.join() # get the final matrix for 'nth' no of process, where n is the no. of processes while q.empty() is False: final = q.get() finish = time.perf_counter( ) # to mark the current time after the whole processes are executed o = args.output # write the resulting matrix to the output file in form of string for i in range(len(final[0])): for j in range(len(final[0][i])): o.write(final[0][i][j]) o.write('\n') print( f'Simulation complete. Final result stored in the output file "{args.output.name}"', end="\n\n")
right_p.start() l_cnt = 0 r_cnt = 0 for ep_i in range(args.episodes): done_n = [False for _ in range(env.n_agents)] ep_reward = 0 env.seed(ep_i) obs_n = env.reset() frame = [] while not all(done_n): left_q1.put(obs_n) right_q1.put(obs_n) l_action = left_q2.get() r_action = right_q2.get() obs_n, reward_n, done_n, info = env.step([l_action, r_action]) # frame.append(env.render(mode='rgb_array')) #action_n = env.action_space.sample() #obs_n, reward_n, done_n, info = env.step(action_n) l_reward = reward_n[0] r_reward = reward_n[1] l_cnt += l_reward r_cnt += r_reward env.render() print('Episode #{} left: {} right: {} '.format(ep_i, l_cnt, r_cnt)) left_q1.put(None) right_q1.put(None)
from multiprocessing import Process, Queue import numpy as np Size = 127 def producer(mp_queue): print("Child entered") window1 = np.arange(Size * Size).reshape((Size, Size)) print(window1.dtype) print("Array created") mp_queue.put(window1) print("Data added to Pipe") return if __name__ == '__main__': q = Queue() window_reader = Process(target=producer, args=(q, )) window_reader.start() window_reader.join() print("Child process exited, entered parent") # Acting like a consumer a = q.get() print(a)
def main(): parser = argparse.ArgumentParser(description="Perform workflow for scan") parser.add_argument('-d', '--directory', help='scan directory') parser.add_argument('-r', '--refine', action='store_false', help='refine lattice parameters') parser.add_argument('-t', '--transform', action='store_true', help='perform CCTW transforms') parser.add_argument('-c', '--cwd', default='/data/user6idd', help='directory containing GUP directories') parser.add_argument('-g', '--gup', default='GUP-57342', help='GUP number, e.g., GUP-57342') args = parser.parse_args() directory = args.directory.rstrip('/') sample = os.path.basename(os.path.dirname(directory)) label = os.path.basename(directory) if args.refine: refine = '-r' else: refine = '' if args.transform: transform = '-t' else: transform = '' print("Processing directory '%s'" % directory) path = os.path.join(args.cwd, args.gup) if 'tasks' not in os.listdir(path): os.mkdir(os.path.join(path, 'tasks')) scans = sorted([ scan for scan in os.listdir(directory) if (os.path.isdir(os.path.join(directory, scan)) and not scan.endswith('_1')) ], key=natural_sort) parent = os.path.join(sample, label, '%s_%s.nxs' % (sample, scans[0])) commands = [ 'nxtask -d %s -p %s %s %s' % (os.path.join(directory, scan), parent, refine, transform) for scan in scans ] tasks = JoinableQueue() results = Queue() nodes = [ProcessNode(node, tasks, results) for node in orthros_nodes] for node in nodes: node.start() # Enqueue jobs for command in commands: print(command) tasks.put(Task(path, command)) # Add a poison pill for each node for node in nodes: tasks.put(None) # Wait for all of the tasks to finish tasks.join() # Start printing results num_jobs = len(commands) while num_jobs: result = results.get() print('Completed:', result) num_jobs -= 1
def main(workers=0, limit=None): globs = Globs() # Configure logging logging.basicConfig( level=logging.ERROR, filename='main.log', format='%(relativeCreated)6d %(threadName)s %(message)s') logging.error('STARTING NEW ENTRY AT %s' % str(dt.now())) # Load settings with open('settings.txt', 'r') as f: try: filename = f.readline().split('=')[1].strip() print('Loading', filename) except Exception as e: print('Failed to read filename from settings.txt'.upper(), e) sys.exit() try: globs.set('minmatch', int(f.readline().split('=')[1].strip())) print('Minmatch set to', globs.get('minmatch')) except Exception as e: print('Failed to read minimum from settings.txt'.upper(), e) sys.exit() # Load csv try: df = pd.read_csv(filename, header=0) except Exception as e: print('Failed to read csv file with filename', filename, 'from settings.txt', e) sys.exit() # Limit as needed if limit: df = df.iloc[:limit] # Determine number of workers(threads) to use if workers == 0: workers = cpu_count() # Create jobs st = time() jobs = [] q = Queue() for n in range(workers): batch_size = len(df) // workers + 1 start = n * batch_size end = start + batch_size p = Process(target=start_worker, args=(n, df.iloc[start:end], globs, q)) jobs.append((n, p)) print('Starting worker', n) p.start() # Wait until all links are scraped while q.qsize() < len(df): print('Main process is waiting for workers to finish...') sleep(10) for n, p in jobs: p.join(timeout=.1) print('Worker %s process has joined' % n) # Get worker results and save as a csv called results.csv results = pd.DataFrame(columns=['Email', 'Results']) i = 0 while q.qsize() > 0: r = q.get(timeout=.2) # print(i, r) results.loc[len(results)] = r i += 1 results.to_csv('results.csv') print('Results written to results.csv...') print("Time Elapsed: {:.2f}s".format(time() - st))
p_mels = [] for i in range(num_part): p_mels.append( Process(target=preprocess_audio, args=(args, 'mel', metalist_list[i], q))) for i in range(num_part): p_mels[i].daemon = True p_mels[i].start() filtered_list = [] error_list = [] for _ in range(num_part): _, _filtered, _error = q.get() filtered_list.extend(_filtered) error_list.extend(_error) filtered_list.sort() print(len(filtered_list)) with open('MSD_audio_filtered_track_file_path_list.p', 'wb') as handle: pickle.dump(filtered_list, handle, protocol=pickle.HIGHEST_PROTOCOL) with open('MSD_audio_error_list.p', 'wb') as handle: pickle.dump(error_list, handle, protocol=pickle.HIGHEST_PROTOCOL) for i in range(num_part):
def MultiProcess(nproc, config, job_func, tasks, item, logger=None, done_func=None, except_func=None, except_abort=True): """A helper function for performing a task using multiprocessing. A note about the nomenclature here. We use the term "job" to mean the job of building a single file or image or stamp. The output of each job is gathered into the list of results that is returned. A task is a collection of one or more jobs that are all done by the same processor. For simple cases, each task is just a single job, but for things like a Ring test, the task needs to have the jobs for a full ring. The tasks argument is a list of tasks. Each task in that list is a list of jobs. Each job is a tuple consisting of (kwargs, k), where kwargs is the dict of kwargs to pass to the job_func and k is the index of this job in the full list of jobs. Parameters: nproc: How many processes to use. config: The configuration dict. job_func: The function to run for each job. It will be called as:: result = job_func(**kwargs) where kwargs is from one of the jobs in the task list. tasks: A list of tasks to run. Each task is a list of jobs, each of which is a tuple (kwargs, k). item: A string indicating what is being worked on. logger: If given, a logger object to log progress. [default: None] done_func: A function to run upon completion of each job. It will be called as:: done_func(logger, proc, k, result, t) where proc is the process name, k is the index of the job, result is the return value of that job, and t is the time taken. [default: None] except_func: A function to run if an exception is encountered. It will be called as:: except_func(logger, proc, k, ex, tr) where proc is the process name, k is the index of the job that failed, ex is the exception caught, and tr is the traceback. [default: None] except_abort: Whether an exception should abort the rest of the processing. If False, then the returned results list will not include anything for the jobs that failed. [default: True] Returns: a list of the outputs from job_func for each job """ import time import traceback # The worker function will be run once in each process. # It pulls tasks off the task_queue, runs them, and puts the results onto the results_queue # to send them back to the main process. # The *tasks* can be made up of more than one *job*. Each job involves calling job_func # with the kwargs from the list of jobs. # Each job also carries with it its index in the original list of all jobs. def worker(task_queue, results_queue, config, logger): proc = current_process().name # The logger object passed in here is a proxy object. This means that all the arguments # to any logging commands are passed through the pipe to the real Logger object on the # other end of the pipe. This tends to produce a lot of unnecessary communication, since # most of those commands don't actually produce any output (e.g. logger.debug(..) commands # when the logging level is not DEBUG). So it is helpful to wrap this object in a # LoggerWrapper that checks whether it is worth sending the arguments back to the original # Logger before calling the functions. logger = LoggerWrapper(logger) if 'profile' in config and config['profile']: import cProfile, pstats, io pr = cProfile.Profile() pr.enable() else: pr = None for task in iter(task_queue.get, 'STOP'): try: logger.debug('%s: Received job to do %d %ss, starting with %s', proc, len(task), item, task[0][1]) for kwargs, k in task: t1 = time.time() kwargs['config'] = config kwargs['logger'] = logger result = job_func(**kwargs) t2 = time.time() results_queue.put((result, k, t2 - t1, proc)) except KeyboardInterrupt: raise except Exception as e: tr = traceback.format_exc() logger.debug('%s: Caught exception: %s\n%s', proc, str(e), tr) results_queue.put((e, k, tr, proc)) logger.debug('%s: Received STOP', proc) if pr is not None: pr.disable() try: from StringIO import StringIO except ImportError: from io import StringIO s = StringIO() sortby = 'time' # Note: This is now called tottime, but time seems to be a valid # alias for this that is backwards compatible to older versions # of pstats. ps = pstats.Stats(pr, stream=s).sort_stats(sortby).reverse_order() ps.print_stats() logger.error( "*** Start profile for %s ***\n%s\n*** End profile for %s ***", proc, s.getvalue(), proc) njobs = sum([len(task) for task in tasks]) if nproc > 1: logger.warning("Using %d processes for %s processing", nproc, item) from multiprocessing import Process, Queue, current_process from multiprocessing.managers import BaseManager # Send the tasks to the task_queue. task_queue = Queue() for task in tasks: task_queue.put(task) # Temporarily mark that we are multiprocessing, so we know not to start another # round of multiprocessing later. config['current_nproc'] = nproc if 'profile' in config and config['profile']: logger.info( "Starting separate profiling for each of the %d processes.", nproc) # The logger is not picklable, so we need to make a proxy for it so all the # processes can emit logging information safely. logger_proxy = GetLoggerProxy(logger) # Run the tasks. # Each Process command starts up a parallel process that will keep checking the queue # for a new task. If there is one there, it grabs it and does it. If not, it waits # until there is one to grab. When it finds a 'STOP', it shuts down. results_queue = Queue() p_list = [] for j in range(nproc): # The process name is actually the default name that Process would generate on its # own for the first time we do this. But after that, if we start another round of # multiprocessing, then it just keeps incrementing the numbers, rather than starting # over at Process-1. As far as I can tell, it's not actually spawning more # processes, so for the sake of the logging output, we name the processes explicitly. p = Process(target=worker, args=(task_queue, results_queue, config, logger_proxy), name='Process-%d' % (j + 1)) p.start() p_list.append(p) raise_error = None try: # In the meanwhile, the main process keeps going. We pull each set of images off of the # results_queue and put them in the appropriate place in the lists. # This loop is happening while the other processes are still working on their tasks. results = [None for k in range(njobs)] for kk in range(njobs): res, k, t, proc = results_queue.get() if isinstance(res, Exception): # res is really the exception, e # t is really the traceback # k is the index for the job that failed if except_func is not None: # pragma: no branch except_func(logger, proc, k, res, t) if except_abort or isinstance(res, KeyboardInterrupt): for j in range(nproc): p_list[j].terminate() raise_error = res break else: # The normal case if done_func is not None: # pragma: no branch done_func(logger, proc, k, res, t) results[k] = res except BaseException as e: # pragma: no cover # I'm actually not sure how to make this happen. We do a good job of catching # all the normal kinds of exceptions that might occur and dealing with them cleanly. # However, if something happens that we didn't anticipate, this should make an # attempt to clean up the task queue and worker before raising the error further. logger.error( "Caught a fatal exception during multiprocessing:\n%r", e) logger.error("%s", traceback.format_exc()) # Clear any unclaimed jobs that are still in the queue while not task_queue.empty(): task_queue.get() # And terminate any jobs that might still be running. for j in range(nproc): p_list[j].terminate() raise_error = e finally: # Stop the processes # Once you are done with the processes, putting nproc 'STOP's will stop them all. # This is important, because the program will keep running as long as there are running # processes, even if the main process gets to the end. So you do want to make sure to # add those 'STOP's at some point! for j in range(nproc): task_queue.put('STOP') for j in range(nproc): p_list[j].join() task_queue.close() del config['current_nproc'] if raise_error is not None: raise raise_error else: # nproc == 1 results = [None] * njobs for task in tasks: for kwargs, k in task: try: t1 = time.time() kwargs['config'] = config kwargs['logger'] = logger result = job_func(**kwargs) t2 = time.time() if done_func is not None: # pragma: no branch done_func(logger, None, k, result, t2 - t1) results[k] = result except KeyboardInterrupt: raise except Exception as e: tr = traceback.format_exc() if except_func is not None: # pragma: no branch except_func(logger, None, k, e, tr) if except_abort or isinstance(e, KeyboardInterrupt): raise # If there are any failures, then there will still be some Nones in the results list. # Remove them. results = [r for r in results if r is not None] return results
print('ROOT: ') cmd = stdin.readline() q.put(cmd) def root(char): assert isinstance(char, str), "Argument must be string!" ROOT.gROOT.ProcessLine(char) if __name__ == '__main__': ___queue___ = Queue() ___newstdin___ = os.fdopen(os.dup(sys.stdin.fileno())) ___input_p___ = Process(target=input_thread, args=(___queue___, ___newstdin___)) ___input_p___.daemon = True ___input_p___.start() ___g___ = ROOT.gSystem.ProcessEvents try: while 1: if not ___queue___.empty(): ___cmd___ = ___queue___.get() try: exec(___cmd___, globals()) except: print(sys.exc_info()) time.sleep(0.01) ___g___() except (KeyboardInterrupt, SystemExit): ___input_p___.terminate() #
fvs = FileVideoStream('sample.mp4').start() while quit == False and frame_count < 500: frame = fvs.read() input_q.put([ time.time(), frame[int(roi[1]):int(roi[1] + roi[3]), int(roi[0]):int(roi[0] + roi[2])] ]) if output_q.empty(): pass # fill up queue else: frame_count += 1 dummylist = [] for i in range(output_q.qsize()): dummylist.append((quit, output_q.get())) dummylist.sort() for i in dummylist: display_q.put(i) fps.update() print(frame_count) fps.stop() quit = True print('[INFO] elapsed time (total): {:.2f}'.format(fps.elapsed())) print('[INFO] approx. FPS: {:.2f}'.format(fps.fps())) display_q.put((quit, output_q.get())) time.sleep(4) D.terminate() R.terminate()