def batchProcess(self, arr_to_enque, work_method, t=False): q = JoinableQueue() output = JoinableQueue() extra = JoinableQueue() third = JoinableQueue() if t: args = ((q, output, extra, third)) else: args=(q, output, extra) for obj in arr_to_enque: q.put(obj) processes = [Process(target=work_method, args=args, name=str(obj)) for obj in arr_to_enque] for p in processes: p.start() for p in processes: p.join(30) if p.is_alive(): print "ERROR JOINING PROCESS FOR: ", p.name p.terminate() raise Exception("Goal Conversion Error:", (self.account_id, self.project_id, exp_id, var_ids)) print "end batch process" if t: return (output, extra, third) else: return (output, extra)
class Mothership(object): """ Monitor of producer and consumers """ def __init__(self, producer, consumers): self._queue = JoinableQueue() self._producer_proxy = ProducerProxy(self._queue, producer) self._consumer_pool = list(ConsumerProxy(self._queue, consumer) \ for consumer in consumers) def start(self): """ Start working """ logger.info('Starting Producers'.center(20, '=')) self._producer_proxy.start() logger.info('Starting Consumers'.center(20, '=')) for consumer in self._consumer_pool: consumer.start() self._producer_proxy.join() self._queue.join() def __enter__(self): return self def __exit__(self, types, value, tb): return
def cdp_no_split_single(loaded_seq_list, loaded_seq_name_list, ref_file, nt, cores): """ Aligns a single SRNA_seq object to multiple refseq seqs in a Ref object at a time. No splitting of read counts. """ refs = RefSeq() refs.load_ref_file(ref_file) print(colored("------------------ALIGNING READS------------------\n", 'green')) workers = cores work_queue = JoinableQueue() processes = [] mgr = Manager() count = 0 counts_by_ref = mgr.dict() # {header:[count1, count2,.......]} for header, seq in refs: work_queue.put((header, seq,)) count += 1 if count % 10000 == 0: _cdp_no_split_single_queue(counts_by_ref, loaded_seq_list, nt, processes, work_queue, workers) _cdp_no_split_single_queue(counts_by_ref, loaded_seq_list, nt, processes, work_queue, workers) _cdp_single_output(counts_by_ref.copy(), loaded_seq_name_list, ref_file, nt)
def processData(imageList,featuresDir,featuresExt,task): numProcs = 8 taskQueue = JoinableQueue() resultQueue = Queue() processes = [] for i in range(numProcs): t = Process(target=worker, args=(taskQueue, resultQueue, task)) t.daemon = True t.start() processes.append(t) for img in imageList: filename = featuresDir+'/'+img+'.'+featuresExt idxFile = re.sub(r'\..+$',r'.idx',filename) content = open(filename) index = open(idxFile) taskQueue.put( (img,content.read(),index.read()) ) #taskQueue.put( (img,filename,idxFile) ) index.close() content.close() for i in range(len(processes)): taskQueue.put('stop') results = [] retrieved = 0 while retrieved < len(imageList): data = resultQueue.get() retrieved += 1 if data != 'Ignore': results.append(data) return results
def parallelPrepareImg(img, info, name, idx): # Make Color Image if img.ndim == 2: img = np.tile(img[:, :, np.newaxis], (1, 1, 3)) elif img.shape[2] == 4: img = img[:, :, :3] # Prepare processes numProcs = 3 taskQueue = JoinableQueue() resultQueue = ProcQueue() processes = [] for i in range(numProcs): t = Process(target=singleWindowProcess, args=(taskQueue, resultQueue, img)) t.daemon = True t.start() processes.append(t) j = 0 # Add tasks to the queue for b in info: idx.write(b[4]) taskQueue.put( (b,j) ) j += 1 for i in range(len(processes)): taskQueue.put('stop') # Collect results data = np.zeros([len(info), 227, 227, 3]) retrieved = 0 while retrieved < len(info): j,win = resultQueue.get() data[j,:,:,:] = win retrieved += 1 # Substract mean and return data -= imagenet.IMAGENET_MEAN[14:241,14:241,:] return data.swapaxes(2, 3).swapaxes(1, 2)
def task_writer(task: JoinableQueue): for n in News.objects.all()[:50].iterator(): task.put(n) for i in range(PROCESS_NUM): task.put("end") print("task writer ends")
def find_vocabulary(data_dir, stats_dir, category, min_num_images, save_description): print "Start find vocabulary" filequeue = JoinableQueue() photoqueue = Queue() init_dict = initialize_variables(None, None, False) # Create new processes num_processes = cpu_count() temp_dir = os.path.join(stats_dir, "database_temp", "vocab", category) if not os.path.exists(temp_dir): os.makedirs(temp_dir) processes = [FindVocabularyProcess(filequeue, photoqueue, init_dict, 30.0, num_processes, temp_dir, category) for i in xrange(num_processes)] for p in processes: p.start() #Add the files to the process queue add_files_to_queue(data_dir, category, filequeue) #Add a poison pill for each process for i in xrange(num_processes): filequeue.put("Stop") for p in processes: p.join() merge_vocabulary_files(data_dir, temp_dir, min_num_images, save_description) print "Removing temp files" shutil.rmtree(temp_dir) print "Done with find vocabulary"
def save_transaction_list(data_dir, stats_dir, category, concept_vocabulary, save_description): print "Start saving transaction list" filequeue = JoinableQueue() concept_vocabulary_list, concept_vocabulary_freq = zip(*concept_vocabulary) init_dict = initialize_variables(concept_vocabulary_list, None, True) # Create new processes temp_dir = os.path.join(stats_dir, "transaction_list") if not os.path.exists(temp_dir): os.makedirs(temp_dir) else: print "todo" lock = Lock() num_processes = cpu_count() processes = [TransactionListProcess(filequeue, init_dict, 30, num_processes, temp_dir, save_description, lock) for i in xrange(num_processes)] for p in processes: p.start() #Add the files to the process queue add_files_to_queue(data_dir, category, filequeue) #Add a poison pill for each process for i in xrange(num_processes): filequeue.put("Stop") for p in processes: p.join() print "Removing temp files" shutil.rmtree(temp_dir) print "Done with saving transaction list"
def scheduler(db,category): task=JoinableQueue() for i in range(cpu_count()): pid=os.fork() if pid==0: consumer(category,task) os._exit(0) # 防止子进程向下执行 # print('此处不会被执行') elif pid<0: logging.error('创建子进程失败') with ThreadPoolExecutor() as executor: cursor = db['image_match_result_{}'.format(category)].find( {'$or': [{'robot_processed': False}, {'robot_processed': {'$exists': False}}]}, {'_id': 1, 'b_image_url': 1, 'c_image_url': 1} ) for item in cursor: item['mark']=True # 标错 executor.submit(producer, item, task) cursor = db['item_match_result_{}'.format(category)].find( {'$or': [{'robot_processed': False}, {'robot_processed': {'$exists': False}}]}, {'_id': 1, 'b_image_url': 1, 'c_image_url': 1} ) for item in cursor: item['mark']=False # 标对 executor.submit(producer, item, task) task.join() os.kill(0,signal.SIGKILL)
def test_basic(): in_queue = JoinableQueue() algolia_reader = Algoliaio("MyAppID", "MyKey", 1000) algolia_reader.scan_and_queue(in_queue, p_index="INT_Rubriques",p_query=None, p_connect_timeout=30, p_read_timeout=60) assert in_queue.qsize() > 2600
def __init__(self, network_retries=SynchronousScanner.DEFAULT_NETWORK_RETRIES, network_timeout=SynchronousScanner.DEFAULT_NETWORK_TIMEOUT, max_processes_nb=_DEFAULT_MAX_PROCESSES_NB, max_processes_per_hostname_nb=_DEFAULT_PROCESSES_PER_HOSTNAME_NB): # type: (Optional[int], Optional[int], Optional[int], Optional[int]) -> None """Create a scanner for running scanning commands concurrently using a pool of processes. Args: network_retries (Optional[int]): How many times SSLyze should retry a connection that timed out. network_timeout (Optional[int]): The time until an ongoing connection times out. max_processes_nb (Optional[int]): The maximum number of processes to spawn for running scans concurrently. max_processes_per_hostname_nb (Optional[int]): The maximum number of processes that can be used for running scans concurrently against a single server. A lower value will reduce the chances of DOS-ing the server. """ self._network_retries = network_retries self._network_timeout = network_timeout self._max_processes_nb = max_processes_nb self._max_processes_per_hostname_nb = max_processes_per_hostname_nb # Create hostname-specific queues to ensure aggressive scan commands targeting this hostname are never # run concurrently self._hostname_queues_dict = {} self._processes_dict = {} self._task_queue = JoinableQueue() # Processes get tasks from task_queue and self._result_queue = JoinableQueue() # put the result of each task in result_queue self._queued_tasks_nb = 0
def __init__(self, p_max_items_by_queue=50000, p_forkserver=False, p_log_every=10000): """Class creation""" logger = logging.getLogger('swallow') if p_forkserver: mp.set_start_method('forkserver') self.readers = None self.writer = None self.writer_store_args = None self.process = None self.process_args = None if p_max_items_by_queue is None: self.in_queue = JoinableQueue() self.out_queue = JoinableQueue() else: if (sys.platform.lower() == 'darwin'): logger.warn("As running Swallow on a MacOS env, the number of items is limited to 32767.") p_max_items_by_queue = 32767 self.in_queue = JoinableQueue(p_max_items_by_queue) self.out_queue = JoinableQueue(p_max_items_by_queue) self.counters = { 'nb_items_processed': Value('i', 0), 'nb_items_error': Value('i', 0), 'nb_items_scanned': Value('i', 0), 'nb_items_stored': Value('i', 0), 'whole_storage_time': Value('f', 0), 'bulk_storage_time': Value('f', 0), 'whole_process_time': Value('f', 0), 'real_process_time': Value('f', 0), 'idle_process_time': Value('f', 0), 'scan_time': Value('f', 0), 'log_every': p_log_every }
class AlarmExecutor: def __init__(self): self.queue = JoinableQueue(10) self.running = False self.t = Thread(target=self._run, name="AlarmExecutor") def _run(self): while self.running: try: alarm = self.queue.get(block=True, timeout=1) alarm.execute() logging.debug("Alarm executed") self.queue.task_done() except Queue.Empty: continue def start(self): logging.debug("Starting alarm executor") self.running = True self.t.start() def stop(self): if self.running: logging.debug("Stoppping alarm executor") self.running = False self.t.join() else: logging.debug("Attempted to stop alarm executor when it is not running")
def main(): jobs = JoinableQueue() result = JoinableQueue() numToProcess = -1 scores = pd.DataFrame(columns=['query','fmeasure','precision','recall', 'size','maxDistance','topHits',"contextSteps"]) print len(datasets) for key in datasets: jobs.put(key) processed_count = Counter() for i in xrange(NUMBER_OF_PROCESSES): p = Process(target=work, args=(i, jobs, result, processed_count)) p.daemon = True p.start() #work(1, jobs, result, processed_count) automated_annotations = {} distances = {} jobs.join() dataset_index = collections.defaultdict(set) annotated_datasets = set() while not result.empty(): dataset, classes = result.get() if len(classes) == 0: annotated_datasets.add(dataset) for c in classes.keys(): dataset_index[c].add(dataset) owl_class = Class(c, graph=graph) for parent in owl_class.parents: dataset_index[parent.identifier].add(dataset) result.task_done() print '\n' for query, c in queries.items(): manual = ground_truth[query] automated = dataset_index[c] hits = manual & automated misses = manual - automated precision = np.nan if len(automated) == 0 else float(len(hits)) / len(automated) recall = np.nan if len(manual) == 0 else float(len(hits)) / len(manual) if precision != 0 or recall != 0: fmeasure = 0 if np.isnan(precision) or np.isnan(recall) else 2 * (precision * recall) / (precision + recall) else: fmeasure = 0 scores = scores.append(dict(query=query, size=len(manual), precision=precision, recall=recall, fmeasure=fmeasure,topHits=topHits, maxDistance=maxDistance, contextSteps = context_steps), ignore_index=True) print "Hits for", query, c print '\n'.join(sorted(hits)) print scores print "Annotated", len(annotated_datasets), "datasets."
def __init__(self, available_plugins, network_retries=DEFAULT_NETWORK_RETRIES, network_timeout=DEFAULT_NETWORK_TIMEOUT, max_processes_nb=DEFAULT_MAX_PROCESSES_NB, max_processes_per_hostname_nb=DEFAULT_PROCESSES_PER_HOSTNAME_NB): """ Args: available_plugins (PluginsFinder): An object encapsulating the list of available plugins. network_retries (Optional[int)]: How many times plugins should retry a connection that timed out. network_timeout (Optional[int]): The time until an ongoing connection times out within all plugins. max_processes_nb (Optional[int]): The maximum number of processes to spawn for running scans concurrently. max_processes_per_hostname_nb (Optional[int]): The maximum of processes that can be used for running scans concurrently on a single server. Returns: PluginsProcessPool: An object for queueing scan commands to be run concurrently. """ self._available_plugins = available_plugins self._network_retries = network_retries self._network_timeout = network_timeout self._max_processes_nb = max_processes_nb self._max_processes_per_hostname_nb = max_processes_per_hostname_nb # Create hostname-specific queues to ensure aggressive scan commands targeting this hostname are never # run concurrently self._hostname_queues_dict = {} self._processes_dict = {} self._task_queue = JoinableQueue() # Processes get tasks from task_queue and self._result_queue = JoinableQueue() # put the result of each task in result_queue self._queued_tasks_nb = 0
class QuickReader(): def __init__(self, writer_name,handle_raw_assertion,add_lines_to_queue, isTest = False, num_threads = 5): self.writer_name = writer_name self.num_threads = num_threads self.handle_raw_assertion = handle_raw_assertion self.add_lines_to_queue = add_lines_to_queue self.queue = JoinableQueue() self.isTest = isTest def start(self): print "begin writing " + self.writer_name self.create_processes() self.add_lines_to_queue(self.queue) self.queue.join() print "finished writing " + self.writer_name def pull_lines(self,q,writer): while 1: raw_assertion = q.get() edges = self.handle_raw_assertion(raw_assertion) for edge in edges: writer.write(edge) q.task_done() def create_processes(self): for i in range(self.num_threads): writer = MultiWriter(self.writer_name + "_" + str(i),self.isTest) p = Process(target = self.pull_lines, args = (self.queue, writer)) #p.daemon=True p.start()
def main(): num_page = 6000 num_processes = 60 num_works = num_page / num_processes q = JoinableQueue() pool = list() final_set = set() for index in xrange(1,num_processes+1): p = Process(target=fetch_feature,args=(q,index,num_works)) p.start() for index in xrange(1,num_processes+1): final_set = final_set.union(q.get()) #p.join() # pool.append(p) #for p in pool: # p.join() result_file = open('result.out','w'); for feature in final_set: print feature result_file.write(feature+'\n') result_file.close() print len(final_set)
def main(): fetch_queue = JoinableQueue() reject_queue = JoinableQueue(maxsize = 1000) log_processor = Process(target=job_creator, args=(fetch_queue, './search_log_valid_2010_06_17'), name='log-processor') writers = [ ] write_queues = [] for num in DATA_SETS: queue, writer = create_writer(reject_queue, num) writers.append(writer) write_queues.append(queue) fetchers = [ create_fetcher(fetch_queue, write_queues, reject_queue, num) for num in xrange(NUM_FETCHERS) ] reject_writer = Process(target=reject, args=(reject_queue, './rejected-lines'), name='related-search-reject-writer') log_processor.start() reject_writer.start() start_processes(writers) start_processes(fetchers) log_processor.join() print 'DONE? ' fetch_queue.join() write_queue.join() reject_writer.join()
def test_basic(): in_queue = JoinableQueue() mongo_reader = Mongoio(p_host='localhost',p_port='27017',p_user='******',p_password='******',p_base='ACTIVITE',p_rs_xtra_nodes=['localhost:27018','localhost:27019'],p_rs_name='rs0') mongo_reader.scan_and_queue(in_queue,p_collection='rubriques', p_query={}) assert in_queue.qsize() > 2600
class emailSubsystem(object): def __init__(self): ### will move to Celery eventually; with Celery, the app would be able to periodically # wakeup and check on replyQueue to see which emails were send, which were not and # what to do ... self.emailQueue = JoinableQueue() self.replyQueue = JoinableQueue() self.worker = Process(target=sendEmailWorker, args=(self.emailQueue, self.replyQueue)) def start(self): # temporarily comment out starting a new process as it seems to leave zombies # and causes app not to start as max process limit is reached. #self.worker.start() return def shutdown(self): # post poison pill # wait on the queue to be done; ie join on emailQueue # wait on the worker process to die; ie join on worker self.emailQueue.put(None) self.emailQueue.join() self.worker.join()
def __init__(self, config, maxCpus, maxMemory): AbstractBatchSystem.__init__(self, config, maxCpus, maxMemory) #Call the parent constructor if maxMemory != sys.maxint: logger.critical("A max memory has been specified for the parasol batch system class of %i, but currently this batchsystem interface does not support such limiting" % maxMemory) #Keep the name of the results file for the pstat2 command.. self.parasolCommand = config.attrib["parasol_command"] self.parasolResultsFile = getParasolResultsFileName(config.attrib["job_tree"]) #Reset the job queue and results (initially, we do this again once we've killed the jobs) self.queuePattern = re.compile("q\s+([0-9]+)") self.runningPattern = re.compile("r\s+([0-9]+)\s+[\S]+\s+[\S]+\s+([0-9]+)\s+[\S]+") self.killJobs(self.getIssuedJobIDs()) #Kill any jobs on the current stack logger.info("Going to sleep for a few seconds to kill any existing jobs") time.sleep(5) #Give batch system a second to sort itself out. logger.info("Removed any old jobs from the queue") #Reset the job queue and results exitValue = popenParasolCommand("%s -results=%s clear sick" % (self.parasolCommand, self.parasolResultsFile), False)[0] if exitValue != None: logger.critical("Could not clear sick status of the parasol batch %s" % self.parasolResultsFile) exitValue = popenParasolCommand("%s -results=%s flushResults" % (self.parasolCommand, self.parasolResultsFile), False)[0] if exitValue != None: logger.critical("Could not flush the parasol batch %s" % self.parasolResultsFile) open(self.parasolResultsFile, 'w').close() logger.info("Reset the results queue") #Stuff to allow max cpus to be work self.outputQueue1 = Queue() self.outputQueue2 = Queue() #worker = Thread(target=getUpdatedJob, args=(self.parasolResultsFileHandle, self.outputQueue1, self.outputQueue2)) #worker.setDaemon(True) worker = Process(target=getUpdatedJob, args=(self.parasolResultsFile, self.outputQueue1, self.outputQueue2)) worker.daemon = True worker.start() self.usedCpus = 0 self.jobIDsToCpu = {}
class ProcessPool(object): def __init__(self, size=1): self.size = size self.jobs = Queue() self.results = Queue() self.processes = [] def start(self): '''start all processes''' for i in range(self.size): self.processes.append(ProcessWorker(self)) for process in self.processes: process.start() def append_job(self, job, *args, **kwargs): self.jobs.put((job, args, kwargs)) def join(self): '''waiting all jobs done''' self.jobs.join() def stop(self): '''kill all processes''' for process in self.processes: process.stop() for process in self.processes: # waiting processes completing if process.is_alive(): process.join() del self.processes[:] # reset processes to empty
class SimpleSynergeticServer(Process): def __init__(self, authen_key): Process.__init__(self) self.__task_queue = JoinableQueue(1) self.__return_queue = Queue(1) self.serv = Listener(('', 40000), authkey=authen_key) def run(self): print 'Server Works' copy_reg.pickle(types.MethodType, _reduce_method) #Start the synergeticProcess in Deamon Mode worker_p = SynergeticProcess(self.__task_queue, self.__return_queue) worker_p.deamon = True worker_p.start() while True: print 'wait for Client' pool_conn = self.serv.accept() print 'connection Client Accepted' while True: print 'in LOOP Simple Server' #There is no need for task_id in this version try: print 'Try to recv MSG' unpickled_msg = pool_conn.recv() print 'MSG Reseved' except Exception as e: # EOFError: print 'Fail To Receive MSG:', e break if unpickled_msg[0] == 'MODULES': self.import_mods( unpickled_msg[1] ) ret = 'MODULES-READY' else: self.__task_queue.put(unpickled_msg) ret = self.__return_queue.get() try: print 'SEND RESPONCE' try: pool_conn.send( ret ) except EOFError: print 'SENT TO POOL FAILD' print 'RESPONCE SENT ', ret except EOFError: break pool_conn.close() def import_mods(self, mods_d): for mod_name, mod_bytecode in mods_d.items(): try: fobj = open(mod_name + ".pyc", 'wb') except Exception as e: print("Synergeticprocessing.SimpleServer --> Module file error: %s" % e) else: fobj.write( mod_bytecode ) finally: fobj.close() for mod in mods_d: print 'blocking' __import__( mod ) print 'imported ', mod
def aggress(map): global startMap startMap = map #print "Regressing..." state = State() jobs = [] longestSolution = Value('d', 20) highestScore = Value('d', 0) queue = JoinableQueue() manager = Manager() d = manager.dict() d.clear() l = RLock() if multiProc: queue.put((state, map, 1)) for i in range(numProcs): p = Process(target = multiMain, args=(startMap, l, d, queue,highestScore)) p.start() queue.join() else: a(l, highestScore, d, None, state, map, 1)
class QueueTask: def __init__(self): self.queue = JoinableQueue() self.event = Event() atexit.register( self.queue.join ) process = Process(target=self.work) process.daemon = True process.start() def work(self): while True: func, args, wait_for = self.queue.get() for evt in wait_for: evt.wait() func(*args) self.event.set() self.queue.task_done() def enqueue(self, func, args=[], wait_for=[]): self.event.clear() self.queue.put( (func, args, wait_for) ) return self.event
def evaluate(points,meshToBasis,kernel,quadRule,coeffs,nprocs=None): """Evaluate a kernel using the given coefficients""" if nprocs==None: nprocs=cpu_count() inputQueue=JoinableQueue() nelements=meshToBasis.nelements for elem in meshToBasis: inputQueue.put(elem) buf=sharedctypes.RawArray('b',len(points[0])*numpy.dtype(numpy.complex128).itemsize) result=numpy.frombuffer(buf,dtype=numpy.complex128) result[:]=numpy.zeros(1,dtype=numpy.complex128) time.sleep(.5) workers=[] for id in range(nprocs): worker=EvaluationWorker(points,kernel,quadRule,coeffs,inputQueue,result) worker.start() workers.append(worker) inputQueue.join() for worker in workers: worker.join() return result.copy()
def queue_info(iters=None,): work = JoinableQueue() for filename in iters: work.put(obj=filename) time.sleep(1) return work
def getdata_multiprocess(self,task_funcsiter=None,task_funcsiterparas={}, task_funcsconst=None,task_funcsconstparas={},processnum=None, threadnum=2): def _start_processes(taskqueue,resultqueue,taskqueue_lk,task_funcsconst, task_funcsconstparas,processnum,threadnum): for i in range(processnum): p = Process(target=self.multiprocess_task, args=(taskqueue,resultqueue, taskqueue_lk,threadnum, task_funcsconst,task_funcsconstparas ),name='P'+str(i)) p.daemon=True p.start() processnum=processnum if processnum else multiprocessing.cpu_count() #任务传送queue taskqueue=JoinableQueue() #任务写入/唤醒lock taskqueue_lk = multiprocessing.Condition(multiprocessing.Lock()) #结果传送queue resultqueue=Queue() _start_processes(taskqueue,resultqueue,taskqueue_lk,task_funcsconst, task_funcsconstparas,processnum,threadnum) #放入任务,唤醒进程 if task_funcsconst is None: self._put_tasks(zip(task_funcsiter,task_funcsiterparas),taskqueue,taskqueue_lk) else: self._put_tasks(task_funcsiterparas,taskqueue,taskqueue_lk) logger.info('main join!') taskqueue.join() logger.info('main end!') return self._get_results(resultqueue)
def __init__(self, p_max_items_by_queue=50000, p_forkserver=False, p_log_every=10000): """Class creation""" if p_forkserver: mp.set_start_method('forkserver') self.readers = None self.writer = None self.writer_store_args = None self.process = None self.process_args = None if p_max_items_by_queue is None: self.in_queue = JoinableQueue() self.out_queue = JoinableQueue() else: self.in_queue = JoinableQueue(p_max_items_by_queue) self.out_queue = JoinableQueue(p_max_items_by_queue) self.counters = { 'nb_items_processed': Value('i', 0), 'nb_items_error': Value('i', 0), 'nb_items_scanned': Value('i', 0), 'nb_items_stored': Value('i', 0), 'whole_storage_time': Value('f', 0), 'bulk_storage_time': Value('f', 0), 'whole_process_time': Value('f', 0), 'real_process_time': Value('f', 0), 'idle_process_time': Value('f', 0), 'scan_time': Value('f', 0), 'log_every': p_log_every }
def run(self): # Changes the process name shown by ps for instance setProcTitle ("agentcluster master [version: %s] [monitoring: %d seconds]" % (__version__,self.monitoring_period) ); try: logger.info ( 'Agent cluster server starting' ); logger.info ( 'Configurations will be scanned in directories:' ); for directory in confdir.data: logger.info ( ' o %s', os.path.abspath(directory) ); self.watchdog = Watchdog(self.monitoring_period) self.watchdog.start() # Generates a deadlock to enter in sleep mode # Only an external signal can break this deadlock logger.info ( 'Agent cluster server started' ); queue = JoinableQueue() queue.put(object()); queue.join(); except KeyboardInterrupt: logger.info ( 'Agent cluster server interrupted' ); except Exception: logger.error ( 'Exception catched in main process: %s', sys.exc_info()[1] ); logger.debug ( "", exc_info=True ); finally: # First stop the monitoring to avoid restarting killed agents if self.watchdog is not None: self.watchdog.shutdown = True self.watchdog.join() logger.info ( 'Agent cluster server end' ); logging.shutdown()
for x in range(2, int(math.sqrt(n)) + 1): if n % x == 0: self.answer = False return self.answer = True if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('-f', '--file', type=str, required=True) parser.add_argument('-p', '--processes', type=int, default=2) args = parser.parse_args() queue = JoinableQueue() workers = [Worker(queue, i) for i in range(args.processes)] for worker in workers: worker.start() tasks_file = open(args.file, 'r') begin_time = time.time() while True: task_str = tasks_file.readline() if not task_str: break task = task_str.split('\t') if task[0] == 'prime': queue.put(PrimeTask(int(task[1]))) elif task[0] == 'check':
def fit_for_all(drop_non_countries=False): """Main function to perform fit for all countries.""" #################################################################### # Read files train_df = pd.read_csv(TRAIN_FILE, encoding='cp1252', index_col='Country Name').dropna(axis=0) test_df = pd.read_csv(TEST_FILE, encoding='cp1252', index_col='Country Name').dropna(axis=0) # The test_df has one extra country. Line up train and test. test_df = test_df.loc[train_df.index] if drop_non_countries: train_df = train_df.drop(NON_COUNTRIES) test_df = test_df.drop(NON_COUNTRIES) # Get matrices. train_mat = train_df.values.T.astype(int) test_mat = test_df.values.T.astype(int) # Grab list and number of countries for convenience. countries = train_df.index.values num_countries = countries.shape[0] # Initialize queues for parallel processing. queue_in = JoinableQueue() queue_out = Queue() # Start processes. processes = [] for i in range(NUM_PROCESSES): p = Process(target=fit_for_country_worker, args=(train_mat, test_mat, queue_in, queue_out)) p.start() processes.append(p) # Loop over all the countries (columns of the train matrix). for i in range(num_countries): # Put boolean array in the queue. queue_in.put((i, num_countries)) # Wait for processing to finish. queue_in.join() # Track coefficients. best_coeff = pd.DataFrame(0.0, columns=countries, index=countries) # Track training scores. best_scores = pd.Series(0.0, index=countries) # Track predictions. predictions = pd.DataFrame(0.0, columns=test_df.columns, index=countries) # Map data. for _ in range(num_countries): # Grab data from the queue. other_countries, s, c, p = queue_out.get() country = countries[~other_countries][0] # Map. best_scores.loc[~other_countries] = s best_coeff.loc[other_countries, country] = c # p needs to be transformed (17x1 vs 1x17) predictions.loc[~other_countries, :] = p.T # Shut down processes. for p in processes: queue_in.put_nowait(None) p.terminate() predictions.transpose().to_csv(PRED_OUT, index_label='Id', encoding='cp1252') best_coeff.to_csv(COEFF_OUT, encoding='cp1252') # Print MSE print('Summary of MSE:') print(best_scores.describe())
def fn_main(self): apk_list = self.fn_parse_args_and_populate_apk_list() # Keep track of the number of processes we create. num_processes = 0 if apk_list == []: print('All APK files checked.') sys.exit(0) length_apk_list = len(apk_list)/NUMBER_OF_PROCESSES length_apk_list = int(length_apk_list) print( 'Total number of APKs: ' + str(len(apk_list)) + '\nApproximate number of APKs per thread: ' + str(length_apk_list) ) # Free up memory checked_apks = None # Create two process queues: # one for sending data to, and one for receiving data from, # the worker process(es). process_extractor_send_queue = JoinableQueue() process_extractor_receive_queue = JoinableQueue() # List for keeping track of processes. self.process_list = [] # Create worker processes. for i in range(0, NUMBER_OF_PROCESSES): worker_uuid_process = UUIDExtractor( self.trace_params, self.named_special_case_object, self.nonnamed_special_case_object ) worker = Process( target=worker_uuid_process.fn_main, args=( process_extractor_send_queue, process_extractor_receive_queue, num_processes ) ) worker.start() self.process_list.append(worker) num_processes += 1 apks_to_check = 0 # Send work to worker process. for match in apk_list: if os.path.isfile(match): apks_to_check += 1 process_extractor_send_queue.put(str(match)) else: apk_list.remove(match) if os.path.isfile(self.error_log): fo_error = open(self.error_log, 'a') else: fo_error = open(self.error_log, 'w') completed_apk_count = 0 while True: # Get information sent by worker process. [analysed_file, pkg, status, result] = process_extractor_receive_queue.get() process_extractor_receive_queue.task_done() # Analyse the output string. if status == STATUS_ERROR: print('\n Error encountered with ' + analysed_file + ': ' + result + '\n') # Log the error to a separate file. fo_error.write(analysed_file+','+result+'\n') completed_apk_count += 1 elif status == STATUS_LOGGING: print(result) elif status == STATUS_DONE: print('\n Finished analysing ' + analysed_file + ' with result ' + str(result) + '\n') # Write the output to temporary JSON. raw_filename = self.fn_get_filename_from_path(analysed_file) filename_no_ext = raw_filename.replace('.apk', '') json_obj = {} json_obj[filename_no_ext] = {} json_obj[filename_no_ext]['pkg'] = pkg json_obj[filename_no_ext]['uuids'] = result outfile = os.path.join( self.io_tmp_dir, raw_filename.replace('.apk', '.json') ) with open(outfile, 'w') as f: json.dump(json_obj, f, indent=4) completed_apk_count += 1 else: print('Unhandled condition from worker.') # Check if any processes have become zombies. if len(active_children()) < NUMBER_OF_PROCESSES: for p in self.process_list: if not p.is_alive(): self.process_list.remove(p) # Create a new process in its place. worker_uuid_process = UUIDExtractor( self.trace_params, self.named_special_case_object, self.nonnamed_special_case_object ) replacement_worker = Process( target=worker_uuid_process.fn_main, args=( process_extractor_send_queue, process_extractor_receive_queue, num_processes ) ) replacement_worker.start() self.process_list.append(replacement_worker) num_processes += 1 # Check if all APKs have been analysed. if completed_apk_count == apks_to_check: break print('All done.') # Tell child processes to stop for i in range(NUMBER_OF_PROCESSES): process_extractor_send_queue.put('STOP') # Collate. self.fn_collate_json()
def __init__(self, config_in=None, min_occur=10, min_percent=5, window=2, threads=8, period=24, min_interval=2, es_host='localhost', es_port=9200, es_timeout=480, es_index='logstash-flow-*', kibana_version='4', verbose=True, debug=True): """ :param min_occur: Minimum number of triads to be considered beaconing :param min_percent: Minimum percentage of all connection attempts that must fall within the window to be considered beaconing :param window: Size of window in seconds in which we group connections to determine percentage, using a large window size can give inaccurate interval times, multiple windows contain all interesting packets, so the first window to match is the interval :param threads: Number of cores to use :param period: Number of hours to locate beacons for :param min_interval: Minimum interval betweeen events to consider for beaconing behavior :param es_host: IP Address of elasticsearch host (default is localhost) :param es_timeout: Sets timeout to 480 seconds :param kibana_version: 4 or 5 (query will depend on version) """ #self.config_in = config_in if config_in is not None: try: self.config = flareConfig(config_in) self.es_host = self.config.get('beacon', 'es_host') self.es_port = int(self.config.get('beacon', 'es_port')) self.es_index = self.config.get('beacon', 'es_index') self.use_ssl = self.config.config.getboolean('beacon', 'use_ssl') self.MIN_OCCURRENCES = int(self.config.get('beacon','min_occur')) self.MIN_PERCENT = int(self.config.get('beacon','min_percent')) self.WINDOW = int(self.config.get('beacon','window')) self.NUM_PROCESSES = int(self.config.get('beacon','threads')) self.period = int(self.config.get('beacon','period')) self.min_interval = int(self.config.get('beacon', 'min_interval')) self.es_timeout = int(self.config.get('beacon','es_timeout')) self.kibana_version = self.config.get('beacon','kibana_version') self.beacon_src_ip = self.config.get('beacon','field_source_ip') self.beacon_dest_ip = self.config.get('beacon', 'field_destination_ip') self.beacon_destination_port = self.config.get('beacon', 'field_destination_port') self.beacon_timestamp = self.config.get('beacon', 'field_timestamp') self.beacon_flow_bytes_toserver = self.config.get('beacon', 'field_flow_bytes_toserver') self.beacon_flow_id = self.config.get('beacon', 'field_flow_id') self.beacon_event_key = self.config.get('beacon','event_key') self.beacon_event_type = self.config.get('beacon','event_type') self.filter = self.config.get('beacon','filter') self.verbose = self.config.config.getboolean('beacon', 'verbose') self.auth_user = self.config.config.get('beacon','username') self.auth_password = self.config.config.get('beacon', 'password') self.suricata_defaults = self.config.config.getboolean('beacon','suricata_defaults') try: self.debug = self.config.config.getboolean('beacon', 'debug') except: pass except Exception as e: print(('{red}[FAIL]{endc} Could not properly load your config!\nReason: {e}'.format(red=bcolors.FAIL, endc=bcolors.ENDC, e=e))) sys.exit(0) else: self.es_host = es_host self.es_port = es_port self.es_index = es_index self.use_ssl = False self.MIN_OCCURRENCES = min_occur self.MIN_PERCENT = min_percent self.WINDOW = window self.NUM_PROCESSES = threads self.period = period self.min_interval = min_interval self.kibana_version = kibana_version self.es_timeout = es_timeout self.beacon_src_ip = 'src_ip' self.beacon_dest_ip = 'dest_ip' self.beacon_destination_port = 'dest_port' self.beacon_timestamp = '@timestamp' self.beacon_flow_bytes_toserver = 'bytes_toserver' self.beacon_flow_id = 'flow_id' self.beacon_event_type = 'flow' self.beacon_event_key = 'event_type' self.filter = '' self.verbose = verbose self.suricata_defaults = False self.ver = {'4': {'filtered': 'query'}, '5': {'bool': 'must'}} self.filt = list(self.ver[self.kibana_version].keys())[0] self.query = list(self.ver[self.kibana_version].values())[0] self.whois = WhoisLookup() self.info = '{info}[INFO]{endc}'.format(info=bcolors.OKBLUE, endc=bcolors.ENDC) self.success = '{green}[SUCCESS]{endc}'.format(green=bcolors.OKGREEN, endc=bcolors.ENDC) self.fields = [self.beacon_src_ip, self.beacon_dest_ip, self.beacon_destination_port, self.beacon_flow_bytes_toserver, 'dest_degree', 'occurrences', 'percent', 'interval'] try: _ = (self.auth_user, self.auth_password) self.auth = "Enabled" except AttributeError as e: self.auth = "None" try: self.vprint('{info}[INFO]{endc} Attempting to connect to elasticsearch...'.format(info=bcolors.OKBLUE, endc=bcolors.ENDC)) if self.auth == "None": self.es = Elasticsearch(self.es_host, port=self.es_port, timeout=self.es_timeout, verify_certs=False, use_ssl=self.use_ssl, connection_class=RequestsHttpConnection) else: self.es = Elasticsearch(self.es_host, port=self.es_port, timeout=self.es_timeout, http_auth=(self.auth_user, self.auth_password), verify_certs=False, use_ssl=self.use_ssl, connection_class=RequestsHttpConnection) self.vprint('{green}[SUCCESS]{endc} Connected to elasticsearch on {host}:{port}'.format(green=bcolors.OKGREEN, endc=bcolors.ENDC, host=self.es_host, port=str(self.es_port))) except Exception as e: self.vprint(e) raise Exception( "Could not connect to ElasticSearch -- Please verify your settings are correct and try again.") self.q_job = JoinableQueue() self.l_df = Lock() self.l_list = Lock() self.high_freq = None self.flow_data = self.run_query()
def writer_loop(queue: JoinableQueue): stop = False file = None group = None while not stop: item = queue.get() assert 'action' in item action = item['action'] # get parameter if any parameters = {} if 'parameters' in item: parameters = item['parameters'] if action == 'create_file': assert 'fname' in parameters fname = parameters['fname'] file = h5py.File(fname, 'w', libver='latest') group = file.create_group('group') pass elif action == 'set_swmr_mode': file.swmr_mode = True elif action == 'create_dataset': assert 'name' in parameters assert 'value' in parameters name = parameters['name'] data = parameters['value'] group.create_dataset(name, maxshape=data.shape, data=data) elif action == 'update_dataset': assert 'name' in parameters assert 'value' in parameters name = parameters['name'] data = parameters['value'] group[name][:] = data elif action == 'create_attribute': assert 'name' in parameters assert 'value' in parameters name = parameters['name'] value = parameters['value'] if isinstance(value, str): value = np.string_(value) group.attrs.create(name, value) elif action == 'update_attribute': assert 'name' in parameters assert 'value' in parameters name = parameters['name'] value = parameters['value'] if isinstance(value, str): value = np.string_(value) group.attrs[name] = value elif action == 'flush_file': file.flush() elif action == 'flush_group': group.flush() elif action == 'close_file': file.close() elif action == 'stop': stop = True else: assert False if action == 'stop': stop = True queue.task_done()
q.put(res) q.join() def consumer(q): while True: res = q.get() if res is None: break time.sleep(1) print('消费者吃了%s' % res) q.task_done() # 发出信号项目已处理完成,省去了后面需另外添加的q.put(None) if __name__ == '__main__': # 容器 q = JoinableQueue() # 生产者们 p1 = Process(target=producer, args=(q, )) p2 = Process(target=producer, args=(q, )) p3 = Process(target=producer, args=(q, )) # 消费者们 c1 = Process(target=consumer, args=(q, )) c2 = Process(target=consumer, args=(q, )) # c1.daemon = True # 若不添加守护进程,则会卡在消费者 # c2.daemon = True p1.start() p2.start() p3.start()
def main(): freeze_support() #--PLUGINS INITIALIZATION-- start_time = time() print '\n\n\n' + _format_title('Registering available plugins') sslyze_plugins = PluginsFinder() available_plugins = sslyze_plugins.get_plugins() available_commands = sslyze_plugins.get_commands() print '' for plugin in available_plugins: print ' ' + plugin.__name__ print '\n\n' # Create the command line parser and the list of available options sslyze_parser = CommandLineParser(available_plugins, PROJECT_VERSION) try: # Parse the command line (command_list, target_list, shared_settings) = sslyze_parser.parse_command_line() except CommandLineParsingError as e: print e.get_error_msg() return #--PROCESSES INITIALIZATION-- # Three processes per target from MIN_PROCESSES up to MAX_PROCESSES nb_processes = max(MIN_PROCESSES, min(MAX_PROCESSES, len(target_list) * 3)) if command_list.https_tunnel: nb_processes = 1 # Let's not kill the proxy task_queue = JoinableQueue() # Processes get tasks from task_queue and result_queue = JoinableQueue( ) # put the result of each task in result_queue # Spawn a pool of processes, and pass them the queues process_list = [] for _ in xrange(nb_processes): priority_queue = JoinableQueue() # Each process gets a priority queue p = WorkerProcess(priority_queue, task_queue, result_queue, available_commands, \ shared_settings) p.start() process_list.append( (p, priority_queue)) # Keep track of each process and priority_queue #--TESTING SECTION-- # Figure out which hosts are up and fill the task queue with work to do print _format_title('Checking host(s) availability') targets_OK = [] targets_ERR = [] # Each server gets assigned a priority queue for aggressive commands # so that they're never run in parallel against this single server cycle_priority_queues = cycle(process_list) target_results = ServersConnectivityTester.test_server_list( target_list, shared_settings) for target in target_results: if target is None: break # None is a sentinel here # Send tasks to worker processes targets_OK.append(target) (_, current_priority_queue) = cycle_priority_queues.next() for command in available_commands: if getattr(command_list, command): args = command_list.__dict__[command] if command in sslyze_plugins.get_aggressive_commands(): # Aggressive commands should not be run in parallel against # a given server so we use the priority queues to prevent this current_priority_queue.put((target, command, args)) else: # Normal commands get put in the standard/shared queue task_queue.put((target, command, args)) for exception in target_results: targets_ERR.append(exception) print ServersConnectivityTester.get_printable_result( targets_OK, targets_ERR) print '\n\n' # Put a 'None' sentinel in the queue to let the each process know when every # task has been completed for (proc, priority_queue) in process_list: task_queue.put(None) # One sentinel in the task_queue per proc priority_queue.put(None) # One sentinel in each priority_queue # Keep track of how many tasks have to be performed for each target task_num = 0 for command in available_commands: if getattr(command_list, command): task_num += 1 # --REPORTING SECTION-- processes_running = nb_processes # XML output xml_output_list = [] # Each host has a list of results result_dict = {} for target in targets_OK: result_dict[target] = [] # If all processes have stopped, all the work is done while processes_running: result = result_queue.get() if result is None: # Getting None means that one process was done processes_running -= 1 else: # Getting an actual result (target, command, plugin_result) = result result_dict[target].append((command, plugin_result)) if len(result_dict[target]) == task_num: # Done with this target # Print the results and update the xml doc print _format_txt_target_result(target, result_dict[target]) if shared_settings['xml_file']: xml_output_list.append( _format_xml_target_result(target, result_dict[target])) result_queue.task_done() # --TERMINATE-- # Make sure all the processes had time to terminate task_queue.join() result_queue.join() #[process.join() for process in process_list] # Causes interpreter shutdown errors exec_time = time() - start_time # Output XML doc to a file if needed if shared_settings['xml_file']: result_xml_attr = { 'httpsTunnel': str(shared_settings['https_tunnel_host']), 'totalScanTime': str(exec_time), 'defaultTimeout': str(shared_settings['timeout']), 'startTLS': str(shared_settings['starttls']) } result_xml = Element('results', attrib=result_xml_attr) # Sort results in alphabetical order to make the XML files (somewhat) diff-able xml_output_list.sort(key=lambda xml_elem: xml_elem.attrib['host']) for xml_element in xml_output_list: result_xml.append(xml_element) xml_final_doc = Element('document', title="SSLyze Scan Results", SSLyzeVersion=PROJECT_VERSION, SSLyzeWeb=PROJECT_URL) # Add the list of invalid targets xml_final_doc.append( ServersConnectivityTester.get_xml_result(targets_ERR)) # Add the output of the plugins xml_final_doc.append(result_xml) # Hack: Prettify the XML file so it's (somewhat) diff-able xml_final_pretty = minidom.parseString( tostring(xml_final_doc, encoding='UTF-8')) with open(shared_settings['xml_file'], 'w') as xml_file: xml_file.write( xml_final_pretty.toprettyxml(indent=" ", encoding="utf-8")) print _format_title('Scan Completed in {0:.2f} s'.format(exec_time))
class WholeSlideTiler(object): """Handles generation of tiles and metadata for all images in a slide.""" def __init__(self, slide_path, outpath, img_format, tile_size, overlap, limit_bounds, rotate, quality, nworkers, only_last): self._slide = open_slide(slide_path) # the whole slide image self._outpath = outpath # baseline name of each tiled image self._img_format = img_format # image format (jpeg or png) self._tile_size = tile_size # tile size. default: 256x256 pixels self._overlap = overlap self._limit_bounds = limit_bounds self._queue = JoinableQueue( 2 * nworkers) # setup multiprocessing worker queues. self._nworkers = nworkers # number of workers self._only_last = only_last self._dzi_data = {} for _i in range(nworkers): TileWorker(self._queue, slide_path, tile_size, overlap, limit_bounds, rotate, quality).start() def run(self): self._run_image() for name in self._slide.associated_images: self._run_image(name) # self._write_static() self._shutdown() def _run_image(self, associated=None): """Run a single image from self._slide.""" if associated is None: image = self._slide outpath = self._outpath else: image = ImageSlide(self._slide.associated_images[associated]) outpath = os.path.join(self._outpath, self._slugify(associated)) dz = DeepZoomGenerator(image, self._tile_size, self._overlap, self._limit_bounds) tiler = SingleImageTiler(dz, outpath, self._img_format, associated, self._queue, self._only_last) tiler.run() self._dzi_data[self._url_for(associated)] = tiler.get_dzi() def _url_for(self, associated): if associated is None: base = 'slide' else: base = self._slugify(associated) return '%s.dzi' % base @staticmethod def _copydir(src, dest): if not os.path.exists(dest): os.makedirs(dest) for name in os.listdir(src): srcpath = os.path.join(src, name) if os.path.isfile(srcpath): shutil.copy(srcpath, os.path.join(dest, name)) @classmethod def _slugify(cls, text): text = normalize('NFKD', text.lower()).encode('ascii', 'ignore').decode() return re.sub('[^a-z0-9]+', '_', text) def _shutdown(self): for _i in range(self._nworkers): self._queue.put(None) self._queue.join()
def __init__(self, config_location="/etc/redeem"): """ config_location: provide the location to look for config files. - default is installed directory - allows for running in a local directory when debugging """ firmware_version = "1.2.8~Predator" logging.info("Redeem initializing " + firmware_version) printer = Printer() self.printer = printer Path.printer = printer printer.firmware_version = firmware_version printer.config_location = config_location # Set up and Test the alarm framework Alarm.printer = self.printer Alarm.executor = AlarmExecutor() alarm = Alarm(Alarm.ALARM_TEST, "Alarm framework operational") # check for config files file_path = os.path.join(config_location, "default.cfg") if not os.path.exists(file_path): logging.error( file_path + " does not exist, this file is required for operation") sys.exit() # maybe use something more graceful? local_path = os.path.join(config_location, "local.cfg") if not os.path.exists(local_path): logging.info(local_path + " does not exist, Creating one") os.mknod(local_path) os.chmod(local_path, 0o777) # Parse the config files. printer.config = CascadingConfigParser([ os.path.join(config_location, 'default.cfg'), os.path.join(config_location, 'printer.cfg'), os.path.join(config_location, 'local.cfg') ]) # Check the local and printer files printer_path = os.path.join(config_location, "printer.cfg") if os.path.exists(printer_path): printer.config.check(printer_path) printer.config.check(os.path.join(config_location, 'local.cfg')) # Get the revision and loglevel from the Config file level = self.printer.config.getint('System', 'loglevel') if level > 0: logging.getLogger().setLevel(level) # Set up additional logging, if present: if self.printer.config.getboolean('System', 'log_to_file'): logfile = self.printer.config.get('System', 'logfile') formatter = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s' printer.redeem_logging_handler = logging.handlers.RotatingFileHandler( logfile, maxBytes=2 * 1024 * 1024) printer.redeem_logging_handler.setFormatter( logging.Formatter(formatter)) printer.redeem_logging_handler.setLevel(level) logging.getLogger().addHandler(printer.redeem_logging_handler) logging.info("-- Logfile configured --") # Find out which capes are connected self.printer.config.parse_capes() self.revision = self.printer.config.replicape_revision if self.revision: logging.info("Found Replicape rev. " + self.revision) printer.replicape_key = printer.config.get_key() else: logging.warning("Oh no! No Replicape present!") self.revision = "0B3A" # We set it to 5 axis by default Printer.NUM_AXES = 5 if self.printer.config.reach_revision: logging.info("Found Reach rev. " + self.printer.config.reach_revision) if self.printer.config.reach_revision == "00A0": Printer.NUM_AXES = 8 elif self.printer.config.reach_revision == "00B0": Printer.NUM_AXES = 7 if self.revision in ["00A4", "0A4A", "00A3"]: PWM.set_frequency(100) elif self.revision in ["00B1", "00B2", "00B3", "0B3A"]: PWM.set_frequency(printer.config.getint('Cold-ends', 'pwm_freq')) # Init the Watchdog timer printer.watchdog = Watchdog() # Enable PWM and steppers printer.enable = Enable("P9_41") printer.enable.set_disabled() # Init the Paths printer.axis_config = printer.config.getint('Geometry', 'axis_config') # Init the end stops EndStop.inputdev = self.printer.config.get("Endstops", "inputdev") # Set up key listener Key_pin.listener = Key_pin_listener(EndStop.inputdev) homing_only_endstops = self.printer.config.get('Endstops', 'homing_only_endstops') for es in ["Z2", "Y2", "X2", "Z1", "Y1", "X1"]: # Order matches end stop inversion mask in Firmware pin = self.printer.config.get("Endstops", "pin_" + es) keycode = self.printer.config.getint("Endstops", "keycode_" + es) invert = self.printer.config.getboolean("Endstops", "invert_" + es) self.printer.end_stops[es] = EndStop(printer, pin, keycode, es, invert) self.printer.end_stops[es].stops = self.printer.config.get( 'Endstops', 'end_stop_' + es + '_stops') # activate all the endstops self.printer.set_active_endstops() # Init the 5 Stepper motors (step, dir, fault, DAC channel, name) Stepper.printer = printer if self.revision == "00A3": printer.steppers["X"] = Stepper_00A3("GPIO0_27", "GPIO1_29", "GPIO2_4", 0, "X") printer.steppers["Y"] = Stepper_00A3("GPIO1_12", "GPIO0_22", "GPIO2_5", 1, "Y") printer.steppers["Z"] = Stepper_00A3("GPIO0_23", "GPIO0_26", "GPIO0_15", 2, "Z") printer.steppers["E"] = Stepper_00A3("GPIO1_28", "GPIO1_15", "GPIO2_1", 3, "E") printer.steppers["H"] = Stepper_00A3("GPIO1_13", "GPIO1_14", "GPIO2_3", 4, "H") elif self.revision == "00B1": printer.steppers["X"] = Stepper_00B1("GPIO0_27", "GPIO1_29", "GPIO2_4", 11, 0, "X") printer.steppers["Y"] = Stepper_00B1("GPIO1_12", "GPIO0_22", "GPIO2_5", 12, 1, "Y") printer.steppers["Z"] = Stepper_00B1("GPIO0_23", "GPIO0_26", "GPIO0_15", 13, 2, "Z") printer.steppers["E"] = Stepper_00B1("GPIO1_28", "GPIO1_15", "GPIO2_1", 14, 3, "E") printer.steppers["H"] = Stepper_00B1("GPIO1_13", "GPIO1_14", "GPIO2_3", 15, 4, "H") elif self.revision == "00B2": printer.steppers["X"] = Stepper_00B2("GPIO0_27", "GPIO1_29", "GPIO2_4", 11, 0, "X") printer.steppers["Y"] = Stepper_00B2("GPIO1_12", "GPIO0_22", "GPIO2_5", 12, 1, "Y") printer.steppers["Z"] = Stepper_00B2("GPIO0_23", "GPIO0_26", "GPIO0_15", 13, 2, "Z") printer.steppers["E"] = Stepper_00B2("GPIO1_28", "GPIO1_15", "GPIO2_1", 14, 3, "E") printer.steppers["H"] = Stepper_00B2("GPIO1_13", "GPIO1_14", "GPIO2_3", 15, 4, "H") elif self.revision in ["00B3", "0B3A"]: printer.steppers["X"] = Stepper_00B3("GPIO0_27", "GPIO1_29", 90, 11, 0, "X") printer.steppers["Y"] = Stepper_00B3("GPIO1_12", "GPIO0_22", 91, 12, 1, "Y") printer.steppers["Z"] = Stepper_00B3("GPIO0_23", "GPIO0_26", 92, 13, 2, "Z") printer.steppers["E"] = Stepper_00B3("GPIO1_28", "GPIO1_15", 93, 14, 3, "E") printer.steppers["H"] = Stepper_00B3("GPIO1_13", "GPIO1_14", 94, 15, 4, "H") elif self.revision in ["00A4", "0A4A"]: printer.steppers["X"] = Stepper_00A4("GPIO0_27", "GPIO1_29", "GPIO2_4", 0, 0, "X") printer.steppers["Y"] = Stepper_00A4("GPIO1_12", "GPIO0_22", "GPIO2_5", 1, 1, "Y") printer.steppers["Z"] = Stepper_00A4("GPIO0_23", "GPIO0_26", "GPIO0_15", 2, 2, "Z") printer.steppers["E"] = Stepper_00A4("GPIO1_28", "GPIO1_15", "GPIO2_1", 3, 3, "E") printer.steppers["H"] = Stepper_00A4("GPIO1_13", "GPIO1_14", "GPIO2_3", 4, 4, "H") # Init Reach steppers, if present. if printer.config.reach_revision == "00A0": printer.steppers["A"] = Stepper_reach_00A4("GPIO2_2", "GPIO1_18", "GPIO0_14", 5, 5, "A") printer.steppers["B"] = Stepper_reach_00A4("GPIO1_16", "GPIO0_5", "GPIO0_14", 6, 6, "B") printer.steppers["C"] = Stepper_reach_00A4("GPIO0_3", "GPIO3_19", "GPIO0_14", 7, 7, "C") elif printer.config.reach_revision == "00B0": printer.steppers["A"] = Stepper_reach_00B0("GPIO1_16", "GPIO0_5", "GPIO0_3", 5, 5, "A") printer.steppers["B"] = Stepper_reach_00B0("GPIO2_2", "GPIO0_14", "GPIO0_3", 6, 6, "B") # Enable the steppers and set the current, steps pr mm and # microstepping for name, stepper in self.printer.steppers.iteritems(): stepper.in_use = printer.config.getboolean('Steppers', 'in_use_' + name) stepper.direction = printer.config.getint('Steppers', 'direction_' + name) stepper.has_endstop = printer.config.getboolean( 'Endstops', 'has_' + name) stepper.set_current_value( printer.config.getfloat('Steppers', 'current_' + name)) stepper.set_steps_pr_mm( printer.config.getfloat('Steppers', 'steps_pr_mm_' + name)) stepper.set_microstepping( printer.config.getint('Steppers', 'microstepping_' + name)) stepper.set_decay( printer.config.getint("Steppers", "slow_decay_" + name)) # Add soft end stops printer.soft_min[Printer.axis_to_index( name)] = printer.config.getfloat('Endstops', 'soft_end_stop_min_' + name) printer.soft_max[Printer.axis_to_index( name)] = printer.config.getfloat('Endstops', 'soft_end_stop_max_' + name) slave = printer.config.get('Steppers', 'slave_' + name) if slave: printer.add_slave(name, slave) logging.debug("Axis " + name + " has slave " + slave) # Commit changes for the Steppers #Stepper.commit() Stepper.printer = printer # Delta printer setup if printer.axis_config == Printer.AXIS_CONFIG_DELTA: opts = [ "Hez", "L", "r", "Ae", "Be", "Ce", "A_radial", "B_radial", "C_radial", "A_tangential", "B_tangential", "C_tangential" ] for opt in opts: Delta.__dict__[opt] = printer.config.getfloat('Delta', opt) # Discover and add all DS18B20 cold ends. paths = glob.glob("/sys/bus/w1/devices/28-*/w1_slave") logging.debug("Found cold ends: " + str(paths)) for i, path in enumerate(paths): self.printer.cold_ends.append(ColdEnd(path, "ds18b20-" + str(i))) logging.info("Found Cold end " + str(i) + " on " + path) # Make Mosfets, temperature sensors and extruders heaters = ["E", "H", "HBP"] if self.printer.config.reach_revision: heaters.extend(["A", "B", "C"]) for e in heaters: # Mosfets channel = self.printer.config.getint("Heaters", "mosfet_" + e) self.printer.mosfets[e] = Mosfet(channel) # Thermistors adc = self.printer.config.get("Heaters", "path_adc_" + e) if not self.printer.config.has_option("Heaters", "sensor_" + e): sensor = self.printer.config.get("Heaters", "temp_chart_" + e) logging.warning("Deprecated config option temp_chart_" + e + " use sensor_" + e + " instead.") else: sensor = self.printer.config.get("Heaters", "sensor_" + e) self.printer.thermistors[e] = TemperatureSensor( adc, 'MOSFET ' + e, sensor) self.printer.thermistors[e].printer = printer # Extruders onoff = self.printer.config.getboolean('Heaters', 'onoff_' + e) prefix = self.printer.config.get('Heaters', 'prefix_' + e) max_power = self.printer.config.getfloat('Heaters', 'max_power_' + e) if e != "HBP": self.printer.heaters[e] = Extruder(self.printer.steppers[e], self.printer.thermistors[e], self.printer.mosfets[e], e, onoff) else: self.printer.heaters[e] = HBP(self.printer.thermistors[e], self.printer.mosfets[e], onoff) self.printer.heaters[e].prefix = prefix self.printer.heaters[e].Kp = self.printer.config.getfloat( 'Heaters', 'pid_Kp_' + e) self.printer.heaters[e].Ti = self.printer.config.getfloat( 'Heaters', 'pid_Ti_' + e) self.printer.heaters[e].Td = self.printer.config.getfloat( 'Heaters', 'pid_Td_' + e) # Min/max settings self.printer.heaters[e].min_temp = self.printer.config.getfloat( 'Heaters', 'min_temp_' + e) self.printer.heaters[e].max_temp = self.printer.config.getfloat( 'Heaters', 'max_temp_' + e) self.printer.heaters[ e].max_temp_rise = self.printer.config.getfloat( 'Heaters', 'max_rise_temp_' + e) self.printer.heaters[ e].max_temp_fall = self.printer.config.getfloat( 'Heaters', 'max_fall_temp_' + e) # Init the three fans. Argument is PWM channel number self.printer.fans = [] if self.revision == "00A3": self.printer.fans.append(Fan(0)) self.printer.fans.append(Fan(1)) self.printer.fans.append(Fan(2)) elif self.revision == "0A4A": self.printer.fans.append(Fan(8)) self.printer.fans.append(Fan(9)) self.printer.fans.append(Fan(10)) elif self.revision in ["00B1", "00B2", "00B3", "0B3A"]: self.printer.fans.append(Fan(7)) self.printer.fans.append(Fan(8)) self.printer.fans.append(Fan(9)) self.printer.fans.append(Fan(10)) if printer.config.reach_revision == "00A0": self.printer.fans.append(Fan(14)) self.printer.fans.append(Fan(15)) self.printer.fans.append(Fan(7)) # Set default value for all fans for i, f in enumerate(self.printer.fans): f.set_value( self.printer.config.getfloat('Fans', "default-fan-{}-value".format(i))) # Init the servos printer.servos = [] servo_nr = 0 while (printer.config.has_option("Servos", "servo_" + str(servo_nr) + "_enable")): if printer.config.getboolean("Servos", "servo_" + str(servo_nr) + "_enable"): channel = printer.config.get( "Servos", "servo_" + str(servo_nr) + "_channel") pulse_min = printer.config.getfloat( "Servos", "servo_" + str(servo_nr) + "_pulse_min") pulse_max = printer.config.getfloat( "Servos", "servo_" + str(servo_nr) + "_pulse_max") angle_min = printer.config.getfloat( "Servos", "servo_" + str(servo_nr) + "_angle_min") angle_max = printer.config.getfloat( "Servos", "servo_" + str(servo_nr) + "_angle_max") angle_init = printer.config.getfloat( "Servos", "servo_" + str(servo_nr) + "_angle_init") s = Servo(channel, pulse_min, pulse_max, angle_min, angle_max, angle_init) printer.servos.append(s) logging.info("Added servo " + str(servo_nr)) servo_nr += 1 # Connect thermitors to fans for t, therm in self.printer.heaters.iteritems(): for f, fan in enumerate(self.printer.fans): if not self.printer.config.has_option( 'Cold-ends', "connect-therm-{}-fan-{}".format(t, f)): continue if printer.config.getboolean( 'Cold-ends', "connect-therm-{}-fan-{}".format(t, f)): c = Cooler(therm, fan, "Cooler-{}-{}".format(t, f), True) # Use ON/OFF on these. c.ok_range = 4 opt_temp = "therm-{}-fan-{}-target_temp".format(t, f) if printer.config.has_option('Cold-ends', opt_temp): target_temp = printer.config.getfloat( 'Cold-ends', opt_temp) else: target_temp = 60 c.set_target_temperature(target_temp) c.enable() printer.coolers.append(c) logging.info("Cooler connects therm {} with fan {}".format( t, f)) # Connect fans to M106 printer.controlled_fans = [] for i, fan in enumerate(self.printer.fans): if not self.printer.config.has_option( 'Cold-ends', "add-fan-{}-to-M106".format(i)): continue if self.printer.config.getboolean('Cold-ends', "add-fan-{}-to-M106".format(i)): printer.controlled_fans.append(self.printer.fans[i]) logging.info("Added fan {} to M106/M107".format(i)) # Connect the colds to fans for ce, cold_end in enumerate(self.printer.cold_ends): for f, fan in enumerate(self.printer.fans): option = "connect-ds18b20-{}-fan-{}".format(ce, f) if self.printer.config.has_option('Cold-ends', option): if self.printer.config.getboolean('Cold-ends', option): c = Cooler(cold_end, fan, "Cooler-ds18b20-{}-{}".format(ce, f), False) c.ok_range = 4 opt_temp = "cooler_{}_target_temp".format(ce) if printer.config.has_option('Cold-ends', opt_temp): target_temp = printer.config.getfloat( 'Cold-ends', opt_temp) else: target_temp = 60 c.set_target_temperature(target_temp) c.enable() printer.coolers.append(c) logging.info( "Cooler connects temp sensor ds18b20 {} with fan {}" .format(ce, f)) # Init roatray encs. printer.filament_sensors = [] # Init rotary encoders printer.rotary_encoders = [] for ex in ["E", "H", "A", "B", "C"]: if not printer.config.has_option('Rotary-encoders', "enable-{}".format(ex)): continue if printer.config.getboolean("Rotary-encoders", "enable-{}".format(ex)): logging.debug("Rotary encoder {} enabled".format(ex)) event = printer.config.get("Rotary-encoders", "event-{}".format(ex)) cpr = printer.config.getint("Rotary-encoders", "cpr-{}".format(ex)) diameter = printer.config.getfloat("Rotary-encoders", "diameter-{}".format(ex)) r = RotaryEncoder(event, cpr, diameter) printer.rotary_encoders.append(r) # Append as Filament Sensor ext_nr = Printer.axis_to_index(ex) - 3 sensor = FilamentSensor(ex, r, ext_nr, printer) alarm_level = printer.config.getfloat( "Filament-sensors", "alarm-level-{}".format(ex)) logging.debug("Alarm level" + str(alarm_level)) sensor.alarm_level = alarm_level printer.filament_sensors.append(sensor) # Make a queue of commands self.printer.commands = JoinableQueue(10) # Make a queue of commands that should not be buffered self.printer.sync_commands = JoinableQueue() self.printer.unbuffered_commands = JoinableQueue(10) # Bed compensation matrix printer.matrix_bed_comp = printer.load_bed_compensation_matrix() logging.debug("Loaded bed compensation matrix: \n" + str(printer.matrix_bed_comp)) for axis in printer.steppers.keys(): i = Printer.axis_to_index(axis) printer.max_speeds[i] = printer.config.getfloat( 'Planner', 'max_speed_' + axis.lower()) printer.min_speeds[i] = printer.config.getfloat( 'Planner', 'min_speed_' + axis.lower()) printer.jerks[i] = printer.config.getfloat( 'Planner', 'max_jerk_' + axis.lower()) printer.home_speed[i] = printer.config.getfloat( 'Homing', 'home_speed_' + axis.lower()) printer.home_backoff_speed[i] = printer.config.getfloat( 'Homing', 'home_backoff_speed_' + axis.lower()) printer.home_backoff_offset[i] = printer.config.getfloat( 'Homing', 'home_backoff_offset_' + axis.lower()) printer.steps_pr_meter[i] = printer.steppers[ axis].get_steps_pr_meter() printer.backlash_compensation[i] = printer.config.getfloat( 'Steppers', 'backlash_' + axis.lower()) printer.e_axis_active = printer.config.getboolean( 'Planner', 'e_axis_active') dirname = os.path.dirname(os.path.realpath(__file__)) # Create the firmware compiler pru_firmware = PruFirmware(dirname + "/firmware/firmware_runtime.p", dirname + "/firmware/firmware_runtime.bin", dirname + "/firmware/firmware_endstops.p", dirname + "/firmware/firmware_endstops.bin", self.printer, "/usr/bin/pasm") printer.move_cache_size = printer.config.getfloat( 'Planner', 'move_cache_size') printer.print_move_buffer_wait = printer.config.getfloat( 'Planner', 'print_move_buffer_wait') printer.min_buffered_move_time = printer.config.getfloat( 'Planner', 'min_buffered_move_time') printer.max_buffered_move_time = printer.config.getfloat( 'Planner', 'max_buffered_move_time') printer.max_length = printer.config.getfloat('Planner', 'max_length') self.printer.processor = GCodeProcessor(self.printer) self.printer.plugins = PluginsController(self.printer) # Path planner travel_default = False center_default = False home_default = False # Setting acceleration before PathPlanner init for axis in printer.steppers.keys(): printer.acceleration[Printer.axis_to_index( axis)] = printer.config.getfloat( 'Planner', 'acceleration_' + axis.lower()) self.printer.path_planner = PathPlanner(self.printer, pru_firmware) for axis in printer.steppers.keys(): i = Printer.axis_to_index(axis) # Sometimes soft_end_stop aren't defined to be at the exact hardware boundary. # Adding 100mm for searching buffer. if printer.config.has_option('Geometry', 'travel_' + axis.lower()): printer.path_planner.travel_length[ axis] = printer.config.getfloat('Geometry', 'travel_' + axis.lower()) else: printer.path_planner.travel_length[axis] = ( printer.soft_max[i] - printer.soft_min[i]) + .1 if axis in ['X', 'Y', 'Z']: travel_default = True if printer.config.has_option('Geometry', 'offset_' + axis.lower()): printer.path_planner.center_offset[ axis] = printer.config.getfloat('Geometry', 'offset_' + axis.lower()) else: printer.path_planner.center_offset[axis] = ( printer.soft_min[i] if printer.home_speed[i] > 0 else printer.soft_max[i]) if axis in ['X', 'Y', 'Z']: center_default = True if printer.config.has_option('Homing', 'home_' + axis.lower()): printer.path_planner.home_pos[axis] = printer.config.getfloat( 'Homing', 'home_' + axis.lower()) else: printer.path_planner.home_pos[ axis] = printer.path_planner.center_offset[axis] if axis in ['X', 'Y', 'Z']: home_default = True if printer.axis_config == Printer.AXIS_CONFIG_DELTA: if travel_default: logging.warning( "Axis travel (travel_*) set by soft limits, manual setup is recommended for a delta" ) if center_default: logging.warning( "Axis offsets (offset_*) set by soft limits, manual setup is recommended for a delta" ) if home_default: logging.warning( "Home position (home_*) set by soft limits or offset_*") logging.info("Home position will be recalculated...") # convert home_pos to effector space Az = printer.path_planner.home_pos['X'] Bz = printer.path_planner.home_pos['Y'] Cz = printer.path_planner.home_pos['Z'] delta_bot = self.printer.path_planner.native_planner.delta_bot z_offset = delta_bot.vertical_offset(Az, Bz, Cz) # vertical offset xyz = delta_bot.forward_kinematics(Az, Bz, Cz) # effector position # The default home_pos, provided above, is based on effector space # coordinates for carriage positions. We need to transform these to # get where the effector actually is. xyz[2] += z_offset for i, a in enumerate(['X', 'Y', 'Z']): printer.path_planner.home_pos[a] = xyz[i] logging.info("Home position = %s" % str(printer.path_planner.home_pos)) # Read end stop value again now that PRU is running for _, es in self.printer.end_stops.iteritems(): es.read_value() # Enable Stepper timeout timeout = printer.config.getint('Steppers', 'timeout_seconds') printer.swd = StepperWatchdog(printer, timeout) if printer.config.getboolean('Steppers', 'use_timeout'): printer.swd.start() # Set up communication channels printer.comms["USB"] = USB(self.printer) printer.comms["Eth"] = Ethernet(self.printer) if Pipe.check_tty0tty() or Pipe.check_socat(): printer.comms["octoprint"] = Pipe(printer, "octoprint") printer.comms["toggle"] = Pipe(printer, "toggle") printer.comms["testing"] = Pipe(printer, "testing") printer.comms["testing_noret"] = Pipe(printer, "testing_noret") # Does not send "ok" printer.comms["testing_noret"].send_response = False else: logging.warning( "Neither tty0tty or socat is installed! No virtual tty pipes enabled" )
from multiprocessing import JoinableQueue q = JoinableQueue() ''' q.join()# 用于生产者。等待 q.task_done的返回结果,通过返回结果,生产者就能获得消费者当前消费了多少个数据 q.task_done() # 用于消费者,是指每消费队列中一个数据,就给join返回一个标识。 详细解释: 假设生产者生产了100个数据,join就能记录下100这个数字。每次消费者消费一个数据, 就必须要task_done返回一个标识,当生产者(join)接收到100个消费者返回来的标识的时候, 生产者就能知道消费者已经把所有数据都消费完了。 ''' from multiprocessing import Process def consumer(q, name, color): while 1: info = q.get() print('%s %s 拿走了%s \033[0m' % (color, name, info)) q.task_done() def producer(q, product): for i in range(20): info = product + '的娃娃%s号' % str(i) q.put(info) q.join() # 记录了生产了20个数据在队列中,此时会阻塞等待消费者消费完队列中所有数据 if __name__ == '__main__': q = JoinableQueue(10)
class Toggle: def __init__(self): Clutter.init(None) ui = Clutter.Script() ui.load_from_file("ui-threads.json") stage = ui.get_object("stage") stage.connect("destroy", self.stop) stage.connect("destroy", lambda w: Clutter.main_quit()) self.loader = ui.get_object("loader") self.loader.set_from_file("style/loading.png") self.t = Clutter.PropertyTransition(property_name='rotation-angle-z') self.t.set_from(0) self.t.set_to(360) self.t.set_duration(3000) self.t.set_animatable(self.loader) self.t.set_repeat_count(-1) self.t.start() button1 = ui.get_object("button1") button1.connect("button-press-event", self.execute_in_main) button1.set_reactive(True) button2 = ui.get_object("button2") button2.connect("button-press-event", self.idle_add_event) button2.set_reactive(True) button3 = ui.get_object("button3") button3.connect("button-press-event", self.threads_idle_add_event) button3.set_reactive(True) button4 = ui.get_object("button4") button4.connect("button-press-event", self.execute_in_thread) button4.set_reactive(True) stage = ui.get_object("stage") stage.set_title("Test threads") stage.connect("destroy", self.stop) stage.show_all() self.events = JoinableQueue(10) # UI events needs to happen from within the # main thread. This was the only way I found that would do that. # It looks weirdm, but it works. def execute(event): print("Execute " + event + " from " + str(current_thread())) for i in range(100): hashlib.md5(str(list(range(100000)))) print("Done executing") self.execute = execute stage.show() def execute_in_main(self, btn, other): self.execute("main") def idle_add_event(self, btn, other): self.events.put("glib_idle_add") def threads_idle_add_event(self, btn, other): self.events.put("threads_add_idle") def execute_in_thread(self, btn, other): self.events.put("execute_in_thread") def run(self): """ Start the process """ self.running = True # Start the processes self.p0 = Thread(target=self.loop, args=(self.events, "Push updates")) self.p0.start() logging.info("Toggle ready") Clutter.main() def loop(self, queue, name): """ When a new event comes in, execute it """ try: while self.running: try: event = queue.get(block=True, timeout=1) except queue.Empty: continue if event == "glib_idle_add": print("adding with Glib") GLib.idle_add(self.execute, event) elif event == "threads_add_idle": print("adding with Clutter") Clutter.threads_add_idle(0, self.execute, event) elif event == "execute_in_thread": print("Executing from thread") self.execute(event) # Must hand it over to the main thread. queue.task_done() except Exception: logging.exception("Exception in {} loop: ".format(name)) def stop(self, w): logging.debug("Stopping Toggle") self.running = False self.p0.join() Clutter.main_quit() logging.debug("Done")
class Servo: if PY2: range = xrange def __init__(self, channel, pulse_width_min, pulse_width_max, angle_min, angle_max, init_angle, turnoff_timeout=0): """Define a new software controllable servo with adjustable speed control Keyword arguments: pulse_width_min -- The minimum pulse width defining the lowest angle pulse_width_max -- The maximum pulse width defining the biggest angle init_angle -- Initial angle that the servo should take when it is powered on. Range is 0 to 180deg turnoff_timeout -- number of seconds after which the servo is turned off if no command is received. 0 = never turns off """ self.angle_min = angle_min self.angle_max = angle_max self.angle_range = angle_max - angle_min self.pulse_width_min = pulse_width_min self.pulse_width_max = pulse_width_max self.pulse_width_range = pulse_width_max - pulse_width_min self.turnoff_timeout = turnoff_timeout self.current_pulse_width = self.angle_to_pulse_width(init_angle) self.last_pulse_width = self.current_pulse_width self.last_angle = init_angle self.pulse_length = 20.0 * 10.0**-3.0 # 20 ms logging.debug("Angle min: {} deg".format(self.angle_min)) logging.debug("Angle max: {} deg".format(self.angle_max)) logging.debug("Angle tot: {} deg".format(self.angle_range)) logging.debug("Pulse min: {} ms".format(self.pulse_width_min * 1000.0)) logging.debug("Pulse max: {} ms".format(self.pulse_width_max * 1000.0)) logging.debug("Pulse tot: {} ms".format(self.pulse_width_range * 1000.0)) self.queue = JoinableQueue(1000) self.lastCommandTime = 0 self.t = Thread(target=self._wait_for_event, name="Servo") self.t.daemon = True self.running = True self.t.start() # Branch based on channel type. if type(channel) == int: # Revision A self.pwm = PWM(channel, 50, self.current_pulse_width) else: # Revision B # Set up the Shift register for enabling this servo if channel == "P9_14": shiftreg_nr = 1 self.pwm = PWM_pin(channel, 50, self.current_pulse_width) elif channel == "P9_16": shiftreg_nr = 2 self.pwm = PWM_pin(channel, 50, self.current_pulse_width) else: logging.warning( "Tried to assign servo to an unknown channel/pin: " + str(channel)) return ShiftRegister.make(5) self.shift_reg = ShiftRegister.registers[shiftreg_nr] self.set_enabled() self.pwm.set_value( self.angle_to_pulse_width(init_angle) / self.pulse_length) def set_enabled(self, is_enabled=True): if is_enabled: self.shift_reg.add_state(0x01) else: self.shift_reg.remove_state(0x01) def set_angle(self, angle, speed=60, asynchronous=True): ''' Set the servo angle to the given value, in degree, with the given speed in deg / sec ''' angle = max(min(self.angle_max, angle), self.angle_min) pulse_width = self.angle_to_pulse_width(angle) last_angle = self.last_angle logging.debug("Updating angle from {} (pw={}) to {} (pw={}) ".format( last_angle, self.last_pulse_width, angle, pulse_width)) if angle == last_angle: return t = (math.fabs(angle - last_angle) / speed) / math.fabs(angle - last_angle) if angle >= last_angle: increment = 1 else: increment = -1 for a in range(int(last_angle + increment), int(angle), increment): self.queue.put((self.angle_to_pulse_width(a), t)) self.queue.put((self.angle_to_pulse_width(angle), t)) self.last_pulse_width = pulse_width self.last_angle = angle if not asynchronous: self.queue.join() def turn_off(self): self.pwm.set_value(0) def stop(self): self.running = False self.t.join() self.turn_off() def _wait_for_event(self): while self.running: try: ev = self.queue.get(block=True, timeout=1) except queue.Empty: if self.turnoff_timeout > 0 and self.lastCommandTime > 0 and time.time( ) - self.lastCommandTime > self.turnoff_timeout: self.lastCommandTime = 0 self.turn_off() continue except Exception: # To avoid exception printed on output pass self.current_pulse_width = ev[0] #logging.debug("setting pulse width to "+str(self.current_pulse_width)) self.pwm.set_value(self.current_pulse_width / self.pulse_length) self.lastCommandTime = time.time() time.sleep(ev[1]) self.queue.task_done() def angle_to_pulse_width(self, angle): return ( (angle - self.angle_min) / self.angle_range) * self.pulse_width_range + self.pulse_width_min def pulse_width_to_angle(self, pulse_width): return (((pulse_width - self.pulse_width_min) / (self.pulse_width_range)) * self.angle_range) + self.angle_min
def __init__(self, channel, pulse_width_min, pulse_width_max, angle_min, angle_max, init_angle, turnoff_timeout=0): """Define a new software controllable servo with adjustable speed control Keyword arguments: pulse_width_min -- The minimum pulse width defining the lowest angle pulse_width_max -- The maximum pulse width defining the biggest angle init_angle -- Initial angle that the servo should take when it is powered on. Range is 0 to 180deg turnoff_timeout -- number of seconds after which the servo is turned off if no command is received. 0 = never turns off """ self.angle_min = angle_min self.angle_max = angle_max self.angle_range = angle_max - angle_min self.pulse_width_min = pulse_width_min self.pulse_width_max = pulse_width_max self.pulse_width_range = pulse_width_max - pulse_width_min self.turnoff_timeout = turnoff_timeout self.current_pulse_width = self.angle_to_pulse_width(init_angle) self.last_pulse_width = self.current_pulse_width self.last_angle = init_angle self.pulse_length = 20.0 * 10.0**-3.0 # 20 ms logging.debug("Angle min: {} deg".format(self.angle_min)) logging.debug("Angle max: {} deg".format(self.angle_max)) logging.debug("Angle tot: {} deg".format(self.angle_range)) logging.debug("Pulse min: {} ms".format(self.pulse_width_min * 1000.0)) logging.debug("Pulse max: {} ms".format(self.pulse_width_max * 1000.0)) logging.debug("Pulse tot: {} ms".format(self.pulse_width_range * 1000.0)) self.queue = JoinableQueue(1000) self.lastCommandTime = 0 self.t = Thread(target=self._wait_for_event, name="Servo") self.t.daemon = True self.running = True self.t.start() # Branch based on channel type. if type(channel) == int: # Revision A self.pwm = PWM(channel, 50, self.current_pulse_width) else: # Revision B # Set up the Shift register for enabling this servo if channel == "P9_14": shiftreg_nr = 1 self.pwm = PWM_pin(channel, 50, self.current_pulse_width) elif channel == "P9_16": shiftreg_nr = 2 self.pwm = PWM_pin(channel, 50, self.current_pulse_width) else: logging.warning( "Tried to assign servo to an unknown channel/pin: " + str(channel)) return ShiftRegister.make(5) self.shift_reg = ShiftRegister.registers[shiftreg_nr] self.set_enabled() self.pwm.set_value( self.angle_to_pulse_width(init_angle) / self.pulse_length)
class TestDatasetSwmrWriteRead(TestCase): """ Testing SWMR functions when reading a dataset. Skip this test if the HDF5 library does not have the SWMR features. """ def setUp(self): """ First setup a file with a small chunked and empty dataset. No data written yet. """ # Note that when creating the file, the swmr=True is not required for # write, but libver='latest' is required. self.fname = self.mktemp() self.writer_queue = JoinableQueue() self.writer_process = Process(target=writer_loop, args=(self.writer_queue, ), daemon=True) self.writer_process.start() def test_create_open_read_update_file(self): """ Update and read dataset and an attribute in group with SWMR mode """ self.data = np.arange(13).astype('f') self.new_data = np.arange(13).astype('f') + 2 writer_queue = self.writer_queue parameters = {'fname': self.fname} writer_queue.put({'action': 'create_file', 'parameters': parameters}) writer_queue.join() parameters = {'name': 'data', 'value': self.data} writer_queue.put({ 'action': 'create_dataset', 'parameters': parameters }) writer_queue.join() # create attributes to test attributes = [ { 'name': 'attr_bool', 'value': False, 'new_value': True }, { 'name': 'attr_int', 'value': 1, 'new_value': 2 }, { 'name': 'attr_float', 'value': 1.4, 'new_value': 3.2 }, { 'name': 'attr_string', 'value': 'test', 'new_value': 'essai' }, ] for attribute in attributes: attribute_name = attribute['name'] attribute_value = attribute['value'] parameters = {'name': attribute_name, 'value': attribute_value} writer_queue.put({ 'action': 'create_attribute', 'parameters': parameters }) writer_queue.join() # try opening the file in swmr file = None with self.assertRaises(OSError): file = h5py.File(self.fname, 'r', libver='latest', swmr=True) writer_queue.put({'action': 'set_swmr_mode'}) writer_queue.join() # open file and check group file = h5py.File(self.fname, 'r', libver='latest', swmr=True) self.assertIn('group', file) # check attributes group = file['group'] for attribute in attributes: attribute_name = attribute['name'] attribute_value = attribute['value'] attribute_new_value = attribute['new_value'] self.assertIn(attribute_name, group.attrs) read_value = group.attrs[attribute_name] if isinstance(attribute_value, str): read_value = read_value.decode() self.assertEqual(read_value, attribute_value) parameters = {'name': attribute_name, 'value': attribute_new_value} writer_queue.put({ 'action': 'update_attribute', 'parameters': parameters }) writer_queue.join() read_value = group.attrs[attribute_name] if isinstance(attribute_value, str): read_value = read_value.decode() self.assertEqual(read_value, attribute_value) writer_queue.put({'action': 'flush_group'}) writer_queue.join() # check that read group attribute has not changed read_value = group.attrs[attribute_name] if isinstance(attribute_value, str): read_value = read_value.decode() self.assertEqual(read_value, attribute_value) group.refresh() # check that read group attribute has changed read_value = group.attrs[attribute_name] if isinstance(attribute_value, str): read_value = read_value.decode() self.assertEqual(read_value, attribute_new_value) # check that dataset has been recorder data = group['data'] self.assertArrayEqual(data[:], self.data) # update data parameters = {'name': 'data', 'value': self.new_data} writer_queue.put({ 'action': 'update_dataset', 'parameters': parameters }) writer_queue.join() # check that data has not been updated data = group['data'] self.assertArrayEqual(data[:], self.data) # flush group writer_queue.put({'action': 'flush_group'}) writer_queue.join() # check that data has not been updated data = group['data'] self.assertArrayEqual(data[:], self.data) # refresh group, this won't update dataset group.refresh() # check that data has not been updated data = group['data'] self.assertArrayEqual(data[:], self.data) # refresh dataset, this will update data data.refresh() # check that data has been updated self.assertArrayEqual(data[:], self.new_data) writer_queue.put({'action': 'close_file'}) writer_queue.join() file.close() pass def tearDown(self): self.writer_queue.put({'action': "stop"}) self.writer_queue.join() self.writer_process.join()
foods = q.get() print('%s消费了%s' % (name, foods)) q.task_done() def producer(name, food, q): for i in range(4): time.sleep(random.randint(0, 2)) foods = '%s生产了%s%s' % (name, food, i) print(foods) q.put(foods) q.join() if __name__ == '__main__': q = JoinableQueue(20) p1 = Process(target=producer, args=('egon', '包子', q)) p1.start() p2 = Process(target=producer, args=('jason', '包子', q)) p2.start() c = Process(target=consunmer, args=('alise', q)) c.daemon = True c.start() c2 = Process(target=consunmer, args=('hanke', q)) c2.daemon = True c2.start() p1.join()
def main(_): parser = argparse.ArgumentParser(description='ProjE.') parser.add_argument('--data', dest='data_dir', type=str, help="Data folder", default='./data/FB15k/') parser.add_argument('--lr', dest='lr', type=float, help="Learning rate", default=0.01) parser.add_argument("--dim", dest='dim', type=int, help="Embedding dimension", default=200) parser.add_argument("--batch", dest='batch', type=int, help="Batch size", default=100) parser.add_argument("--comb", dest="combination_method", type=str, help="Combination method", default='simple') parser.add_argument("--worker", dest='n_worker', type=int, help="Evaluation worker", default=3) parser.add_argument("--generator", dest='n_generator', type=int, help="Data generator", default=10) parser.add_argument("--eval_batch", dest="eval_batch", type=int, help="Evaluation batch size", default=500) parser.add_argument("--save_dir", dest='save_dir', type=str, help="Model path", default='./') parser.add_argument("--load_model", dest='load_model', type=str, help="Model file", default="") parser.add_argument("--save_per", dest='save_per', type=int, help="Save per x iteration", default=10) parser.add_argument("--eval_per", dest='eval_per', type=int, help="Evaluate every x iteration", default=1) parser.add_argument("--max_iter", dest='max_iter', type=int, help="Max iteration", default=100) parser.add_argument("--summary_dir", dest='summary_dir', type=str, help="summary directory", default='./ProjE_summary/') parser.add_argument("--keep", dest='drop_out', type=float, help="Keep prob (1.0 keep all, 0. drop all)", default=0.5) parser.add_argument("--optimizer", dest='optimizer', type=str, help="Optimizer", default='adam') parser.add_argument("--prefix", dest='prefix', type=str, help="model_prefix", default='DEFAULT') parser.add_argument("--loss_weight", dest='loss_weight', type=float, help="Weight on parameter loss", default=1e-5) parser.add_argument("--neg_weight", dest='neg_weight', type=float, help="Sampling weight on negative examples", default=0.5) args = parser.parse_args() print(args) model = ProjE(args.data_dir, embed_dim=args.dim, combination_method=args.combination_method, dropout=args.drop_out, neg_weight=args.neg_weight) train_hrt_input, train_hrt_weight, train_trh_input, train_trh_weight, \ train_loss, train_op = train_ops(model, learning_rate=args.lr, optimizer_str=args.optimizer, regularizer_weight=args.loss_weight) test_input, test_head, test_tail = test_ops(model) with tf.Session() as session: tf.initialize_all_variables().run() saver = tf.train.Saver() iter_offset = 0 if args.load_model is not None and os.path.exists(args.load_model): saver.restore(session, args.load_model) iter_offset = int( args.load_model.split('.')[-2].split('_')[-1]) + 1 print("Load model from %s, iteration %d restored." % (args.load_model, iter_offset)) total_inst = model.n_train # training data generator raw_training_data_queue = Queue() training_data_queue = Queue() data_generators = list() for i in range(args.n_generator): data_generators.append( Process(target=data_generator_func, args=(raw_training_data_queue, training_data_queue, model.tr_h, model.hr_t, model.n_entity, args.neg_weight))) data_generators[-1].start() evaluation_queue = JoinableQueue() result_queue = Queue() for i in range(args.n_worker): worker = Process(target=worker_func, args=(evaluation_queue, result_queue, model.hr_t, model.tr_h)) worker.start() for data_func, test_type in zip( [model.validation_data, model.testing_data], ['VALID', 'TEST']): accu_mean_rank_h = list() accu_mean_rank_t = list() accu_filtered_mean_rank_h = list() accu_filtered_mean_rank_t = list() evaluation_count = 0 for testing_data in data_func(batch_size=args.eval_batch): head_pred, tail_pred = session.run([test_head, test_tail], {test_input: testing_data}) evaluation_queue.put((testing_data, head_pred, tail_pred)) evaluation_count += 1 for i in range(args.n_worker): evaluation_queue.put(None) print("waiting for worker finishes their work") evaluation_queue.join() print("all worker stopped.") while evaluation_count > 0: evaluation_count -= 1 (mrh, fmrh), (mrt, fmrt) = result_queue.get() accu_mean_rank_h += mrh accu_mean_rank_t += mrt accu_filtered_mean_rank_h += fmrh accu_filtered_mean_rank_t += fmrt print( "[%s] INITIALIZATION [HEAD PREDICTION] MEAN RANK: %.1f FILTERED MEAN RANK %.1f HIT@10 %.3f FILTERED HIT@10 %.3f" % (test_type, np.mean(accu_mean_rank_h), np.mean(accu_filtered_mean_rank_h), np.mean(np.asarray(accu_mean_rank_h, dtype=np.int32) < 10), np.mean( np.asarray(accu_filtered_mean_rank_h, dtype=np.int32) < 10 ))) print( "[%s] INITIALIZATION [TAIL PREDICTION] MEAN RANK: %.1f FILTERED MEAN RANK %.1f HIT@10 %.3f FILTERED HIT@10 %.3f" % (test_type, np.mean(accu_mean_rank_t), np.mean(accu_filtered_mean_rank_t), np.mean(np.asarray(accu_mean_rank_t, dtype=np.int32) < 10), np.mean( np.asarray(accu_filtered_mean_rank_t, dtype=np.int32) < 10 ))) for n_iter in range(iter_offset, args.max_iter): start_time = timeit.default_timer() accu_loss = 0. accu_re_loss = 0. ninst = 0 print("initializing raw training data...") nbatches_count = 0 for dat in model.raw_training_data(batch_size=args.batch): raw_training_data_queue.put(dat) nbatches_count += 1 print("raw training data initialized.") while nbatches_count > 0: nbatches_count -= 1 hr_tlist, hr_tweight, tr_hlist, tr_hweight = training_data_queue.get( ) l, rl, _ = session.run( [train_loss, model.regularizer_loss, train_op], { train_hrt_input: hr_tlist, train_hrt_weight: hr_tweight, train_trh_input: tr_hlist, train_trh_weight: tr_hweight }) accu_loss += l accu_re_loss += rl ninst += len(hr_tlist) + len(tr_hlist) if ninst % (5000) is not None: print( '[%d sec](%d/%d) : %.2f -- loss : %.5f rloss: %.5f ' % (timeit.default_timer() - start_time, ninst, total_inst, float(ninst) / total_inst, l / (len(hr_tlist) + len(tr_hlist)), args.loss_weight * (rl / (len(hr_tlist) + len(tr_hlist)))), end='\r') print("") print("iter %d avg loss %.5f, time %.3f" % (n_iter, accu_loss / ninst, timeit.default_timer() - start_time)) if n_iter % args.save_per == 0 or n_iter == args.max_iter - 1: save_path = saver.save( session, os.path.join( args.save_dir, "ProjE_" + str(args.prefix) + "_" + str(n_iter) + ".ckpt")) print("Model saved at %s" % save_path) if n_iter % args.eval_per == 0 or n_iter == args.max_iter - 1: for data_func, test_type in zip( [model.validation_data, model.testing_data], ['VALID', 'TEST']): accu_mean_rank_h = list() accu_mean_rank_t = list() accu_filtered_mean_rank_h = list() accu_filtered_mean_rank_t = list() evaluation_count = 0 for testing_data in data_func(batch_size=args.eval_batch): head_pred, tail_pred = session.run( [test_head, test_tail], {test_input: testing_data}) evaluation_queue.put( (testing_data, head_pred, tail_pred)) evaluation_count += 1 for i in range(args.n_worker): evaluation_queue.put(None) print("waiting for worker finishes their work") evaluation_queue.join() print("all worker stopped.") while evaluation_count > 0: evaluation_count -= 1 (mrh, fmrh), (mrt, fmrt) = result_queue.get() accu_mean_rank_h += mrh accu_mean_rank_t += mrt accu_filtered_mean_rank_h += fmrh accu_filtered_mean_rank_t += fmrt print( "[%s] ITER %d [HEAD PREDICTION] MEAN RANK: %.1f FILTERED MEAN RANK %.1f HIT@10 %.3f FILTERED HIT@10 %.3f" % (test_type, n_iter, np.mean(accu_mean_rank_h), np.mean(accu_filtered_mean_rank_h), np.mean( np.asarray(accu_mean_rank_h, dtype=np.int32) < 10 ), np.mean( np.asarray(accu_filtered_mean_rank_h, dtype=np.int32) < 10))) print( "[%s] ITER %d [TAIL PREDICTION] MEAN RANK: %.1f FILTERED MEAN RANK %.1f HIT@10 %.3f FILTERED HIT@10 %.3f" % (test_type, n_iter, np.mean(accu_mean_rank_t), np.mean(accu_filtered_mean_rank_t), np.mean( np.asarray(accu_mean_rank_t, dtype=np.int32) < 10 ), np.mean( np.asarray(accu_filtered_mean_rank_t, dtype=np.int32) < 10)))
exit = close if __name__ == '__main__': import signal from multiprocessing import JoinableQueue from ...Machine import Machine from ...ConfigParser import ConfigParser from ..OscProcessor import OscProcessor logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(name)-12s \ %(levelname)-8s %(message)s', datefmt='%Y/%m/%d %H:%M:%S') m = Machine() c = ConfigParser('../conf/default.conf') m.config = c m.commands = JoinableQueue(10) m.unbuffered_commands = JoinableQueue(10) m.synced_commands = JoinableQueue() m.osc_processor = OscProcessor(m) o = OscServer(m) o.start() signal.pause()
def consumer(input_q): while True: # Get an item from the queue item = input_q.get() # Process item print(item) # Signal completion input_q.task_done() # Signal task completion def producer(sequence,output_q): for item in sequence: # Put the item on the queue output_q.put(item) if __name__ == '__main__': # JoinableQueue(), a Queue subclass, is a queue which additionaly has task_done() and join() methods from multiprocessing import Process, JoinableQueue q = JoinableQueue() # Launch the consumer process cons_p = Process(target=consumer, args=(q,)) cons_p.daemon = True cons_p.start() # Run the producer function on some data sequence = range(100) # Replace with useful data producer(sequence, q) # Wait for the consumer to finish q.join()
from datetime import datetime, timezone import logging from multiprocessing import JoinableQueue import threading import select import sys import weakref from nemubot import __version__ from nemubot.consumer import Consumer, EventConsumer, MessageConsumer from nemubot import datastore import nemubot.hooks logger = logging.getLogger("nemubot") sync_queue = JoinableQueue() def sync_act(*args): sync_queue.put(list(args)) class Bot(threading.Thread): """Class containing the bot context and ensuring key goals""" def __init__(self, ip="127.0.0.1", modules_paths=list(), data_store=datastore.Abstract(), debug=False): """Initialize the bot context
def insert_files(self, out,cfg, producer,return_dict, skip_header=0, rec_delim=os.linesep): self.opt.skip_header = skip_header self.opt.rec_delim = rec_delim log = logging.getLogger('cli') self.scfg, self.tcfg = cfg file_object_cache = FileObjectCache() start = time.time() stat_queue = JoinableQueue() if 1: put_queue = JoinableQueue(1024 * self.opt.processes) if 1: put = {'update': self.put_update}[self.opt.put] putter_processes = list(islice(repeatedly(Process, target=self.putter, args=(put, put_queue, stat_queue, return_dict)), self.opt.processes)) for putter_process in putter_processes: putter_process.start() if 1: statter_process = Process(target=self.statter, args=(stat_queue, start)) statter_process.start() out_names=[] #walk = {'filesystem': self.walk_filesystem}[self.opt.walk] for file in producer[0](*producer[1]): out_names.append(file) put_queue.put(file) #time.sleep(3) out.dump_files=out_names for putter_process in putter_processes: put_queue.put(None) put_queue.close() for putter_process in putter_processes: putter_process.join() stat_queue.put(None) stat_queue.close() statter_process.join() put_queue.join_thread() stat_queue.join_thread() print 77777, counter.value() print 77777, self.total_ins print 7777, (return_dict.values())
def run(self): global sync_queue # Rewrite the sync_queue, as the daemonization process tend to disturb it old_sync_queue, sync_queue = sync_queue, JoinableQueue() while not old_sync_queue.empty(): sync_queue.put_nowait(old_sync_queue.get()) self._poll.register(sync_queue._reader, select.POLLIN | select.POLLPRI) self.stop = False # Relaunch events self._update_event_timer() logger.info("Starting main loop") while not self.stop: for fd, flag in self._poll.poll(): # Handle internal socket passing orders if fd != sync_queue._reader.fileno() and fd in self.servers: srv = self.servers[fd] if flag & (select.POLLERR | select.POLLHUP | select.POLLNVAL): try: srv.exception(flag) except: logger.exception( "Uncatched exception on server exception") if srv.fileno() > 0: if flag & (select.POLLOUT): try: srv.async_write() except: logger.exception( "Uncatched exception on server write") if flag & (select.POLLIN | select.POLLPRI): try: for i in srv.async_read(): self.receive_message(srv, i) except: logger.exception( "Uncatched exception on server read") else: del self.servers[fd] # Always check the sync queue while not sync_queue.empty(): args = sync_queue.get() action = args.pop(0) logger.debug("Executing sync_queue action %s%s", action, args) if action == "sckt" and len(args) >= 2: try: if args[0] == "write": self._poll.modify( int(args[1]), select.POLLOUT | select.POLLIN | select.POLLPRI) elif args[0] == "unwrite": self._poll.modify( int(args[1]), select.POLLIN | select.POLLPRI) elif args[0] == "register": self._poll.register( int(args[1]), select.POLLIN | select.POLLPRI) elif args[0] == "unregister": self._poll.unregister(int(args[1])) except: logger.exception( "Unhandled excpetion during action:") elif action == "exit": self.quit() elif action == "launch_consumer": pass # This is treated after the loop sync_queue.task_done() # Launch new consumer threads if necessary while self.cnsr_queue.qsize() > self.cnsr_thrd_size: # Next launch if two more items in queue self.cnsr_thrd_size += 2 c = Consumer(self) self.cnsr_thrd.append(c) c.start() sync_queue = None logger.info("Ending main loop")
class QiushiSpider(object): def __init__(self): self.url = 'https://www.qiushibaike.com/8hr/page/{}/' self.headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36' } # 三大队列,进程队列要使用JoinableQueue self.url_queue = JoinableQueue() # type:Queue self.page_queue = JoinableQueue() # type:Queue self.data_queue = JoinableQueue() # type:Queue def run(self): # 执行任务 self.excute_task(self.add_url) self.excute_task(self.add_page) self.excute_task(self.add_data) self.excute_task(self.save_data) # 等待队列任务完成 self.url_queue.join() self.page_queue.join() self.data_queue.join() def excute_task(self, task, count=1): """ 使用线程执行任务 :param task: 要执行任务函数(方法) :param count: 开启多少个线程来执行 """ for i in range(count): t = Process(target=task) t.daemon = True t.start() def add_url(self): """获取url列表,添加到url_queue队列""" for i in range(1, 14): url = self.url.format(i) self.url_queue.put(url) def add_page(self): """从url_queue获取url的页面,添加到page_queue队列""" while True: url = self.url_queue.get() response = requests.get(url=url, headers=self.headers) page = response.content.decode() self.page_queue.put(page) self.url_queue.task_done() # 表明url队列已完成 def add_data(self): """从page_queue获取页面,解析页面,添加到data_queue队列""" while True: page = self.page_queue.get() element = etree.HTML(page) divs = element.xpath('//*[@id="content-left"]/div') # 所有段子信息 data_list = [] for div in divs: """遍历divs,拿到每一个段子的div,然后把想要的数据保存在data_list中""" item = {} # 用户头像 item['head_img'] = div.xpath('./div[1]/a[1]/img/@src') item['head_img'] = 'https:' + item['head_img'][0] if len( item['head_img']) != 0 else None # 昵称 item['nick_name'] = div.xpath('./div[1]/a[2]/h2/text()') item['nick_name'] = item['nick_name'][0].strip() if len( item['nick_name']) != 0 else None # 性别 gender_class = div.xpath('./div[1]/div/@class') if len(gender_class) != 0: # 判断是否有填写性别 # articleGender womenIcon item['gender'] = re.findall('(\w+)Icon', gender_class[0])[0] else: item['gender'] = None # 段子内容 item['content'] = ''.join(div.xpath('./a/div/span//text()')) # 好笑数 item['vote_count'] = div.xpath( './/span[@class="stats-vote"]/i/text()')[0] # 评论数 item['comments_count'] = div.xpath( './/span[@class="stats-comments"]/a/i/text()')[0] # print(item) data_list.append(item) self.data_queue.put(data_list) # 添加到data队列 self.page_queue.task_done() # 表明page队列已完成 def save_data(self): """从data_queue拿到数据保存""" while True: data_list = self.data_queue.get() with open('./test/糗事百科热门(多进程版).jsonlines', 'a', encoding='utf-8') as f: for data in data_list: json.dump(data, f, ensure_ascii=False) f.write('\n') self.data_queue.task_done() # 完成数据队列任务
# coding: utf-8 import time from multiprocessing import freeze_support from multiprocessing import JoinableQueue from multiprocessing import Process from multiprocessing import Queue from random import random # 进程的Queue类并不支持task_done和join方法,需要使用特别的JoinableQueue tasks_queue = JoinableQueue() results_queue = Queue() def double(n): return n * 2 def producer(in_queue): while 1: wt = random() time.sleep(wt) in_queue.put((double, wt)) if wt > 0.9: in_queue.put(None) print("Stop Producer") break def consumer(in_queue, out_queue): while 1: task = in_queue.get()
def __init__(self): self.base_filename = "raw_data/conceptnet4_flat_" self.num_threads = 10 self.queue = JoinableQueue()