def init_logging(): """Initialize logging system.""" debug = '--debug' in sys.argv level = logging.DEBUG if debug else logging.INFO multiprocessing.log_to_stderr() logger.setLevel(level)
def __init__(self,cmd_param): ''' 主进程类 为了兼容windows而这样创建类 ''' super(Mtask,self).__init__() self.start_time = datetime.datetime.now() self.parent_timeout = 180 self.parent_timeout_flag = 0 self.child_timeout =120 self.child_num = 10 self.slice_num = 20 self.process_list = [] self.result =[] self.batch_id = 0 self.print_flag = 1 self.mult_debug_flag = 0 self.cmd_param = cmd_param if self.mult_debug_flag: #设置进程log日志 multiprocessing.log_to_stderr() logger=multiprocessing.get_logger() logger.setLevel(logging.INFO)
def main(): multiprocessing.log_to_stderr() logger = multiprocessing.get_logger() logger.setLevel(logging.INFO) pool = RedisPool(30) p = WorkerEngine(pool, 'Fetcher', 25) p.start(logger)
def setup_logger(loglevel=conf.DAEMON_LOG_LEVEL, logfile=None, format=conf.LOG_FORMAT, **kwargs): """Setup the ``multiprocessing`` logger. If ``logfile`` is not specified, ``stderr`` is used. Returns logger object. """ if not _monkeypatched[0]: monkeypatch() _monkeypatched[0] = True logger = get_default_logger(loglevel=loglevel) if logger.handlers: # Logger already configured return logger if logfile: if hasattr(logfile, "write"): log_file_handler = logging.StreamHandler(logfile) else: log_file_handler = logging.FileHandler(logfile) formatter = logging.Formatter(format) log_file_handler.setFormatter(formatter) logger.addHandler(log_file_handler) else: import multiprocessing multiprocessing.log_to_stderr() return logger
def test_log(): multiprocessing.log_to_stderr() logger = multiprocessing.get_logger() logger.setLevel(logging.INFO) t1 = time.time() print(time.time() - t1) logger.info("done")
def run_prevalence(out_dir, remove_rt, working_dir, produce_prevalence): """ Create exact prevalences over a list of DRMs provided by a sequence object when provided with simulated resistant and susceptible files. Args: out_dir: The final folder into which to place the anonymized files. remove_rt: Are we removing a piece of the RT gene? working_dir: The folder in which to place temporary files. produce_prevalence: what prevalence to produce Returns: True on completion. """ print "Building exact prevalences." if not os.path.exists(out_dir): os.makedirs(out_dir) csv_rows = [] threads = [] with open(os.path.join(working_dir, error_filename), 'rb') as handle: error_data = pickle.load(handle) platform = error_data['platform'] paired_end = error_data['paired_end'] manifest_queue = multiprocessing.Manager().Queue() multiprocessing.log_to_stderr() p = LoggingPool(len(error_data['files'])) for result in error_data['files']: if remove_rt: result['sequence'].remove_rt = True process_result = p.apply_async(run_prevalence_thread, [ manifest_queue, platform, paired_end, result, working_dir, out_dir, produce_prevalence ]) p.close() p.join() with open(os.path.join(out_dir, final_csv_filename), 'w') as csv_handle,\ open(os.path.join(out_dir, final_json_filename), 'w') as json_handle: csv_handle.write(','.join(csv_header_rows) + '\n') test = sample.header(plat.platforms[platform], paired_end) test['samples'] = [] while not manifest_queue.empty(): s = manifest_queue.get() test['samples'].append(s.encode(plat.platforms[platform])) csv_handle.write(s.dump_csv()) json_handle.write(json.dumps(test, indent=2)) return True
def SumData(roi_start_list, roi_end_list, file_list, file_path, istart, iend, queue): logger = mp.get_logger() mp.log_to_stderr(logging.INFO) roi_sums = mproc.SHARED_ARRAY data_array = XYDataArray() num_rois = len(roi_start_list) logger.info("Reading files from %d to %d" % (istart, iend)) if istart > 0: istart = istart - 1 # Process each file in the list that falls in the range istart to iend for i in range(istart, iend): # Read in the information from the file and create numpy arrays from that information. data_array.CreateArrays(os.path.join(file_path, file_list[i])) # Sum the data in the arrays that lies between the roi values. Do this # for each roi that was created. for j in range(num_rois): # logger.info("Summing roi %d from file %d" % (j, i)) roi_sums[j][i] = roi_sums[j][i] + data_array.SumROIData(roi_start_list[j], roi_end_list[j]) # Add a value of 1 to the queue so that the user interface can be updated # with the latest progress. queue.put(1) return roi_sums
def SumPixels(file_list, file_path, istart, iend, queue): logger = mp.get_logger() mp.log_to_stderr(logging.INFO) roi_sums = mproc.SHARED_ARRAY data_array = TiffDatatArray() logger.info("Reading files from %d to %d" % (istart, iend)) if istart > 0: istart = istart - 1 # Process each file in the list that falls in the range istart to iend for i in range(istart, iend): # Read in the information from the file and create numpy arrays from that information. data_array.CreateArrays(os.path.join(file_path, file_list[i])) # Sum the data in the arrays that lies between the roi values. Do this # for each roi that was created. new_sum = data_array.GetDataArray() roi_sums = numpy.add(roi_sums, new_sum) # Add a value of 1 to the queue so that the user interface can be updated # with the latest progress. queue.put(1) return roi_sums
def init(self, nprocs=None, spawnonce=True): multiprocessing.log_to_stderr(logging.WARN) if nprocs is None: self.nprocs = multiprocessing.cpu_count() else: self.nprocs = multiprocessing.cpu_count() + nprocs if nprocs < 0 else nprocs self.proc = [] self.spawnonce = spawnonce
def main(): if len(sys.argv) >1: threads = sys.argv[1] else: threads =1 print('Threads to process :',threads) multiprocessing.log_to_stderr() logger =multiprocessing.get_logger() logger.setLevel(logging.INFO) # while True: Threadstart(int(threads))
def which_host(self, urllist, attr): """check every url in the given list against all regular expressions and extract the value of the chosen html attribute. Then use a queue and enough processes to download all matched urls""" # make a queue and enough processes as numprocs self.q = Queue() self.ps = (Process(target=self.use_queue, args=()) for i in range(self.numprocs)) # enable multiprocessing logging feature if debug: logger.setLevel(logging.DEBUG) log_to_stderr(logging.DEBUG) for p in self.ps: # start all processes p.start() # piping the urllist urls into a set to purge duplicates finalset = set() for L in urllist: self.stringl = L.get(attr, None) # remove the anonym.to string before urls if self.stringl.startswith("http://anonym.to/?"): self.stringl = re.sub('http://anonym.to/\?', '', self.stringl) finalset.add(self.stringl) for L in finalset: # iterate over the regexp dictionary items; when finding a url # matching, put the the class name, url and self.basedir in the queue for k, v in regexp_dict.items(): if k.search(L): self.logger.info("downloading %s" % L) # instantiate and then pass the parse method to the queue. # it downloads but doesn't make the queue do its job # parser = v(L, self.basedir) # self.q.put((parser.parse())) # add the class name and the parameters needed for its __init__ # into the queue self.q.put((v, (L, self.basedir))) self.img_counter = self.img_counter + 1 else: continue for i in range(self.numprocs): # put a STOP to end the iter builtin inside use_queue self.q.put("STOP") self.logger.info('%d images were present' % self.img_counter) print("%d images were present" % self.img_counter)
def main(): global csvout if len(sys.argv) >1: threads = sys.argv[1] else: threads = 1 print('Threads to run :',threads) multiprocessing.log_to_stderr() logger = multiprocessing.get_logger() logger.setLevel(logging.INFO) Threadstart(threads)
def multiprocess(args): step = ((args.token_hi - args.token_lo) / args.worker_count) + 1 tr1 = range(args.token_lo, args.token_hi, step) # intermediate points tr2 = [(t, t + 1) for t in tr1[1:]] # add adjacent points tr3 = [t for st in tr2 for t in st] # flatten tr4 = [args.token_lo] + tr3 + [args.token_hi] # add end points token_ranges = [tr4[i:i + 2] for i in range(0, len(tr4), 2)] # make pairs rate = args.throttle_rate / args.worker_count multiprocessing.log_to_stderr().setLevel(logging.INFO) manager = Manager() results = manager.dict() # create a special shared dict to gather results workers = [ Process( target=main, args=( args, worker_index, token_ranges[worker_index], rate, results ) ) for worker_index in range(args.worker_count) ] os_times_start = os.times() for worker in workers: worker.start() for worker in workers: worker.join() os_times_stop = os.times() exitcode = 0 for worker in workers: if worker.exitcode: exitcode = worker.exitcode break if results: # transform the special dict results_dict = analyze_results(results, os_times_stop, os_times_start) if args.json_output: print_json(args, results_dict) else: print_arguments(args) print_results(results_dict) return(exitcode)
def run_error(platform, working_dir, pcr_error, env_error, human_error, paired_end): """ Simulate sequencing error from the output generated by run_diversity. Args: platform: string One of "roche", "illumina" or "ion". working_dir: path The folder in which to place temporary files. pcr_error: bool Should we include a PCR error env_error: bool Should we include an ENV error human_error: bool Should we include human DNA paired_end: bool Are we simulating paired_end data? Returns: True on completion. """ print "Simulating reads from sequence sets." error_data = {} # properties of the simulation error_data['paired_end'] = paired_end error_data['platform'] = platform with open(os.path.join(working_dir, evolved_filename), 'rb') as handle: evolved_data = pickle.load(handle) multiprocessing.log_to_stderr() p = LoggingPool(len(evolved_data)) file_queue = multiprocessing.Manager().Queue() for result in evolved_data: p.apply_async(run_error_thread, [ file_queue, result, platform, working_dir, pcr_error, env_error, human_error, paired_end ]) p.close() p.join() error_data['files'] = [] while not file_queue.empty(): error_data['files'].append(file_queue.get()) with open(os.path.join(working_dir, error_filename), 'wb') as handle: pickle.dump(error_data, handle, protocol=pickle.HIGHEST_PROTOCOL) return True
def doMultiprocess(app): applist = {'pdb2pqr':doPDB2PQR, 'apbs':doAPBS, 'chimera':doChimera} multiprocessing.log_to_stderr(logging.INFO) # set logging to info level rather than DEBUG pdb_ids = glbl.model_input.keys() manager = Manager() # creates shared memory manager object nextPDBid = Queue() # Create Queue object to serve as shared id generator across processes for pid in pdb_ids: nextPDBid.put(pid) # Load the ids to be tested into the Queue for x in range(0,multiprocessing.cpu_count()): # Create one process per logical CPU p = Process(target=applist[app], args=(nextPDBid,)) # Assign process to app function, passing in the Queue glbl.jobs.append(p) # Add the process to a list of running processes p.start() # Start process running for j in glbl.jobs: j.join() return
def main(): logger = multiprocessing.log_to_stderr() logger.setLevel(logging.DEBUG) logger.info('Start main module') managerSpeech = multiprocessing.Manager() managerSpeechDict = managerSpeech.dict() managerSpeechEvent = managerSpeech.Event() juliusProc = multiprocessing.Process(target=coreJulius, args=(logger, managerSpeechDict, managerSpeechEvent)) juliusProc.name = "CoreJulius" juliusProc.daemon = True juliusProc.start() speechProc = multiprocessing.Process(target=coreSpeech, args=(logger, managerSpeechDict, managerSpeechEvent)) speechProc.name = "CoreSpeech" speechProc.daemon = True speechProc.start() core_speech.play_sound("do_start.wav", logger) while True: try: time.sleep(1) except KeyboardInterrupt: juliusProc.join(2) if juliusProc.is_alive(): juliusProc.terminate() speechProc.join(2) if speechProc.is_alive(): speechProc.terminate() managerSpeech.shutdown() break
def main(): multiprocessing.log_to_stderr() logger = multiprocessing.get_logger() logger.setLevel(logging.INFO) path = '' if len(sys.argv) not in (1, 2): print('error: wrong number of arguments.') print('usage: {} (to use hackrf)'.format(sys.argv[0])) print('usage: {} <file> (to use raw file)'.format(sys.argv[0])) sys.exit(1) else: if len(sys.argv) is 2: path = sys.argv[1] spectacle = Spectacle(path)
def setupParallelLogging( self ) : # --------------------------------------------------- # set up Logging # ---------------- # from multiprocessing import enableLogging, getLogger import multiprocessing # preliminaries for handlers/output files, etc. from time import ctime datetime = ctime() datetime = datetime.replace(' ', '_') outfile = open( 'gaudirun-%s.log'%(datetime), 'w' ) # two handlers, one for a log file, one for terminal streamhandler = logging.StreamHandler(strm=outfile) console = logging.StreamHandler() # create formatter : the params in parentheses are variable names available via logging formatter = logging.Formatter( "%(asctime)s - %(name)s - %(levelname)s - %(message)s" ) # add formatter to Handler streamhandler.setFormatter(formatter) console.setFormatter(formatter) # now, configure the logger # enableLogging( level=0 ) # self.log = getLogger() self.log = multiprocessing.log_to_stderr() self.log.setLevel( logging.INFO ) self.log.name = 'Gaudi/Main.py Logger' self.log.handlers = [] # add handlers to logger : one for output to a file, one for console output self.log.addHandler(streamhandler) self.log.addHandler(console) self.log.removeHandler(console) # set level!! self.log.setLevel = logging.INFO
def InternalSet(Achild:Queue, Bchild:Queue, outqueue:Queue): """Take the output of two LeafSet's and take the union.""" logger = multiprocessing.log_to_stderr() logger.setLevel(logging.INFO) AminusB = set() BminusA = set() morestuff = True while morestuff: a = Achild.get() b = Bchild.get() logger.info("Internal:%s:%s" % (a, b)) if a in BminusA: BminusA.remove(a) elif a not in AminusB: AminusB.add(a) outqueue.put(a) if b in AminusB: AminusB.remove(b) elif b not in BminusA: BminusA.add(b) outqueue.put(b) Achild.task_done() Bchild.task_done() if (a == SIGOBJ) or (b == SIGOBJ): outqueue.put(SIGOBJ) morestuff = False logger.info("internal done")
def main(at_once=100): # file detailing what to download # each line should be of format: # fileType, color, brand, model, url, hash fname = sys.argv[1] # multiprocessing.Pool 33sec per 100 photos # multiprocessing.dummy.Pool ncpu*3 22sec per 100 photos # 500 for 120sec, 3*ncpu logger = multiprocessing.log_to_stderr() logger.setLevel(multiprocessing.SUBDEBUG) # NUM_WORKERS = multiprocessing.cpu_count() * 2 p = Pool() # get listing of files to download task_list = read_tasking(fname) print('processed {0} lines to download'.format(len(task_list))) print('tasking loaded') # download in batches good = open(fname + '.good', 'w') bad = open(fname + '.bad', 'w') for batch in tqdm(grouper(at_once, task_list)): retval = p.map(work_func, batch) for r, t in retval: if r: good.write(json.dumps(t)+'\n') else: bad.write(json.dumps(t)+'\n') good.flush() bad.flush() good.close() bad.close()
def par_fit( K, ChunkGenerator, nProc=1, timeout=120, doVerbose=False): logger = mp.log_to_stderr() if doVerbose: logger.setLevel(logging.INFO) item1 = ChunkGenerator.next() if type( item1 ) == str: Xinit = np.loadtxt( item1 ) funcChunk = fitChunkFromFile else: funcChunk = fitChunk Xinit = item1 D = Xinit.shape[1] initShared( Xinit ) # Run in parallel! workerpool = mp.Pool( processes=nProc ) R = workerpool.map_async( funcChunk, ChunkGenerator ) R.get( timeout=timeout ) workerpool.close() # just in case... # Build mixture model with resulting parameters! mygmm = sklearn.mixture.GMM( n_components=K ) mygmm.weights_ = sh2np( shWeights ) mygmm.covars_ = sh2np( shCovars, (K,D) ) mygmm.means_ = sh2np( shMeans, (K,D) ) return mygmm
def Estep_parallel( Xin, w, MuList, SigmaList, nProc=2, chunksize=1, doVerbose=False ): ''' Returns ------- logResp : N x K vector of log posterior probabilities logResp[n,k] : n-th data point's posterior under k-th comp ''' '''def obs_comp_generator( MuList, SigmaList): for k in xrange( len(MuList) ): yield k,MuList[k], SigmaList[k] GMMCompIterator = obs_comp_generator( MuList, SigmaList ) ''' if doVerbose: logger = mp.log_to_stderr() logger.setLevel( logging.INFO ) global X X = Xin GMMCompIterator = [ (k,MuList[k],SigmaList[k]) for k in xrange( len(MuList) )] mypool = mp.Pool( processes=nProc ) myParOp = mypool.map_async( loggausspdf_globaldata, GMMCompIterator, chunksize=chunksize ) resultList = myParOp.get() #st = time.time() logResp = np.vstack(resultList) #print ' Reduction: %.2f sec' % (time.time()-st) # Time to agg results into single matrix: 0.07 sec for N=250000,K=25 # Conclusion: agg results takes almost no time relative to each individual job return logResp.T + np.log(w)
def main(): logger = multiprocessing.log_to_stderr() logger.setLevel(multiprocessing.SUBDEBUG) if len(sys.argv) >= 3: #directory containing samples which contains fastq samplesDir = getAbsPath(sys.argv[1]) #quality scores file qScoreFile = getAbsPath(sys.argv[2]).rstrip('/') #perl script for qscore calculation qScorePerlScript = getAbsPath(sys.argv[3]).rstrip('/') #get all sample directory inside samplesDir allSamplesDir = allSampleDirsPath(samplesDir) #call child workers to do the job results = prepAndRunWorkers(allSamplesDir, qScoreFile, qScorePerlScript) #print final results print "avgQScoreComputation success: ", all(results) else: print 'err: files missing'
def broadFirst(self, key): ''' True means no need to dig more False means need to dig to next level ''' count = 0 logger = multiprocessing.log_to_stderr(multiprocessing.SUBDEBUG) state = downloadTile(key, self.config.basic_url, self.config.image_floder) count = count + 1 bottom = 0 l = self.moveToNextLevel(key) q = Queue.Queue() for i in l: q.put(i) while not q.empty(): tl = [] for _ in range(4): tl.append(q.get()) count = count + 1 for t in tl: state = downloadTile(t, self.config.basic_url, self.config.image_floder) # state = True if state and len(t)< self.config.max_z: l = self.moveToNextLevel(t) for _ in l: q.put(_) if count - bottom >= 1000: logger.debug('%s: Roughly Process: %d/21845' % (time.ctime(), count)) bottom = count # logger.debug('%d' % count) return True
def setUp(self): """Start the agent and wait for it to start""" super(FunctionalBase, self).setUp() mpl = multiprocessing.log_to_stderr() mpl.setLevel(logging.INFO) self.test_port = os.environ.get('TEST_PORT', '9999') # Build a basic standalone agent using the config option defaults. # 127.0.0.1:6835 is the fake Ironic client. self.agent = agent.IronicPythonAgent( 'http://127.0.0.1:6835', 'localhost', ('0.0.0.0', int(self.test_port)), 3, 10, None, 300, 1, 'agent_ipmitool', True) self.process = multiprocessing.Process( target=self.agent.run) self.process.start() self.addCleanup(self.process.terminate) # Wait for process to start, otherwise we have a race for tests sleep_time = 0.1 tries = 0 max_tries = int(os.environ.get('IPA_WAIT_TRIES', '100')) while tries < max_tries: try: return self.request('get', 'commands') except requests.ConnectionError: time.sleep(sleep_time) tries += 1 raise IOError('Agent did not start after %s seconds.' % (max_tries * sleep_time))
def dispatcher(success_fp, fail_fp, partitions): """Dispatch execution over a pool of processors Parameters ---------- success_fp : file-like object A file-like object to write a list of successful sample IDs too fail_fp : file-like object A file-like object to write a list of unsuccessful sample IDs too, and any associated error information partitions : Iterable of (function, Iterable of str) Yields a function and an iterable of IDs. It is expected that the functions yielded will have the following signature: {str: list} <- function(list of str) """ if ag.is_test_env(): logger = mp.log_to_stderr() logger.setLevel(logging.INFO) pool = mp.Pool(processes=agenv.get_cpu_count()) success_fp.write('%s\n' % '#SampleID') fail_fp.write('%s\t%s\n' % ('#SampleID', 'Error(s)')) for func, ids in partitions: functor = partial(run_functor, func) for success_details in pool.map(functor, list(agru.chunk_list(ids))): for id_, detail in success_details.items(): if detail: fail_fp.write("%s\t%s\n" % (id_, '\t'.join(detail))) else: success_fp.write("%s\n" % id_)
def processQueueThread(self, event_queue, notifier, timeout, processLock, done_sending, log_level): """Send events without blocking swift proxy.""" log = multiprocessing.log_to_stderr() log.setLevel(log_level) event_buffer = self.start_sender_process(self._notifier) while True: try: log.debug('Wait for event from send queue') event = event_queue.get() if event: log.debug('Got event %s from queue - trigger ' 'send', event.id) self.trigger_send(event, event_buffer, done_sending, log) while done_sending.wait(timeout) is False: log.warning('Timeout sending event %s - ' 'retry in new thread.', event.id) processLock.acquire() self.event_sender_die(event_buffer, done_sending) event_buffer = self.start_sender_process(notifier) processLock.release() self.trigger_send(event, event_buffer, done_sending, log) else: event_buffer.put(None) break except BaseException: LOG.error("Exception in sending thread", exc_info=True) LOG.exception("SendEventThread loop exception")
def reader(readq, resultq): logger = multiprocessing.log_to_stderr() logger.info("Start reading") BLOCKSIZE = 1024*1024 while True: item = readq.get() if item[0] == 'Q': logger.warning("Got reader quit") break elif item[0] == 'R': path = item[1] bytesRead = 0 fileSize = item[2].st_size try: with open(path, 'rb') as f: while bytesRead < fileSize: buf = f.read(BLOCKSIZE) resultq.put(['W']) if len(buf) == 0: break bytesRead += len(buf) buf = f.read(BLOCKSIZE) if len(buf) != 0: raise Exception("Expected EOF for %s" % path) resultq.put(['E']) except Exception as inst: print("open() failed:", inst) logger.info("Finished reading") resultq.put(['Q'])
def main(): logger = multiprocessing.log_to_stderr() logger.setLevel(multiprocessing.SUBDEBUG) if len(sys.argv) >= 4: #directory containing fastq library fastqsDir = workerForBam.getAbsPath(sys.argv[1]) #directory containing other directories with fasta names fastaDir = workerForBam.getAbsPath(sys.argv[2]) #directory containing file locks lockDirPath = workerForBam.getAbsPath(sys.argv[3]) #directory containing temp output -> fastQ's, jobsFile outDir = workerForBam.getAbsPath(sys.argv[4]) #write all fastq's processing in job file combineJobPath = writeCombineBAMJobsFromSAI(outDir, fastqsDir,\ fastaDir,\ lockDirPath) #call workers to generate paired BAMs from SAIs #results = callPairedSAIToBAMWorkers(fastqsDir, fastaDir) #print results else: print 'err: files missing'
def run(host, port, params, force): # configure logging mplogger = mp.log_to_stderr() mplogger.setLevel(params['loglevel']) logger.setLevel(params['loglevel']) if logger.getEffectiveLevel() <= 10: logRequests = True else: logRequests = False # create the server manager = TaskManager(params, force) server = TaskManagerServer( (host, port), logRequests=logRequests, allow_none=True) server.register_function(manager.load_tasks, 'panda_reload') server.register_function(manager.get_sim_root, 'panda_connect') server.register_function(manager.get_next_task, 'panda_request') server.register_function(manager.set_complete, 'panda_complete') server.register_function(manager.set_error, 'panda_error') server.register_function(manager.get_status, 'panda_status') server.register_multicall_functions() server.register_introspection_functions() logger.info("Started XML-RPC server at %s:%d" % (host, port)) server.serve_forever()
def main(): logger = multiprocessing.log_to_stderr() logger.setLevel(logging.INFO) logger.info('Odometry processor started') odometryprocessor = OdometryProcessor(logger) rospy.spin()
import datetime import json import logging import multiprocessing as mp import sys import time import paho.mqtt.client as mqtt MQTT_SERVER = "127.0.0.1" MQTT_TOPIC_PREFIX = "home/rtl_433" TIMEOUT_STALE_SENSOR = 600 # Seconds before showing a timeout indicator # log = logging.getLogger() # Single process logger log = mp.log_to_stderr() # Multiprocessing capable logger mqtt_client = mqtt.Client("RTL_433_Test") sensor_state = dict() # Dictionary containing accumulated sensor state def print_sensor_state(): """ Print accumulated sensor state """ time_now = datetime.datetime.utcnow().replace(microsecond=0) print("\nUpdate per {} UTC".format(time_now.isoformat(sep=' '))) for model in sensor_state: print(model) for ID in sensor_state[model]: data = sensor_state[model][ID]['data'].copy() timestamp = data.pop('time') timedelta = (time_now - timestamp).total_seconds()
self.dose_duration_405_seconds * self.analog_out.rate, #Autorounds self.analog_out.num_channels), dtype=np.float64) voltages['dose_405'][:-1, 2] = self.dose_voltage_405 self.voltages = voltages self.last_played_voltage = None return None def close(self): self.idp.close() self.analog_out.close() self.stage.close() if __name__ == '__main__': logger = mp.log_to_stderr() logger.setLevel(logging.INFO) scanner = Scanner(zaber_stage_port_name='COM3') try: for x in range(1, 100000, 1000): print("New x position:", x) for i in range(3): scanner.snap('488', x=x, y=20000, stage_cooldown_seconds=0.1) ## input("Hit enter...") finally: scanner.close() input("Hit enter to finish")
from django.core.management.base import BaseCommand from django.db import close_old_connections, reset_queries from django.utils.encoding import force_text, smart_bytes from django.utils.timezone import now from haystack import connections as haystack_connections from haystack.exceptions import NotHandled from haystack.query import SearchQuerySet from haystack.utils.app_loading import haystack_get_models, haystack_load_apps DEFAULT_BATCH_SIZE = None DEFAULT_AGE = None DEFAULT_MAX_RETRIES = 5 LOG = multiprocessing.log_to_stderr(level=logging.WARNING) def update_worker(args): if len(args) != 10: LOG.error('update_worker received incorrect arguments: %r', args) raise ValueError('update_worker received incorrect arguments') model, start, end, total, using, start_date, end_date, verbosity, commit, max_retries = args # FIXME: confirm that this is still relevant with modern versions of Django: # We need to reset the connections, otherwise the different processes # will try to share the connection, which causes things to blow up. from django.db import connections for alias, info in connections.databases.items():
else: raise RuntimeError( "User choose option {} , close the program!".format( op)) #test 3.5.1- multiprocessing loop #hrv - 0.114s - 100 samples #warning: only works with amp object initiated inside the child process #warning: passing amp object as an arg to the child process, or directly to the worker function does not work TEST_35_1 = False if TEST_35_1: import multiprocessing import logging import sys multiprocessing.log_to_stderr() logger = multiprocessing.get_logger() logger.setLevel(logging.DEBUG) #Sharing state between processes¶ - Method: shared memory map Value run = multiprocessing.Value('b', False) #start False print("run.value", run.value) starttime = time.time() p = multiprocessing.Process(name="loop_get_data", target=hrv_worker, args=(run, )) p.daemon = True #To exit the main process even if the child process p didn't finished p.start() #start process
def synchronous_pull_with_lease_management(project_id, subscription_id): """Pulling messages synchronously with lease management""" # [START pubsub_subscriber_sync_pull_with_lease] import logging import multiprocessing import random import time from google.cloud import pubsub_v1 # TODO(developer) # project_id = "your-project-id" # subscription_id = "your-subscription-id" subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path(project_id, subscription_id) NUM_MESSAGES = 2 ACK_DEADLINE = 30 SLEEP_TIME = 10 # The subscriber pulls a specific number of messages. response = subscriber.pull(request={ "subscription": subscription_path, "max_messages": NUM_MESSAGES }) multiprocessing.log_to_stderr() logger = multiprocessing.get_logger() logger.setLevel(logging.INFO) def worker(msg): """Simulates a long-running process.""" RUN_TIME = random.randint(1, 60) logger.info("{}: Running {} for {}s".format( time.strftime("%X", time.gmtime()), msg.message.data, RUN_TIME)) time.sleep(RUN_TIME) # `processes` stores process as key and ack id and message as values. processes = dict() for message in response.received_messages: process = multiprocessing.Process(target=worker, args=(message, )) processes[process] = (message.ack_id, message.message.data) process.start() while processes: for process in list(processes): ack_id, msg_data = processes[process] # If the process is still running, reset the ack deadline as # specified by ACK_DEADLINE once every while as specified # by SLEEP_TIME. if process.is_alive(): # `ack_deadline_seconds` must be between 10 to 600. subscriber.modify_ack_deadline( request={ "subscription": subscription_path, "ack_ids": [ack_id], "ack_deadline_seconds": ACK_DEADLINE, }) logger.info("{}: Reset ack deadline for {} for {}s".format( time.strftime("%X", time.gmtime()), msg_data, ACK_DEADLINE, )) # If the processs is finished, acknowledges using `ack_id`. else: subscriber.acknowledge(request={ "subscription": subscription_path, "ack_ids": [ack_id] }) logger.info("{}: Acknowledged {}".format( time.strftime("%X", time.gmtime()), msg_data)) processes.pop(process) # If there are still processes running, sleeps the thread. if processes: time.sleep(SLEEP_TIME) print("Received and acknowledged {} messages. Done.".format( len(response.received_messages))) # Close the underlying gPRC channel. Alternatively, wrap subscriber in # a 'with' block to automatically call close() when done. subscriber.close()
def get_mp_logger(level=logging.DEBUG): logger = multiprocessing.log_to_stderr() logger.setLevel(level) return logger
while True: _, img = cap.read() if img is not None: mainQ.put(img) def show(mainQ): cv2.namedWindow('test1') while True: frame1 = mainQ.get() frame1 = cv2.flip(frame1, 1) cv2.imshow('test1', frame1) if cv2.waitKey(1) & 0xFF == ord('q'): break if __name__ == '__main__': logMessages = multiprocessing.log_to_stderr() logMessages.setLevel(multiprocessing.SUBDEBUG) mainQ = multiprocessing.Queue() read_process = multiprocessing.Process(target=read, args=(mainQ, )) read_process.start() show_process = multiprocessing.Process(target=show, args=(mainQ, )) show_process.start() read_process.join() show.join()
def main(): logger = mp.log_to_stderr() #logger.setLevel(logging.CRITICAL) logger.setLevel(logging.INFO) #logger.setLevel(logging.FATAL) # create shared array mgr = mp.Manager() #d = mgr.dict() d = {} N, M = 15000, 11 #base_ptr = mp.RawArray(ctypes.c_double, N) #arr = tonumpyarray(base_ptr) #arr[:] = np.array(np.random.uniform(size=N)*1000, np.int) #arr_orig = arr.copy() arr = np.array(np.random.uniform(size=N) * 1000, np.int) tmp = np.ctypeslib.as_ctypes(arr) shared_arr = mp.RawArray(tmp._type_, tmp) print(shared_arr) d["array"] = shared_arr lst_event = [] for i in range(5): lst_event.append(mp.Event()) lst_event[0].set() value_ptr = mp.RawArray(ctypes.c_int, 2) value_ptr[0] = 0 value_ptr[1] = 0 d["value"] = [0, 0] p1 = mp.Process(target=exec1, args=( d, lst_event, )) p2 = mp.Process(target=exec2, args=( d, lst_event, )) t = time.time() p1.start() p2.start() info("Starting") lst_event[3].wait() lst_event[4].wait() info("All Started") lst_event[0].clear() lst_event[1].wait() lst_event[2].wait() time.sleep(1) arr = np.ctypeslib.as_array(d["array"]) #info(arr) crit(arr[0]) crit(arr[arr.size // 2 - 1]) crit(arr[arr.size // 2]) crit(arr[arr.size - 1]) # write to arr from different processes # with closing(mp.Pool(initializer=init, initargs=(shared_arr,))) as p: # # many processes access the same slice # stop_f = N // 10 # p.map_async(f, [slice(stop_f)]*M) # # many processes access different slices of the same array # assert M % 2 # odd # step = N // 10 # p.map_async(g, [slice(i, i + step) for i in range(stop_f, N, step)]) # p.join() # assert np.allclose(((-1)**M)*tonumpyarray(shared_arr), arr_orig) return t
import logging def daemon(): p = multiprocessing.current_process() print('Starting:', p.name, p.pid, flush=True) time.sleep(2) print('Exiting :', p.name, p.pid, flush=True) def non_daemon(): p = multiprocessing.current_process() print('Starting:', p.name, p.pid, flush=True) print('Exiting :', p.name, p.pid, flush=True) if __name__ == '__main__': print(print('==='*15 + ' < ' + 'MAIN PROCESS' + ' > ' + '==='*15)) logger = multiprocessing.log_to_stderr(logging.INFO) d = multiprocessing.Process(name='MyDaemon', target=daemon) d.daemon = True n = multiprocessing.Process(name='non-daemon', target=non_daemon) n.daemon = False d.start() print('==='*15) time.sleep(1) n.start() d.join() n.join()
# a python entry point for a function if __name__ == "__main__": start = time() path = r"/Users/jake/Desktop/test" fanout_unziptar(path) end = time() print('script ended after {} mins'.format((end-start)/60)) # set special list for results of multiprocess to pump within distances = mp.Manager().list() # set error logging import logging mpl = mp.log_to_stderr() mpl.setLevel(logging.INFO) # Using pandas ---------------- # IMPT: does not work when applying on string columns def some_func(x): x = x*29/2 return x import swifter train['new_col'] = train['non_string'].swifter.apply(some_func)
def get_stderr_logger(level=multiprocessing.SUBDEBUG): lug = multiprocessing.log_to_stderr() lug.setLevel(level) return lug
for pair in self.prod: params = f'{pair[0]}_{pair[1]}' t2.append( Thread(name=params, target=self.translate, args=(params, self.requests[params], sock))) t1.start() for t in t2: t.start() if __name__ == '__main__': app.url_map.converters['uuid'] = UUIDConverter app.config['JSON_SORT_KEYS'] = False logger = log_to_stderr() logger.setLevel(logging.INFO) logging.basicConfig( level=logging.DEBUG, format='[%(levelname)s](%(threadName)-10s)%(message)s', ) parser = ConfigParser() parser.read('dev.ini') params = ['text', 'auth', 'olang', 'odomain'] odomain_code_mapping = { 'fml': 'Formal', 'inf': 'Informal', 'auto': 'Auto', 'tt': 'tt', 'cr': 'cr'
def multiprocessing_debug_info(): """ Run this to see what multiprocessing is doing. :return: """ multiprocessing.log_to_stderr().setLevel(logging.DEBUG)
# """thread worker function""" # start = time.time() # time.sleep(num) # print 'Worker:', num, time.time() - start # # return # if __name__ == '__main__': # # jobs = [] # start = time.time() # for i in range(5): # p = Process(target=worker, args=(i,)) # # jobs.append(p) # p.start() # print "finished", time.time()-start # ============================================== import multiprocessing import logging import sys def worker(): print 'Doing some work' sys.stdout.flush() if __name__ == '__main__': multiprocessing.log_to_stderr(logging.DEBUG) p = multiprocessing.Process(target=worker) p.start() p.join()
def main(argv=None): """Command line interface to fragment extraction.""" if argv is None: argv = sys.argv[2:] try: opts, args = gnu_getopt(argv, 'ho:', FLAGS + OPTIONS) except GetoptError as err: print('error:', err, file=sys.stderr) print(SHORTUSAGE) sys.exit(2) opts = dict(opts) for flag in FLAGS: PARAMS[flag] = '--' + flag in opts PARAMS['disc'] = opts.get('--fmt', 'bracket') != 'bracket' PARAMS['fmt'] = opts.get('--fmt', 'bracket') numproc = int(opts.get('--numproc', 1)) if numproc == 0: numproc = cpu_count() if not numproc: raise ValueError('numproc should be an integer > 0. got: %r' % numproc) limit = int(opts.get('--numtrees', 0)) or None PARAMS['cover'] = None if '--cover' in opts and ',' in opts['--cover']: a, b = opts['--cover'].split(',') PARAMS['cover'] = int(a), int(b) elif '--cover' in opts: PARAMS['cover'] = int(opts.get('--cover', 0)), 999 PARAMS['twoterms'] = opts.get('--twoterms') encoding = opts.get('--encoding', 'utf8') batchdir = opts.get('--batch') if len(args) < 1: print('missing treebank argument') if batchdir is None and len(args) not in (1, 2): print('incorrect number of arguments:', args, file=sys.stderr) print(SHORTUSAGE) sys.exit(2) if batchdir: if numproc != 1: raise ValueError('Batch mode only supported in single-process ' 'mode. Use the xargs command for multi-processing.') tmp = None for n, fname in enumerate(args): if fname == '-': if numproc != 1: # write to temp file so that contents can be read # in multiple processes if tmp is not None: raise ValueError('can only read from stdin once.') tmp = tempfile.NamedTemporaryFile() tmp.write(open(sys.stdin.fileno(), 'rb').read()) tmp.flush() args[n] = tmp.name elif not os.path.exists(fname): raise ValueError('not found: %r' % fname) if PARAMS['complete']: if len(args) < 2: raise ValueError('need at least two treebanks with --complete.') if PARAMS['twoterms'] or PARAMS['adjacent']: raise ValueError('--twoterms and --adjacent are incompatible ' 'with --complete.') if PARAMS['approx'] or PARAMS['nofreq']: raise ValueError('--complete is incompatible with --nofreq ' 'and --approx') level = logging.WARNING if PARAMS['quiet'] else logging.DEBUG logging.basicConfig(level=level, format='%(message)s') if PARAMS['debug'] and numproc > 1: logger = multiprocessing.log_to_stderr() logger.setLevel(multiprocessing.SUBDEBUG) logging.info('Disco-DOP Fragment Extractor') logging.info('parameters:\n%s', '\n'.join(' %s:\t%r' % kv for kv in sorted(PARAMS.items()))) logging.info('\n'.join('treebank%d: %s' % (n + 1, a) for n, a in enumerate(args))) if numproc == 1 and batchdir: batch(batchdir, args, limit, encoding, '--debin' in opts) else: fragmentkeys, counts = regular(args, numproc, limit, encoding) out = (io.open(opts['-o'], 'w', encoding=encoding) if '-o' in opts else None) if '--debin' in opts: fragmentkeys = debinarize(fragmentkeys) printfragments(fragmentkeys, counts, out=out) if tmp is not None: del tmp
'--process_names', help= ('space delimited list of file of process names, used for model saving filepaths, ' 'one per review file'), nargs='+', type=str) parser.add_argument( '-c', '--categories', help=('space delimited list of categories to map to review files, ' 'one per review file'), nargs='+', type=str) args = parser.parse_args() loglevel = args.loglevel.upper() if args.logfile is None: logging.basicConfig(datefmt='%Y-%m-%d %H:%M:%S', format='%(asctime)s %(levelname)-8s %(message)s', level=loglevel) else: logging.basicConfig(datefmt='%Y-%m-%d %H:%M:%S', format='%(asctime)s %(levelname)-8s %(message)s', filename=args.logfile, level=loglevel) LOGGER = multiprocessing.log_to_stderr() LOGGER.info(args) # hbase_table_prefix = None if str.lower(args.hbase_table_prefix) == 'none' else args.hbase_table_prefix main(args.hbase_table, args.model_prefix, args.predictions_prefix, args.reviews_files, args.process_names, args.hdfs_base_url, args.categories)
for container in active_containers: logger.info("Container {0} running".format(container.name)) time.sleep(3) status, description = grp.check_health() if description: print "\n**** WEB APP HEALTH STATUS ****" print description print "*******************************\n" logs_stream_generators.update(grp.get_logs_stream()) if results.containers_logs: jobs = [] for container_name, log_stream_generator in logs_stream_generators.items( ): multiprocessing.log_to_stderr(logging.ERROR) p = multiprocessing.Process(name=container_name, target=print_log_stream, args=(log_stream_generator, )) p.daemon = True p.start() try: while (True): time.sleep(1) except KeyboardInterrupt: for p in jobs: p.terminate() p.join() logger.info("Bye Bye.. Have a nice day.") if results.containers_stats:
def enable_debug(): """ Enables the full debug, including for sub processes. """ logger = multiprocessing.log_to_stderr(logging.DEBUG) logger.setLevel(multiprocessing.SUBDEBUG)
FONT_SIZE = 32 FONT = ImageFont.truetype("NotoSansCJKsc-Regular.otf", size=FONT_SIZE) LOGGER = multiprocessing.get_logger() FOLDER = "char_img" CHUNKSIZE = 50 def gen_img(codepoint, font=FONT, folder=FOLDER): char = chr(int(codepoint, 16)) test_img = Image.new("L", (FONT_SIZE, 4 * FONT_SIZE), 0) out_img = Image.new("L", (FONT_SIZE, FONT_SIZE), 0) d = ImageDraw.Draw(test_img) d.text((0, 0), char, fill=255, font=font) bbox = test_img.getbbox() d = ImageDraw.Draw(out_img) y = FONT_SIZE // 2 - (bbox[1] + bbox[3]) // 2 if (bbox[3] - bbox[1]) > FONT_SIZE: LOGGER.warning(f"Character {char} at codepoint {codepoint} has height {bbox[3] - bbox[1]}") d.text((0, y), char, fill=255, font=font) out_img.save(f"{folder}/{codepoint}.png") if __name__ == "__main__": multiprocessing.log_to_stderr().setLevel(logging.WARNING) with open("TGSCC-Unicode.txt", "r") as f: codepoints = [line.split()[1][2:] for line in f.readlines()[2:]] pool = multiprocessing.Pool() results = pool.imap_unordered(gen_img, codepoints, chunksize=CHUNKSIZE) for _ in tqdm(results, total=len(codepoints)): pass pool.close()
cfg_from_list(['DATASET', args.dataset]) if args.exp is not None: cfg_from_list(['TEST.EXP_NAME', args.exp]) if args.diff_backprop is not None: cfg_from_list(['TRAIN.DIFF_BACKPROP', args.diff_backprop]) if args.stabilizer is not None: cfg_from_list(['TRAIN.STABILIZER', args.stabilizer]) if args.out_path is not None: cfg_from_list(['DIR.OUT_PATH', args.out_path]) if args.weights is not None: cfg_from_list([ 'CONST.WEIGHTS', args.weights, 'TRAIN.RESUME_TRAIN', True, 'TRAIN.INITIAL_ITERATION', int(args.init_iter) ]) print('Using config:') pprint.pprint(cfg) if not args.test: train_net() else: test_net() if __name__ == '__main__': mp.log_to_stderr() logger = mp.get_logger() logger.setLevel(logging.INFO) main()
################################################################################ import packages.buskill from packages.garden.navigationdrawer import NavigationDrawer from packages.garden.progressspinner import ProgressSpinner from buskill_version import BUSKILL_VERSION import os, sys, re, webbrowser import multiprocessing from multiprocessing import util import logging logger = logging.getLogger(__name__) util.get_logger().setLevel(util.DEBUG) multiprocessing.log_to_stderr().setLevel(logging.DEBUG) #from multiprocessing import get_context import kivy #kivy.require('1.0.6') # replace with your current kivy version ! from kivy.app import App from kivy.properties import ObjectProperty, StringProperty from kivy.clock import Clock from kivy.core.window import Window Window.size = (300, 500) from kivy.uix.widget import Widget from kivy.uix.label import Label from kivy.uix.button import Button
from __future__ import print_function from future.utils import iteritems from builtins import range, input import multiprocessing import random import sys import traceback import operator import logging import argparse from collections import namedtuple from rpn import RPN, RPNError, is_number from time import time logger = multiprocessing.log_to_stderr() logger.setLevel(logging.ERROR) class EquationError(Exception): pass class UnsolvableError(Exception): pass def manually(): number_of_variables = None while number_of_variables is None: number_of_variables = int(input("Number of variables: "))
""" import logging import multiprocessing import os def printer(item_, lock_): # Locking lock_.acquire() try: print('{} is printing: {}'.format(os.getpid(), item_)) finally: # Unlocking lock_.release() if __name__ == '__main__': # Lock object lock = multiprocessing.Lock() items = ['tango', 'foxtrot', 10] multiprocessing.log_to_stderr() # To redirect to stderr pipe logger = multiprocessing.get_logger() # Getting logger object logger.setLevel(logging.INFO) for item in items: p = multiprocessing.Process(target=printer, args=(item, lock)) p.start()
def __init__(self, blockchain, mempool): mp.log_to_stderr() mp_logger = mp.get_logger() mp_logger.setLevel(logging.DEBUG) self.blockchain = blockchain self.mempool = mempool
lock.release() def sub_500_lock(total, lock): for i in range(100): time.sleep(0.01) lock.acquire() total.value -= 5 lock.release() if __name__ == '__main__': total = Value('i', 500) lock = Lock() log_to_stderr() logger = get_logger() logger.setLevel(logging.INFO) add_proc = Process(target=add_500_lock, args=(total, lock)) sub_proc = Process(target=sub_500_lock, args=(total, lock)) add_proc.start() sub_proc.start() add_proc.join() sub_proc.join() print(total.value)
def main(): # This incantation is forced on us so the IDP won't print everything twice: import logging import multiprocessing as mp logger = mp.log_to_stderr() logger.setLevel(logging.INFO) # Set parameters for IDP (Image Data Pipeline) set_num_buffers = 3 image_height_pixels = 128 image_width_pixels = 380 # Set parameters for DAQ (analog out card) num_daq_channels = 3 daq_rate = 8e5 ############################################################## # Set exposure parameters for camera and laser illumination: # ############################################################## green_AOM_mV = [ 300, ] #calibrated green_powers = [ '1010mW', ] red_AOM_mV = [ 269, ] #calibrated red_powers = [ '0mW', ] angle_string = '14' # Set laser pulse duration VERY SHORT green_pulse_duration_pixels = 1 red_pulse_duration_pixels = 1 # Set green pulse train repetition time short enough to # thermally stabilize the sample green_rep_time_us = 600 green_rep_time_pixels = int(np.ceil( green_rep_time_us * 1e-6 * daq_rate)) # how many red laser shots in an exposure? pulses_per_exposure = 8 # you don't want red light leaking into next exposure so set this to # 1 if you're imaging 720 nm. # set to zero if you're looking for depletion, because you need # every green pulse matched with a red for that measurement less_red_pulses = 0 desired_effective_exposure_time_pixels = (green_rep_time_pixels * pulses_per_exposure) assert desired_effective_exposure_time_pixels > 0 #define red/green pulse delays red_start_pixel_array = np.array([-2, 0, 2]) num_delays = red_start_pixel_array.shape[0] print('Red/green delay (us) =', red_start_pixel_array / daq_rate * 1e6) # number of exposures should be the first dimension of the idp buffer num_delay_scan_repetitions = 1 num_exposures = num_delays * num_delay_scan_repetitions # actual roll time is 640 us, which should be a multiple of # green_rep_time_us, but may not always be # this only works for the current field of view height 128 pixels # 10 us per line, rolling is symmetrical around middle of chip rolling_time_us = 640 #experimentally determined for this field of view rolling_time_pixels = int(np.ceil( rolling_time_us * 1e-6 * daq_rate)) extra_time_after_roll_pixels = (green_rep_time_pixels - rolling_time_pixels % green_rep_time_pixels) effective_exposure_time_pixels = (extra_time_after_roll_pixels + desired_effective_exposure_time_pixels) # reminder: negative delay values (red before green) are only valid if the # camera roll finishes before the red pulse gets there assert extra_time_after_roll_pixels > -min(red_start_pixel_array) set_exposure_time_pixels = (rolling_time_pixels + effective_exposure_time_pixels) # set exposure time must be an integer multiple of green rep time assert (set_exposure_time_pixels % green_rep_time_pixels) == 0 set_exposure_time_us = int(np.ceil( set_exposure_time_pixels / daq_rate * 1e6)) # Initialize the IDP: idp = image_data_pipeline.Image_Data_Pipeline( num_buffers=set_num_buffers, buffer_shape=(num_exposures, image_height_pixels, image_width_pixels), camera_child_process=pco_edge_camera_child_process) assert idp.buffer_shape[0] == num_exposures # Initialize the DAQ: daq = ni.PCI_6733( num_channels=num_daq_channels, rate=daq_rate, verbose=True) assert daq.rate == daq_rate try: # Apply camera settings: idp.display.set_intensity_scaling('median_filter_autoscale') idp.apply_camera_settings( trigger='external_trigger', exposure_time_microseconds = set_exposure_time_us, region_of_interest ={'bottom': 1088, 'top': 961, 'left': 841, 'right': 1220}, preframes=0) # UNCOMMON COMMAND: the daq voltage string can get very long, so # Andy wrote a new part of pco.py that adjusts the set timeout # for waiting for the FIRST camera trigger (Oct 4, 2016) idp.camera.commands.send(('set_first_trigger_timeout_seconds', {'first_trigger_timeout_seconds': 3})) assert idp.camera.commands.recv() == 3 # clear command queue # Figure out some basic timing information: This is what the # camera thinks it's doing. Is it what we want it to do? exposure_time_us = idp.camera.get_setting('exposure_time_microseconds') print('I want exposure time to be (us)',set_exposure_time_us) print('Exposure time actually is (us)',exposure_time_us) assert exposure_time_us == set_exposure_time_us rolling_time_us = idp.camera.get_setting('rolling_time_microseconds') rolling_time_jitter_us = 15 #experimentally measured and also in spec rolling_time_us += rolling_time_jitter_us pulse_tail_us = 25 #experimentally measured response of buffer amp and AOM print("\nCamera exposure time:", exposure_time_us, "(us)\n") print("\nCamera rolling time:", rolling_time_us, "(us)\n") effective_exposure_us = exposure_time_us - rolling_time_us print("\nCamera effective exposure:", effective_exposure_us, "(us)\n") for [red_voltage_num, my_red_voltage_mV] in enumerate(red_AOM_mV): for [green_voltage_num, my_green_voltage_mV] in enumerate(green_AOM_mV): # Calculate DAQ voltages # Set voltages to play on analog out card green_voltage = my_green_voltage_mV/1000 red_voltage = my_red_voltage_mV/1000 trig_voltage = 3 # time between exposures must be greater than camera trigger # jitter and a multiple of the green rep time # trigger jitter is about 10 us time_between_exposures_pixels = 2 * green_rep_time_pixels camera_rep_time_pixels = (set_exposure_time_pixels + time_between_exposures_pixels) camera_rep_time_us = camera_rep_time_pixels / daq_rate * 1e6 voltages = np.zeros((camera_rep_time_pixels * num_exposures, num_daq_channels)) # green laser pulses on for the duration of the daq play green_chunk = np.zeros(green_rep_time_pixels) green_chunk[0:green_pulse_duration_pixels] = green_voltage voltages[:,1] = np.tile( green_chunk, int(voltages.shape[0]/green_rep_time_pixels)) # camera trigger duration should be 3us or greater trigger_duration_us = 3 trigger_duration_pixels = int(np.ceil( trigger_duration_us / 1e6 * daq_rate)) # loop used to define camera trigger and red laser pulse # voltages for which_exposure in range(num_exposures): cursor = which_exposure * camera_rep_time_pixels # Camera triggers: voltages[cursor:cursor + trigger_duration_pixels, 0] = ( trig_voltage) # Red laser pulses red_start_pixel = ( red_start_pixel_array[which_exposure % num_delays]) red_series_start = (cursor + rolling_time_pixels + extra_time_after_roll_pixels + red_start_pixel) red_chunk = np.zeros(green_rep_time_pixels) red_chunk[0:red_pulse_duration_pixels] = red_voltage red_exposure_array = np.tile(red_chunk, ( pulses_per_exposure - less_red_pulses)) voltages[red_series_start:(red_series_start + red_exposure_array.shape[0]), 2] = red_exposure_array # save voltages that will be sent to daq with open('voltages_green_' + green_powers[green_voltage_num] + '_red_' + red_powers[red_voltage_num] + '_depletion.pickle', 'wb') as f: pickle.dump(voltages, f) # Put it all together idp.load_permission_slips( num_slips=1, file_saving_info=[ {'filename': ( 'STE_depletion_angle_' + angle_string + '_green_' + green_powers[green_voltage_num] + '_red_' + red_powers[red_voltage_num] + '.tif'), 'channels': num_delays, 'slices': num_delay_scan_repetitions, }]) daq.play_voltages(voltages, block=True) finally: # Shut everything down. This can be important! daq.close() idp.close()
def main(): # send it all to stderr mp.log_to_stderr() # get access to a logger and set its logging level to INFO logger = mp.get_logger() logger.setLevel(logging.INFO) dataset = read_data(DATASET_PATH) # global train_x, test_x, train_y, test_y train_x, test_x, train_y, test_y = split_dataset(dataset, 0.25) # print("--- Testing Sequence DE ---") # start_time_seq = time.time() # result_seq = list(de_sequence(fobj, bounds=[(-100, 100)] * 6)) # print(result_seq[-1]) # print("") # print("--- %s seconds ---" % (time.time() - start_time_seq)) # # sleep(5) print("--- Tuning Random Forest with Parallel DE ---") start_time_rf_tuning_para = time_RF.time() # result_para = list(de_parallel(fobj, bounds=[(-100, 100)] * 6)) # print(result_para[-1]) # initialization bounds = [(10, 150), (1, 20), (2, 20), (2, 50), (0.01, 1), (1, 10)] mut = 0.8 crossp = 0.7 popsize = 60 its = 100 dimensions = len(bounds) pop = np.random.rand(popsize, dimensions) # pdb.set_trace() min_b, max_b = np.asarray(bounds).T diff = np.fabs(min_b - max_b) pop_denorm = min_b + pop * diff # convert from float to integer pop_denorm_convert = pop_denorm.tolist() result_list = [] temp_list = [] for index in pop_denorm_convert: temp_list.append(np.int_(np.round_(index[0]))) temp_list.append(np.int_(np.round_(index[1]))) temp_list.append(np.int_(np.round_(index[2]))) temp_list.append(np.int_(np.round_(index[3]))) temp_list.append(float('%.2f' % index[4])) temp_list.append(np.int(np.round_(index[5]))) result_list.append(temp_list) temp_list = [] fitness = np.asarray([ rf_tuning(index[0], index[1], index[2], index[3], index[4], index[5], train_x, test_x, train_y, test_y) for index in result_list ]) best_idx = np.argmax(fitness) best = pop_denorm[best_idx] print("Dimension:", dimensions) print("pop:", pop) print("min_b:", min_b) print("max_b:", max_b) print("diff:", diff) print("pop_denorm:", pop_denorm) print("fitness:", fitness) print("best_idx:", best_idx) print("best:", best) lock = mp.Lock() # execute loops in each process processes = [] for x in range(mp.cpu_count()): processes.append( mp.Process(target=de_innerloop, args=(output, its, popsize, pop, mut, dimensions, crossp, min_b, diff, lock, fitness, best_idx, best, train_x, test_x, train_y, test_y))) # Run processes for p in processes: p.start() # Exit the completed processes # Without join() function call, process will remain idle and won’t terminate for p in processes: p.join() # Get process results from the output queue results = [output.get() for p in processes] print(results) print("") print("--- %s seconds ---" % (time_RF.time() - start_time_rf_tuning_para)) print("")
def realtime(args): """ Read and apply object detection to input real time stream (webcam) """ # If display is off while no number of frames limit has been define: set diplay to on if ((not args["display"]) & (args["num_frames"] < 0)): print("\nSet display to on\n") args["display"] = 1 # Set the multiprocessing logger to debug if required if args["logger_debug"]: logger = multiprocessing.log_to_stderr() logger.setLevel(multiprocessing.SUBDEBUG) # Multiprocessing: Init input and output Queue and pool of workers input_q = Queue(maxsize=args["queue_size"]) output_q = Queue(maxsize=args["queue_size"]) crop_q = Queue(maxsize=args["queue_size"]) pool = Pool(1, worker, (input_q, output_q, crop_q)) pool2 = Pool(1, crop_worker, (crop_q, )) # created a threaded video stream and start the FPS counter vs = WebcamVideoStream(src=args["input_device"]).start() fps = FPS().start() # Define the output codec and create VideoWriter object if args["output"]: fourcc = cv2.VideoWriter_fourcc(*'XVID') out = cv2.VideoWriter('outputs/{}.avi'.format(args["output_name"]), fourcc, vs.getFPS() / args["num_workers"], (vs.getWidth(), vs.getHeight())) # Start reading and treating the video stream if args["display"] > 0: print() print( "=====================================================================" ) print( "Starting video acquisition. Press 'q' (on the video windows) to stop." ) print( "=====================================================================" ) print() countFrame = 0 while True: # Capture frame-by-frame ret, frame = vs.read() countFrame = countFrame + 1 if ret: input_q.put(frame) output_rgb = cv2.cvtColor(output_q.get(), cv2.COLOR_RGB2BGR) #print("size of crop_q:", crop_q.qsize()) # write the frame if args["output"]: out.write(output_rgb) # Display the resulting frame if args["display"]: ## full screen if args["full_screen"]: cv2.namedWindow("frame", cv2.WND_PROP_FULLSCREEN) cv2.setWindowProperty("frame", cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN) cv2.imshow("frame", output_rgb) fps.update() elif countFrame >= args["num_frames"]: break else: break k = cv2.waitKey(1) if k == ord('q'): #path = "/home/mohak/Desktop/image/" #test = os.listdir(path) #for item in test: #if item.endswith(".jpg"): #os.remove(os.path.join(path, item)) #else: #pass break cv2.destroyAllWindows() # When everything done, release the capture fps.stop() pool.terminate() pool2.terminate() vs.stop() if args["output"]: out.release() cv2.destroyAllWindows()
overwrite_scores = parser.parse_args().overwrite_scores # This will re-run plots parser.add_argument('-oo','--overwrite_output',action='store_true',default=False) overwrite_output = parser.parse_args().overwrite_output parser.add_argument('-T','--tests',action='store_true',default=False) basic_tests = parser.parse_args().tests parser.add_argument('-C','--check_only',action='store_true',default=False) check_only = parser.parse_args().check_only parser.add_argument('-S','--subcpus',dest='subcpus',default=1,type=int) subcpus = parser.parse_args().subcpus ##### ERROR LOGGING ##### mp_log = multiprocessing.log_to_stderr() mp_log.setLevel(logging.INFO) ### SETTINGS ### # Folder key for final experimental wrfouts key_wrf = 'ForReal_nco' # Folder key for post-processed fields (objects, lat/lon, etc) key_pp = 'Xmas' # Folder key for scores (FSS, etc) key_scores = 'Xmas' # Folder key for plots key_plot = 'Xmas' ensroot = '/scratch/john.lawson/WRF/VSE_reso/{}'.format(key_wrf) reproj_obs_root = '/work/john.lawson/VSE_reso/reproj_obs/{}'.format(key_pp)