VERSION_NONE = -1 # Data directories ROOT_DIR = '/mnt' DEST_DIR = ROOT_DIR + '/idseq/data' # Generated data go here REF_DIR = ROOT_DIR + '/idseq/ref' # Reference genome / ref databases go here OUTPUT_VERSIONS = [] print_lock = multiprocessing.RLock() # Peak network and storage perf for a typical small instance is saturated by # just a few concurrent streams. MAX_CONCURRENT_COPY_OPERATIONS = 8 iostream = multiprocessing.Semaphore(MAX_CONCURRENT_COPY_OPERATIONS) # Make a second semaphore for uploads to reserve some capacity for downloads. MAX_CONCURRENT_UPLOAD_OPERATIONS = 4 iostream_uploads = multiprocessing.Semaphore(MAX_CONCURRENT_UPLOAD_OPERATIONS) # Definitions for integration with web app TAX_LEVEL_SPECIES = 1 TAX_LEVEL_GENUS = 2 TAX_LEVEL_FAMILY = 3 NULL_SPECIES_ID = -100 NULL_GENUS_ID = -200 NULL_FAMILY_ID = -300 NULL_LINEAGE = (str(NULL_SPECIES_ID), str(NULL_GENUS_ID), str(NULL_FAMILY_ID)) JOB_SUCCEEDED = "succeeded" JOB_FAILED = "failed"
def makeInactive(self, name): with self.lock: self.active.remove(name) def __str__(self): with self.lock: return str(self.active) def worker(s, pool): name = multiprocessing.current_process().name with s: pool.makeActive(name) print 'Now running: %s' % str(pool) time.sleep(random.randint(5, 10)) pool.makeInactive(name) if __name__ == '__main__': pool = ActivePool() s = multiprocessing.Semaphore(3) jobs = [ multiprocessing.Process(target=worker, args=(s, pool), name=str(i)) for i in range(10) ] for j in jobs: j.start() for j in jobs: j.join() print 'Now running: %s' % str(pool)
import requests import math from datetime import datetime import pathlib import timeit import time import sys import multiprocessing import utils if __name__ == '__main__': procs = [] num_process = min(multiprocessing.cpu_count() - 1, 8) sema = multiprocessing.Semaphore(num_process) exclude_list_endswith = [ '스팩', '리츠', '증권', '은행', '홀딩스', '지주', '건설', '화재', '종금', '캐피탈', '투자' ] exclude_list_exact = [ '한국테크놀로지그룹', '인터파크', '아세아', 'CJ', 'LG', '경동인베스트', '엘브이엠씨', '대웅', '아모레퍼시픽그룹', '지투알', 'BGF', '코오롱', 'GS', 'SK', '한화', '현대모비스', 'DL', 'HDC', '효성', '동원개발', '금호산업', '휴온스글로벌' ] #holdings exclude_list_contain = ['스팩'] required_ror_percent = utils.get_required_rate_of_return() path = str(pathlib.Path().absolute()) + '\\'
set_parts[site][content].add(taxon) # Remove ignored characters from set_parts for c in ignored_chars: removed = set_parts[site].pop(c, None) if set_parts[site] == {}: print("Error: Empty character alignment at position " + str(site) + ". TIGER rates not calculated.", file=sys.stderr) exit(1) # Steps 2 and 3: calculate partition agreements and TIGER rates concurrently pool = ActivePool() pool.data.update(char_dict) s = multiprocessing.Semaphore(args.n_processes) jobs = [ multiprocessing.Process(target=calculate_tiger_rates, name=str(k), args=(k, pool)) for k in split_list(list(char_dict.keys()), args.n_processes) ] for j in jobs: j.start() for j in jobs: j.join() for k in sorted(pool.result.keys()): line = ""
def CreateModelDiagrams(M, options): # This function is a "master", calling many other functions based on command # line input. Other than code cleanliness, there is no reason that the # logic couldn't be in main() # if the user has listed more than one dot_dat, arbitrarily choose the first # as the name of this run. datname = os.path.basename(options.dot_dat[0])[:-4] images_dir = "images_" + datname if os.path.exists(images_dir): rmtree(images_dir) os.mkdir(images_dir) os.chdir(images_dir) os.makedirs('commodities') os.makedirs('processes') os.makedirs('results') ############################################## #MAIN MODEL AND RESULTS AND EVERYTHING ELSE kwargs = dict( model=M, images_dir='images_%s' % datname, image_format=options.graph_format.lower(), options=options, tech_color='darkseagreen', commodity_color='lightsteelblue', unused_color='powderblue', arrowheadout_color='forestgreen', arrowheadin_color='firebrick', usedfont_color='black', unusedfont_color='chocolate', menu_color='hotpink', home_color='gray75', #MODELDETAILED, md_tech_color='hotpink', #SUBGRAPHS (option 1), sb_incom_color='lightsteelblue', sb_outcom_color='lawngreen', sb_vpbackg_color='lightgrey', sb_vp_color='white', sb_arrow_color='forestgreen', #SUBGRAPH 1 ARROW COLORS # feel free to add more colors here color_list=('red', 'orange', 'gold', 'green', 'blue', 'purple', 'hotpink', 'cyan', 'burlywood', 'coral', 'limegreen', 'black', 'brown'), ) #################################### # Do all necessary work in parallel, taking advantage of whatever cores # are available to the computer on which this code is run. To add a # function to the pool, ensure that the function is in the 'gvizFunctions' # tuple below, and that the function has all it needs to work passed in # via the 'kwargs' dict above. gvizFunctions = ( CreateCompleteEnergySystemDiagram, CreateCommodityPartialGraphs, CreateProcessPartialGraphs, CreateMainModelDiagram, CreateTechResultsDiagrams, CreateCommodityPartialResults, CreateMainResultsDiagram, CreatePartialSegmentsDiagram, ) if 'win' in sys.platform: msg = ('\n\nRunning in Windows ... Temoa is currently unable to use ' 'multiple processes. Generating graphs will take a bit longer.' '\n') SE.write(msg) for func in gvizFunctions: func(**kwargs) else: sem = MP.Semaphore(MP.cpu_count()) def do_work(func): sem.acquire() func(**kwargs) sem.release() processes = [ MP.Process(target=do_work, args=(func, )) for func in gvizFunctions ] for p in processes: p.start() for p in processes: p.join() os.chdir('..')
def _build_parallel_ctrl(self, n_worker): super()._build_parallel_ctrl(n_worker) self.sync.obs_ready = [mp.Semaphore(0) for _ in range(n_worker)] self.sync.act_ready = [mp.Semaphore(0) for _ in range(n_worker)]
def main(): global flagTerminate global beanstalk global logger logger = setupLogger() # define a Handler which writes INFO messages or higher to the sys.stderr console = logging.StreamHandler() console.setLevel(logging.INFO) # set a format which is simpler for console use formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s') # tell the handler to use this format console.setFormatter(formatter) # add the handler to the root logger logging.getLogger().addHandler(console) random.seed() pool = ActivePool() assignments = pool.mgr.dict() count = pool.mgr.Value('i', 0) ADBFAILCOUNT = pool.mgr.Value('i', 0) VMs = pool.mgr.dict() s = multiprocessing.Semaphore(NUMACTIVEWORKERS) q = multiprocessing.Queue() r = multiprocessing.Queue() results = [] al = multiprocessing.RLock() vml = multiprocessing.RLock() cl = multiprocessing.RLock() get_VMs(VMs) list_VMs(VMs) #make sure any pcaps are killed and we are starting with a clean filesystem for i in range(30): #q.put("item " + str(i) + " content") q.put("live" + str(i)) #adb ports are 5554 - 5584, only even port = ADBPORT jobs = [ multiprocessing.Process(target=worker, name=str(i), args=(s, pool, q, "hi", VMs, vml, str(port + (i * 2)), count, al, r, ADBFAILCOUNT)) for i in range(NUMWORKERS) ] for j in jobs: logger.info("starting worker %s" % j.name) j.start() # for j in jobs: # j.join() # logger.info( 'Now running: %s' % str(pool)) #make the connection after the child spawning, each child makes thier own connections rebeanstalk() while True: logger.info("==============================") logger.info("processed %d samples so far" % count.value) try: #not great, but the parent is the only consumer of r items thing = r.get(block=False) results.append(thing) except: logger.warning("results queue exception") if len(results) > 0: logger.info("average sample time: %s" % str(sum(results) / float(len(results)))) try: if BSMSBOXQ not in beanstalk.tubes(): logger.info("there are no samples in the beanstalk") else: if beanstalk.peek_ready() is not None: logger.info( str( beanstalk.stats_tube(BSMSBOXQ) ['current-jobs-ready']) + " samples ready to process") else: logger.info("0 samples ready to process") if beanstalk.peek_reserved() is not None: logger.info( str( beanstalk.stats_tube(BSMSBOXQ) ['current-jobs-reserved']) + " samples are processing") else: logger.info("0 samples are processing") if beanstalk.peek_delayed() is not None: logger.info( str( beanstalk.stats_tube(BSMSBOXQ) ['current-jobs-delayed']) + " samples are delayed") else: logger.info("0 samples are delayed") except KeyError: logger.warning("beanstalk keyerror") except: logger.warning("beanstalk general error") logger.info("about " + str(q.qsize()) + " current q items") logger.info("adb has failed " + str(ADBFAILCOUNT.value) + " times since last kick") logger.debug("should be " + str(NUMWORKERS) + " processes:") walive = 0 wdead = 0 for j in jobs: if j.is_alive(): logger.debug(" " + str(j.pid)) walive += 1 else: logger.debug(" dead") wdead += 1 #this will kill the controller, leaving the children pipes wanting more (you will get sigpipe errors) #this is pretty harsh and shouldn't probably be done in production #TODO #sys.exit(1) logger.info("%s alive; %s dead" % (walive, wdead)) updateStatus(NUMWORKERS, walive, wdead) list_VMs(VMs) time.sleep(VM_POLL_REST + 10) #blocks until all items in q are processed #safe to exit #should check for sigint #if q.qsize() == 0: # for i in jobs: # q.put("die") # q.close() sys.exit(0) signal.signal(signal.SIGINT, sigint_handler) syslog.syslog('Startup - Entering main polling loop...') while True: get_VMs(VMs) list_VMs(VMs) #make sure any pcaps are killed and we are starting with a clean filesystem #pcap_terminate(m) #job_cleanup(m) #check for a job in the beanstalk Q if not beanstalk: error_log('Re-establishing connection to beanstalkd') beanstalk = beanstalkc.Connection(host=BSHOST, port=BSPORT) beanstalk.watch(BSMSBOXQ) beanstalk.use(BSMSBOXQ) beanstalk.ignore('default') #we want to reserve, not peak_ready because if we can't process, we want to release with a delay later beanstalkjob = beanstalk.reserve() #if there was a job and we got it then queue up analysis if beanstalkjob: try: job = eval(beanstalkjob.body) except: error_log('ERROR: could not eval job - invalid job') beanstalkjob.delete() return False #is there a VM compatible with this sample that is ready to be used? m = findCompatibleVM(job, VMs, vml) if m is not None: logger.info("yay using VM " + str(m)) #process sample sys.exit(0) #delete from queue if successfull #beanstalkjob.delete() else: #no VM found, put it back in the queue (nobody will process for 60 seconds beanstalkjob.release(delay=60) logger.info("requeuing job requiring a target version " + job['target']) # # #break ot of the loop and exit if the assignments are clear and the terminate flag is set if flagTerminate and len(assignments) == 0: syslog.syslog('All assigned jobs are completed (EXITING)...') break #put a pause in the polling cycle...set to 0 for no pause time.sleep(VM_POLL_REST) sys.exit(0) syslog.syslog('Shutdown - Leaving main polling loop...') return 0
sem1.release() def cons2(Q2, sem1, sem2): n = 0 while n != -1: sem1.acquire() n = Q2.get() print(n) sem2.release() Q1 = mp.Queue() Q2 = mp.Queue() sem1 = mp.Semaphore(1) sem2 = mp.Semaphore(0) P1 = mp.Process(target=prod1, args=(Q1, )) P2 = mp.Process(target=prod2, args=(Q2, )) C1 = mp.Process(target=cons1, args=(Q1, sem1, sem2)) C2 = mp.Process(target=cons2, args=(Q2, sem1, sem2)) P1.start() P2.start() C1.start() C2.start()
def grow(self, nsims=10, gamma=0.9, parallelize=False): """ Grow Tree by one node gamma between 0.5 and 1 """ ### SELECTION start_node = self.root_id # Sample actions of other robots # NOTE: Sampling done at the begining for dependency graph reasons state = self._get_system_state(start_node) # Propagate down the tree # check how _select handles mu, N = 0 while len(self._childNodes(start_node)) > 0: start_node = self._select(self._childNodes(start_node)) ### EXPANSION # check if _expansion changes start_node to the node after jumping self._expansion(start_node) # print("Grow!") ### SIMULATION avg_reward = 0 best_reward = float("-Inf") best_rollout = None # Parallization if parallelize: processes = [] pipe_list = [] mp_context = multiprocessing.get_context("fork") sema = multiprocessing.Semaphore(12) for i in range(nsims): recv_end, send_end = mp_context.Pipe(False) p = mp_context.Process(target=self._simulate, args=(start_node, state, send_end, sema)) processes.append(p) pipe_list.append(recv_end) p.start() for i in range(nsims): if parallelize: recvd = pipe_list[i].recv() processes[i].join() else: recvd = self._simulate(start_node, state) rew = recvd["reward"] temp_state = recvd["temp_state"] avg_reward += rew # new_return = 0 # for j in reversed(range(rew.len())): # new_return = rew[j] + gamma * new_return # avg_reward += new_return # if best reward so far, store the rollout in the new node if rew > best_reward: best_reward = rew best_rollout = copy(temp_state) avg_reward = avg_reward / nsims self.graph.nodes[start_node]["mu"] = avg_reward self.graph.nodes[start_node]["best_reward"] = best_reward self.graph.nodes[start_node]["N"] = 1 self.graph.nodes[start_node]["best_rollout"] = deepcopy(best_rollout) ### BACKPROPOGATION while start_node != 1: # while not root node start_node = self._parent(start_node) self.graph.nodes[start_node]["mu"] = \ (gamma * self.graph.nodes[start_node]["mu"] * \ self.graph.nodes[start_node]["N"] + avg_reward) \ / (self.graph.nodes[start_node]["N"] + 1) self.graph.nodes[start_node]["N"] = \ gamma * self.graph.nodes[start_node]["N"] + 1 if best_reward > self.graph.nodes[start_node]["best_reward"]: self.graph.nodes[start_node]["best_reward"] = best_reward self.graph.nodes[start_node]["best_rollout"] = deepcopy( best_rollout) self._update_distribution() return avg_reward
import time import select from qalib.qabase.exceptions import ConnectionError logging.getLogger('paramiko').setLevel(logging.WARNING) log = logging.getLogger(__name__) if not log.handlers: log.addHandler(logging.NullHandler()) DEFAULT_SHELL_MATCH_TIMEOUT = 60 INTERACTIVE_SHELL_TIMEOUT = 60 OPEN_SESSION_TIMEOUT = 10 sema = multiprocessing.Semaphore(9) def _open_session_with_timeout(transport): """ Helper function which encapsulates a difference between older and newer Paramiko versions. For a given paramiko.Transport object, returns a session opened from the tranport, with a timeout of OPEN_SESSION_TIMEOUT seconds. New versions have a timeout parameter when opening a session, so it's easy. But on old versions, the timeout is very long and not configurable, so it can take us a long time to fail over if we catch the node just as it goes down. So we use a background thread to emulate this timeout.
print("Je suis le process E") s2.acquire() s2.acquire() print("Je suis le process E et je vais generer 1 jeton dans 5 sec") time.sleep(5) s3.release() sys.exit(0) def processF(s3): print("Je suis le process F") s3.acquire() s3.acquire() print("Je suis le process F et je suis DEBLOQUE") sys.exit(0) sem = mp.Semaphore(0) sem2 = mp.Semaphore(0) sem3 = mp.Semaphore(0) pA = mp.Process(target = processA, args =(sem,)) pB = mp.Process(target = processB, args =(sem,sem2,)) pC = mp.Process(target = processC, args =(sem,sem2,)) pD = mp.Process(target = processD, args =(sem,sem3,)) pE = mp.Process(target = processE, args =(sem2,sem3,)) pF = mp.Process(target = processF, args =(sem3,)) pA.start() pB.start() pC.start() pD.start() pE.start() pF.start() pA.join()
multiprocessing.Process.__init__(self) self.loop = loop self.lock = lock def run(self): for count in range(self.loop): time.sleep(0.1) self.lock.acquire() print 'Pid: ' + str(self.pid) + ' LoopCount: ' + str(count) self.lock.release() ############################# ###Semaphore####### buffer1 = multiprocessing.Queue(10) empty = multiprocessing.Semaphore(2) full = multiprocessing.Semaphore(0) lock = multiprocessing.Lock() class Consumer(multiprocessing.Process): """docstring for Consumer""" def __init__(self, pipe): multiprocessing.Process.__init__(self) self.pipe = pipe def run(self): self.pipe.send('Consumer Words') print 'Consumer Received:', self.pipe.recv() '''global buffer1, empty, full, lock while True:
def __init__(self): self.process = mp.Process self.mgr = mp.Manager() self.sem = mp.Semaphore(mp.cpu_count()) self.result = self.mgr.dict() #result dict self.ppipe, self.cpipe = mp.Pipe() #parent pipe, #child pipe
import random import multiprocessing as mp from multiprocessing import Process # from multiprocessing import Queue # ne fonctionne pas en python 3 import queue permission_de_prendre_un_cookie = mp.Semaphore(1) def put_random_int(q: queue): print("Start put_random_int") nombre_production = 10 cpt = 0 while cpt < nombre_production: q.put(random.randint(0, 20)) cpt = cpt + 1 print("End put_random_int") def get_random_int(q: queue): print("Start get_random_int") permission_de_prendre_un_cookie.acquire() cookie = q.get() print(f"j'ai pris ce cookie: {cookie}") permission_de_prendre_un_cookie.release() print("End get_random_int") if __name__ == "__main__": print("Start main") q1 = queue.Queue()
# -*- coding: utf-8 -*- """ Created on Fri Jun 8 08:37:18 2018 @author: antoine.bralet """ import multiprocessing as mp variable_partagee = mp.Value("i", 0) # ce sera un entier initialisé à0 verrou = mp.Semaphore() # Val init=1 def count2_SC_sem(nb_iterations): """ Chacun écrit àson rythme (non protégée)""" global variable_partagee for i in range(nb_iterations): verrou.acquire() variable_partagee.value += 1 verrou.release() def test_SC_protege_par_Sem(): # if __name__ == ’__main__’ : nb_iterations = 5000 # La variable partagée : placée hors cette fonction (sinon, la passr en param) # variable_partagee = mp.Value(’i’,0) # ce sera un entier initialisé à0 print("la valeur de variable_partagee AVANT les incrémentations : ", variable_partagee.value) # On crée 2 process pid1 = mp.Process(target=count2_SC_sem, args=(nb_iterations, )) pid1.start()
def test5(): s = multiprocessing.Semaphore(2) for i in range(5): p = multiprocessing.Process(target=worker, args=(s, i * 2)) p.start()
os.makedirs(output_folder) mxd_paths = make_file_paths_list(input_folder, 'mxd', filename_pattern) if keyword_filters: mxd_paths = [ path for path in mxd_paths if any(kw in path for kw in keyword_filters if kw) ] logging.info(' Filtered files: {0}'.format(len(mxd_paths))) print total = len(mxd_paths) jobs = [] sema = multiprocessing.Semaphore(cpu_limit) logging.info('\n' + 'Exporting' + '\n') start_time = datetime.datetime.now() for i, path in enumerate(mxd_paths): mxd_basename = os.path.basename(path) output = os.path.join( output_folder, mxd_basename.replace('mxd', EXPORT_FUNCTIONS[output_type][1])) if not overwrite: if os.path.exists(output): logging.warning('[ EXISTS ] {0} ({1} z {2})'.format( mxd_basename, i + 1, total))
except Exception as e: print(e) print('------------response error') r.put((-1, name)) semaphore.release() if __name__ == '__main__': getRank() st = time() iq = mp.Queue() oq = mp.Queue() mpnum = mp.Semaphore(pnum) keyword = search pid = 0 f = open('img/log.txt', 'w') f.close() keyword = search print(keyword) for key in keyword: iq.put(key) procs = [] l = len(keyword)
semE.acquire() print("fin processus E") semF.release() def F(semE): print("F: attend E et D") semF.acquire() print("F:1 jeton") semF.acquire() print("fin processus F") semB = mp.Semaphore(0) semC = mp.Semaphore(0) semD = mp.Semaphore(0) semE = mp.Semaphore(0) semF = mp.Semaphore(0) ProcessB = mp.Process(target=B, args=(semB, semE)) ProcessC = mp.Process(target=C, args=(semC, semE)) ProcessD = mp.Process(target=D, args=(semD, semF)) ProcessE = mp.Process(target=E, args=(semE, semF)) ProcessF = mp.Process(target=F, args=(semE, )) ProcessB.start() ProcessC.start() ProcessD.start() ProcessE.start()
def fastload(fname, firstn): t0 = time.time() f = h5py.File(fname, 'r') assert ('X' in f.keys() and 'var' in f.keys() and 'obs' in f.keys()) # get obs dataframe rows = f['obs'][list(f['obs'].keys())[0]].size print("File has", rows, "rows") if firstn > 1 and firstn < rows: rows = firstn print("Loading", rows, "rows") if '_index' in f['obs'].keys(): dfobsind = pd.Series(f['obs']['_index'].asstr()[0:rows]) dfobs = pd.DataFrame(index=dfobsind) else: dfobs = pd.DataFrame() for k in f['obs'].keys(): if k == '_index': continue dfobs[k] = f['obs'][k].asstr()[...] # get var dataframe if '_index' in f['var'].keys(): dfvarind = pd.Series(f['var']['_index'].asstr()[...]) dfvar = pd.DataFrame(index=dfvarind) else: dfvar = pd.DataFrame() for k in f['var'].keys(): if k == '_index': continue dfvar[k] = f['var'][k].asstr()[...] print("Pandas load done", time.time() - t0) # load index pointers, prepare shared arrays indptr = f['X']['indptr'][0:rows + 1] datalen = int(indptr[-1]) if use_fastload == 1: data = f['X']['data'][0:datalen] indices = f['X']['indices'][0:datalen] if use_fastload == 2: f.close() dataArray = mp.Array('f', datalen, lock=False) # should be in shared memory indicesArray = mp.Array(indices_shm_type, datalen, lock=False) # should be in shared memory print("created shared memory buffers", time.time() - t0) #k = mkl.get_max_threads() k = numba.get_num_threads() procs = [ mp.Process(target=_load_helper, args=(fname, i * datalen // k, (i + 1) * datalen // k, dataArray, indicesArray)) for i in range(k) ] for p in procs: p.start() for p in procs: p.join() print("multiproc load done", time.time() - t0) data = np.frombuffer(dataArray, dtype=np.float32) indices = np.frombuffer(indicesArray, dtype=indices_type) if use_fastload == 3: f.close() #k = mkl.get_max_threads() k = numba.get_num_threads() dataArray = mp.Array('f', k * 1024 * 1024, lock=False) # should be in shared memory indicesArray = mp.Array(indices_shm_type, k * 1024 * 1024, lock=False) # should be in shared memory startsArray = mp.Array('l', k, lock=False) # start index of data read endsArray = mp.Array( 'l', k, lock=False) # end index (noninclusive) of data read barrier = mp.Barrier(k + 1) # k workers plus this process dataA = np.frombuffer(dataArray, dtype=np.float32) indicesA = np.frombuffer(indicesArray, dtype=indices_type) startsA = np.frombuffer(startsArray, dtype=np.int64) endsA = np.frombuffer(endsArray, dtype=np.int64) data = np.empty(datalen, dtype=np.float32) indices = np.empty(datalen, dtype=indices_type) print("created shared memory buffers", time.time() - t0) procs = [ mp.Process(target=_load_helper2, args=(fname, i, k, datalen, barrier, dataArray, indicesArray, startsArray, endsArray)) for i in range(k) ] for p in procs: p.start() for _ in range(datalen // (k * 1024 * 1024) + 1): barrier.wait() # wait for data loads _fast_copy(data, dataA, startsA, endsA, k) _fast_copy(indices, indicesA, startsA, endsA, k) barrier.wait() # finish data copy for p in procs: p.join() print("multiproc load done", time.time() - t0) if use_fastload == 4 or use_fastload == 5: f.close() #k = mkl.get_max_threads() k = numba.get_num_threads() dataArray = mp.Array('f', k * 1024 * 1024, lock=False) # should be in shared memory indicesArray = mp.Array(indices_shm_type, k * 1024 * 1024, lock=False) # should be in shared memory startsArray = mp.Array('l', k, lock=False) # start index of data read endsArray = mp.Array( 'l', k, lock=False) # end index (noninclusive) of data read global semDataLoaded global semDataCopied semDataLoaded = [mp.Semaphore(0) for _ in range(k)] semDataCopied = [mp.Semaphore(0) for _ in range(k)] dataA = np.frombuffer(dataArray, dtype=np.float32) indicesA = np.frombuffer(indicesArray, dtype=indices_type) startsA = np.frombuffer(startsArray, dtype=np.int64) endsA = np.frombuffer(endsArray, dtype=np.int64) data = np.empty(datalen, dtype=np.float32) indices = np.empty(datalen, dtype=indices_type) print("created shared memory buffers", time.time() - t0) procs = [ mp.Process(target=_load_helper3, args=(fname, i, k, datalen, dataArray, indicesArray, startsArray, endsArray)) for i in range(k) ] for p in procs: p.start() if use_fastload == 4: threads = [ threading.Thread(target=_copy_thread, args=(data, dataA, indices, indicesA, startsA, endsA, i, datalen // (k * 1024 * 1024) + 1)) for i in range(k) ] for t in threads: t.start() else: _fast_copy3(data, dataA, indices, indicesA, startsA, endsA, k, datalen // (k * 1024 * 1024) + 1) for p in procs: p.join() if use_fastload == 4: for t in threads: t.join() print("multiproc load done", time.time() - t0) #X = scipy.sparse.csr_matrix((data,indices,indptr)) #This is too slow X = scipy.sparse.csr_matrix((0, 0)) #X = scipy.sparse.csr_matrix((rows,dfvar.shape[0])) X.data = data X.indices = indices #X.data = np.empty(data.shape,dtype=data.dtype) #X.data[:] = data #X.indices = np.empty(indices.shape,dtype=indices.dtype) #X.indices[:] = indices X.indptr = indptr X._shape = ((rows, dfvar.shape[0])) print("Created scipy sparse array", time.time() - t0) # create AnnData adata = anndata.AnnData(X, dfobs, dfvar) print("anndata created", time.time() - t0) return adata
if len(sys.argv) != 4: raise ValueError( '\n\nSyntax must be:\n\t python multiprocexec -n N_proc sim_class=module.class\n\n' ) if '-n' not in sys.argv[1] or 'sim_class' not in sys.argv[3]: raise ValueError( '\n\nSyntax must be:\n python multiprocexec -n N_proc sim_class=module.class\n\n' ) N_proc = int(sys.argv[2]) sim_module_string = sys.argv[3].split('=')[-1] queue_list = [mp.Queue() for _ in xrange(N_proc)] mutex = mp.Semaphore(1) barrier = mp.Semaphore(0) turnstile = mp.Semaphore(0) turnstile2 = mp.Semaphore(1) cnt = mp.Value('i', 0) proc_list = [] for pid in xrange(N_proc): proc_list.append( mp.Process(target=todo, args=(sim_module_string, pid, N_proc, queue_list, mutex, barrier, turnstile, turnstile2, cnt))) for p in proc_list: p.start() for p in proc_list: p.join()
def sem_test(): s = multiprocessing.Semaphore(1) for i in range(5): p = multiprocessing.Process(target=sem_worker, args=(s, i * 2)) p.start()
import sys import os import requests import time try: from urllib.parse import urlparse except ImportError: from urlparse import urlparse from nhentai.logger import logger from nhentai.parser import request from nhentai.utils import Singleton requests.packages.urllib3.disable_warnings() semaphore = multiprocessing.Semaphore(1) class NHentaiImageNotExistException(Exception): pass class Downloader(Singleton): def __init__(self, path='', size=5, timeout=30, delay=0): self.size = size self.path = str(path) self.timeout = timeout self.delay = delay def download_(self, url, folder='', filename='', retried=0): if self.delay:
def predict( data=None, predicted_column: None | str | int = None, config: predictit.configuration.Config | dict | None = None, **kwargs, ) -> predictit._result_classes.Result: """Make predictions mostly on time-series data. Data input and other config options can be set up in configuration.py or overwritten on the fly. Setup can be also done as function input arguments or as command line arguments (it will overwrite config values). There are working examples in main readme and also in test_it module. Function can be configured from with config from configuration, with command line arguments as wel as with function parameters. There are only two possible positional parameters - `data` and `predicted_column`. Rest of parameters must be named parameters. Params are not documented here, because all config params works here in function passed as kwargs. Args: data (np.ndarray, pd.DataFrame, str): Time series. Can be 2-D - more columns. Can be numpy array, DataFrame, path to file or url. Examples: "/home/user/my.json", or "https://yoururl/your.csv" or np.random.randn(100, 2). predicted_column (None | str | int, optional): Index of predicted column or it's name (dataframe). If list with more values only the first one will be evaluated (use predict_multiple_columns function if you need that. Default to None. config (predictit.configuration.Config | dict | None, optional): Settings as Config instance or dictionary. Check class for what you can use. If None, then default config will be used. Defaults to None. **kwargs (dict, optional): There is much more parameters' of predict function. Check configuration.py for parameters details. Returns: Depend on 'return_type' config value - return best prediction {np.ndarray}, all models results {np.ndarray}, detailed results{dict} or interactive plot or print tables of results """ from mypythontools.plots import plot if config is None or isinstance(config, dict): update_config = config config = config_default config = config.copy() if update_config: config.update(update_config) elif isinstance(config, predictit.configuration.Config): config = config.copy() if config.use_config_preset and config.use_config_preset != "none": updated_config = config.presets[config.use_config_preset] config.update(updated_config) # Edit configuration.py default values with arguments values if exist if data is not None: config.data = data if predicted_column is not None: config.predicted_column = predicted_column config.update(kwargs) predictit._helpers.logger_init_from_config(config.output.logger_subconfig) # Do not repeat actually mean evaluate once if not config.repeatit: config.repeatit = 1 _GUI = GLOBAL_VARS.GUI # Add everything printed + warnings to variable to be able to print in GUI if _GUI: stdout = sys.stdout sys.stdout = io.StringIO() # Don't want to define in gui condition, so if not gui, do nothing if _GUI: def update_gui(content, html_id): try: predictit.gui_start.edit_gui_py(content, html_id) except (Exception, ): pass else: def update_gui(content, html_id): pass # Definition of the table for spent time on code parts time_df = [] def update_time_table(time_last): time_df.append([progress_phase, round((time.time() - time_last), 3)]) return time.time() time_point = time_begin = time.time() ############### ### ANCHOR ### Data ############# progress_phase = "Data loading and preprocessing" update_gui(progress_phase, "progress_phase") data = mdp.load_data.load_data( config.data, header=config.header, csv_style=config.csv_style, predicted_table=config.predicted_table, max_imported_length=config.max_imported_length, request_datatype_suffix=config.request_datatype_suffix, data_orientation=config.data_orientation, ) ############### ### ANCHOR ### Data consolidation ############# if not config.predicted_column: config.predicted_column = 0 data_for_predictions_df = mdp.preprocessing.data_consolidation( data, predicted_column=config.predicted_column, other_columns=config.other_columns, datalength=config.datalength, datetime_column=config.datetime_column, unique_threshold=config.unique_threshold, embedding=config.embedding, freq=config.freq, resample_function=config.resample_function, remove_nans_threshold=config.remove_nans_threshold, remove_nans_or_replace=config.remove_nans_or_replace, dtype=config.dtype, ) # In data consolidation predicted column was replaced on index 0 as first column predicted_column_index = 0 predicted_column_name = data_for_predictions_df.columns[0] ############### ### ANCHOR ### Analyze original data ############# column_for_predictions_series = data_for_predictions_df.iloc[:, 0:1] results = {} data_inputs = [] if config.mode == "validate": column_for_predictions_series = column_for_predictions_series.iloc[: -config . output . predicts, :] config.repeatit = 1 for i in config.used_models: data_inputs.append(config.models_input[i]) data_inputs = set(data_inputs) if config.analyzeit == 1 or config.analyzeit == 3: print("Analyze of unprocessed data") try: predictit.analyze.analyze_column(data_for_predictions_df.values[:, 0], window=30) predictit.analyze.analyze_data(data_for_predictions_df) predictit.analyze.decompose( data_for_predictions_df.values[:, 0], **config.analyze_seasonal_decompose, ) except Exception: mylogging.traceback("Analyze failed", level="ERROR") semaphor = None if config.multiprocessing: multiprocessing.freeze_support() if not config.processes_limit: config.processes_limit = multiprocessing.cpu_count() - 1 if config.multiprocessing == "process": pipes = [] semaphor = multiprocessing.Semaphore(config.processes_limit) elif config.multiprocessing == "pool": pool = multiprocessing.Pool(config.processes_limit) # It is not possible easy share data in multiprocessing, so results are resulted via callback function def return_result(result): for i, j in result.items(): results[i] = j ### Optimization loop if (not config.optimization or not config.optimization_variable or not config.optimization_values or len(config.optimization_values) == 1): config.variable_optimization.optimization = False config.optimization_values = ["Not optimized"] config.optimization_variable = None time_point = update_time_table(time_point) progress_phase = "Predict" update_gui(progress_phase, "progress_phase") models_indexed = {i: j for i, j in enumerate(config.used_models)} ############### ### ANCHOR ### Main loop ############# for optimization_index, optimization_value in enumerate( config.optimization_values): # TODO check why setattr - may be wrong after config change if config.optimization_variable: setattr(config, config.optimization_variable, optimization_value) ############### ### ANCHOR ### Feature extraction ############# if config.add_fft_columns: data_for_predictions_df = mdp.feature_engineering.add_frequency_columns( data_for_predictions_df, window=config.feature_engineering.add_fft_columns, ) if config.data_extension: data_for_predictions_df = mdp.feature_engineering.add_derived_columns( data_for_predictions_df, **config.feature_engineering.data_extension) ############### ### ANCHOR ### Feature selection ############# # data_for_predictions_df TODO ############### ### ANCHOR ### Data preprocessing ############# if config.mode == "validate": test_unstandardized = mdp.misc.split( data_for_predictions_df, predicts=config.predicts)[1].values models_test_outputs_unstandardized = [test_unstandardized] else: models_test_outputs_unstandardized = mdp.create_model_inputs.create_tests_outputs( data_for_predictions_df.values[:, 0], predicts=config.predicts, repeatit=config.repeatit, ) data_for_predictions, last_undiff_value, final_scaler = mdp.preprocessing.preprocess_data( data_for_predictions_df.values, remove_outliers=config.remove_outliers, smoothit=config.smoothit, correlation_threshold=config.correlation_threshold, data_transform=config.data_transform, standardizeit=config.standardizeit, bins=config.bins, binning_type=config.binning_type, ) data_for_predictions = cast(np.ndarray, data_for_predictions) if config.mode == "validate": data_for_predictions, test = mdp.misc.split( data_for_predictions, predicts=config.predicts) models_test_outputs = [test] else: models_test_outputs = mdp.create_model_inputs.create_tests_outputs( data_for_predictions[:, 0], predicts=config.predicts, repeatit=config.repeatit, ) column_for_predictions_processed = data_for_predictions[:, predicted_column_index] data_shape = np.shape(data_for_predictions) data_length = len(column_for_predictions_processed) data_std = np.std(column_for_predictions_processed[-30:]) data_mean = np.mean(column_for_predictions_processed[-30:]) data_abs_max = max( abs(column_for_predictions_processed.min()), abs(column_for_predictions_processed.max()), ) multicolumn = 0 if data_shape[1] == 1 else 1 if (config.analyzeit == 2 or config.analyzeit == 3 ) and optimization_index == len(config.optimization_values) - 1: print("\n\nAnalyze of preprocessed data\n") try: predictit.analyze.analyze_column( column_for_predictions_processed, window=30) predictit.analyze.analyze_data(data_for_predictions) predictit.analyze.decompose( column_for_predictions_processed, **config.analyze_seasonal_decompose, ) except Exception: mylogging.traceback("Analyze failed", level="ERROR") min_data_length = 3 * config.predicts + config.default_n_steps_in if (data_length < min_data_length or data_length < config.repeatit + config.default_n_steps_in + config.predicts): config.repeatit = 1 min_data_length = 3 * config.predicts + config.default_n_steps_in assert min_data_length < data_length, mylogging.return_str( "Set up less predicted values in settings or add more data", caption="To few data", ) for data_inputs_name in data_inputs: try: ( model_train_input, model_predict_input, model_test_inputs, ) = mdp.create_model_inputs.create_inputs( data_for_predictions, input_type_name=data_inputs_name, input_type_params=config.data_inputs[data_inputs_name], mode=config.mode, predicts=config.predicts, repeatit=config.repeatit, predicted_column_index=predicted_column_index, ) except Exception: mylogging.traceback( f"Error in creating input type: {data_inputs_name} with option optimization: {optimization_value}", level="WARNING", ) continue for (iterated_model_index, iterated_model_name) in models_indexed.items(): iterated_model = predictit.models.models_assignment[ iterated_model_name] if config.models_input[ iterated_model_name] == data_inputs_name: predict_parameters = { "config": config.get_dict(), # Functions to not import all modules "preprocess_data_inverse": mdp.preprocessing.preprocess_data_inverse, "fitted_power_transform": mdp.preprocessing.fitted_power_transform, # Other "iterated_model_train": iterated_model.train, "iterated_model_predict": iterated_model.predict, "iterated_model_name": iterated_model_name, "iterated_model_index": iterated_model_index, "optimization_index": optimization_index, "optimization_value": optimization_value, "model_train_input": model_train_input, "model_predict_input": model_predict_input, "model_test_inputs": model_test_inputs, "models_test_outputs": models_test_outputs, "models_test_outputs_unstandardized": models_test_outputs_unstandardized, "data_abs_max": data_abs_max, "data_mean": data_mean, "data_std": data_std, "last_undiff_value": last_undiff_value, "final_scaler": final_scaler, "semaphor": semaphor, } if config.models_input[iterated_model_name] in [ "one_step", "one_step_constant", ]: if multicolumn and config.predicts > 1: mylogging.warn( f"Warning in model {iterated_model_name} \n\nOne-step prediction on " "multivariate data (more columns). Use multi_step (y lengt equals to predict) " "or do use some one column data input in config models_input or predict just one value." ) continue if config.multiprocessing == "process": pipes.append(multiprocessing.Pipe(duplex=False)) p = multiprocessing.Process( target=predictit._main_loop.train_and_predict, kwargs={ **predict_parameters, **{ "pipe": pipes[-1][1] } }, ) p.Daemon = True # Baby process will be terminated if parent killed p.start() elif config.multiprocessing == "pool": pool.apply_async( predictit._main_loop.train_and_predict, (), predict_parameters, callback=return_result, ) else: results = { **results, **predictit._main_loop.train_and_predict(**predict_parameters), } if config.multiprocessing: if config.multiprocessing == "process": for i in pipes: try: results = {**results, **i[0].recv()} except Exception: pass if config.multiprocessing == "pool": pool.close() pool.join() for i in results.values(): mylogging.my_logger.log_and_warn_from_lists( i["logs_list"], i["warnings_list"]) # Create confidence intervals if config.confidence_interval: try: lower_bound, upper_bound = predictit.misc.confidence_interval( column_for_predictions_series.values, predicts=config.predicts, confidence=config.confidence_interval, ) grey_area = ["Lower bound", "Upper bound"] bounds = True except Exception: bounds = False grey_area = ["Lower bound", "Upper bound"] mylogging.traceback("Error in compute confidence interval", level="ERROR") else: bounds = False grey_area = False ############### ### ANCHOR ### Results processing ############# # Criterion is the best of average from repetitions time_point = update_time_table(time_point) progress_phase = "Evaluation" update_gui(progress_phase, "progress_phase") # Two kind of results we will create. Both as dataframe # - First are all the details around prediction. Model errors, time, memory peak etc. # - Second we have predicted values # Results such as trained model etc. that cannot be displayed in dataframe are in original results dict. # Convert results from dictionary to dataframe - exclude objects like trained model results_df = pd.DataFrame.from_dict(results, orient="index") if results_df.empty: raise RuntimeError( mylogging.return_str( "None of models finished predictions. Set config.logger_level = 'DEBUG' for more info.", caption="All models failed for some reason", )) evaluated_matrix = np.zeros( (1, len(config.optimization_values), len(config.used_models))) evaluated_matrix.fill(np.nan) for k in results.values(): evaluated_matrix[0, k["Index"][0], k["Index"][1]] = k["Model error"] ( _, best_models_optimized_values, optimized_values_results_df, best_model_name, best_optimized_value, ) = predictit.analyze.analyze_results( evaluated_matrix, config.optimization_values, config.models.used_models, config.prediction.error_criterion, ) # Generate date indexes for result predictions last_date = column_for_predictions_series.index[-1] if isinstance( last_date, (pd.core.indexes.datetimes.DatetimeIndex, pd._libs.tslibs.timestamps.Timestamp), ): date_index = pd.date_range( start=last_date, periods=config.predicts + 1, freq=column_for_predictions_series.index.freq, )[1:] date_index = pd.to_datetime(date_index) else: date_index = list(range(last_date + 1, last_date + config.predicts + 1)) predictions_df = pd.DataFrame(index=date_index) results_df.sort_values("Model error", inplace=True) for i, row in results_df.iterrows(): predictions_df[i] = row["Results"] if predictions_df.empty: raise RuntimeError( mylogging.return_str( "Neither of models finished prediction. Set config.logger_level = 'DEBUG' for more info." )) if config.variable_optimization.optimization: best_optimized_values_dict = { j: best_models_optimized_values[i] for i, j in enumerate(config.used_models) } best_indexes = [] for i, row in results_df.iterrows(): if row["Optimization value"] == best_optimized_values_dict[ row["Name"]]: best_indexes.append(i) optimization_result = predictit._result_classes.Optimization( optimized_variable=config.variable_optimization. optimization_variable, optimized_options=config.variable_optimization.optimization_values, best_value=best_optimized_value, values_results_df=optimized_values_results_df, best_values_for_models=best_optimized_values_dict, all_models_results_df=results_df, all_models_predictions_df=predictions_df, ) predictions_df = predictions_df[best_indexes] predictions_df.columns = [ results_df.loc[i]["Name"] for i in predictions_df.columns ] results_df = results_df.loc[best_indexes] results_df.rename(columns={"A": "Col_1"}, inplace=True) else: optimization_result = None if config.hyperparameter_optimization.optimizeit: hyperparameter_optimization_kwargs = results_df[ "Best optimized parameters"].to_dict() else: hyperparameter_optimization_kwargs = None results_df.set_index("Name", inplace=True) results_to_drop = [ i for i in [ "Index", "Trained model", "Test errors", "Results", "logs_list", "warnings_list", ] if i in results_df.columns ] results_df.drop(columns=results_to_drop, inplace=True) best_model_predicts = predictions_df[best_model_name] ############### ### ANCHOR ### Plot ############# if config.variable_optimization.optimization and config.variable_optimization.plot_all_optimized_models: predictions_for_plot = optimization_result.all_models_predictions_df.copy( ) else: predictions_for_plot = predictions_df.copy() predictions_for_plot.columns = [ f"{i + 1} - {j}" for i, j in enumerate(predictions_for_plot.columns) ] if config.mode == "validate": best_model_name_plot = "Test" predictions_df.insert(0, "Test", test_unstandardized) predictions_for_plot.insert(0, "Test", test_unstandardized) else: best_model_name_plot = predictions_for_plot.columns[0] bounds_df = pd.DataFrame(index=date_index) if bounds: bounds_df["Upper bound"] = upper_bound bounds_df["Lower bound"] = lower_bound last_value = float(column_for_predictions_series.iloc[-1, 0]) predictions_for_plot_limited = pd.concat( [ predictions_for_plot.iloc[:, :config.plot_number_of_models], bounds_df ], axis=1, ) predictions_with_history = pd.concat( [ column_for_predictions_series[-config.plot_history_length:], predictions_for_plot_limited, ], sort=False, ) predictions_with_history.iloc[-config.predicts - 1, :] = last_value if config.sort_results_by == "name": results_df.sort_index(key=lambda x: x.str.lower(), inplace=True) predictions_df.sort_index(key=lambda x: x.str.lower(), inplace=True) if config.general.analyzeit: import matplotlib.pyplot as plt plt.show() time_point = update_time_table(time_point) progress_phase = "plot" update_gui(progress_phase, "progress_phase") if config.output.plot_subconfig.show_plot or config.output.plot_subconfig.save_plot: with warnings.catch_warnings(): warnings.simplefilter("ignore", ResourceWarning) return_div = True if _GUI else False if config.plot_type == "with_history": div = plot( predictions_with_history, plot_library=config.plot_library, plot_name=config.plot_name, legend=config.plot_legend, highlighted_column=predicted_column_name, surrounded_column=best_model_name_plot, grey_area=grey_area, save=config.save_plot, return_div=return_div, show=config.output.plot_subconfig.show_plot, ) elif config.plot_type == "just_results": div = plot( predictions_for_plot, plot_library=config.plot_library, legend=config.plot_legend, highlighted_column=best_model_name_plot, save=config.save_plot, show=config.output.plot_subconfig.show_plot, ) update_time_table(time_point) progress_phase = "Completed" update_gui(progress_phase, "progress_phase") ############### ### ANCHOR ### Table ############# time_df.append(["Complete time", round((time.time() - time_begin), 3)]) time_df = pd.DataFrame(time_df, columns=["Part", "Time"]) simple_table_df = mdp.misc.edit_table_to_printable(results_df[[ "Model error" ]].iloc[:config.print_number_of_models, :].reset_index()) detailed_table_df = results_df.iloc[:config. print_number_of_models, :].reset_index( ) detailed_table_df.drop(["Unstandardized model error"], axis=1, inplace=True) detailed_table_df = mdp.misc.edit_table_to_printable(detailed_table_df) tables = predictit._result_classes.Tables( simple=tabulate( simple_table_df.values, headers=["Model", f"Average {config.error_criterion} error"], **config.table_settings, ), detailed=tabulate( detailed_table_df.values, headers=detailed_table_df.columns, **config.table_settings, ), time=tabulate(time_df.values, headers=time_df.columns, **config.table_settings), simple_table_df=simple_table_df, detailed_table_df=detailed_table_df, ) ############### ### ANCHOR ### Results ############# misc_result = predictit._result_classes.Misc( evaluated_matrix=evaluated_matrix) result = predictit._result_classes.Result( best_prediction=best_model_predicts, best_model_name=best_model_name, predictions=predictions_df, results_df=results_df, results=results, with_history=predictions_with_history, tables=tables, config=config, misc=misc_result, optimization=optimization_result, hyperparameter_optimization_kwargs=hyperparameter_optimization_kwargs, ) ############### ### ANCHOR ### Print ############# if config.print_result_details: print(( f"\nBest model is {best_model_name} with results \n\n{best_model_predicts}\n\nWith model error {config.error_criterion} = " f"{results_df.loc[best_model_name, 'Model error']}")) if config.print_table == "simple": print(f"\n{tables.simple}\n") elif config.print_table == "detailed": print(f"\n{tables.detailed}\n") if config.print_time_table: print(f"\n{tables.time}\n") ############### ### ANCHOR ### Return ############# mylogging.reset_outer_warnings_filter() # Return stdout and stop collect warnings and printed output if _GUI: output = sys.stdout.getvalue() sys.stdout = stdout result.output = output print(output) if _GUI: result.plot = div if config.return_internal_results: return { "data_for_predictions (X, y)": data_for_predictions, "model_train_input": model_train_input, "model_predict_input": model_predict_input, "model_test_inputs": model_test_inputs, "models_test_outputs": models_test_outputs, } return result
if (count == 20): time_end = time.clock() print("消费 " + str(count) + "个商品所用时间: %f s" % (time_end - time_start)) mutex.release() empty.release() time.sleep(1) if __name__ == '__main__': product = multiprocessing.Array('d', range(5)) pindex = multiprocessing.Value('i', 0) cindex = multiprocessing.Value('i', 0) mutex = multiprocessing.Semaphore(1) full = multiprocessing.Semaphore(0) empty = multiprocessing.Semaphore(5) Process(target=produce, args=(1, mutex, full, empty, product, pindex, cindex)).start() Process(target=produce, args=(2, mutex, full, empty, product, pindex, cindex)).start() Process(target=consume, args=(mutex, full, empty, product, pindex, cindex)).start()
csv_bands.close() if __name__ == '__main__': print("Mapping products to csv...") n_processes = config.n_processes_generate_dataset product_directory = paths.product_directory tmp = paths.tmp if not(exists(tmp)): mkdir(tmp) if not(exists(paths.dataset_directory)): mkdir(paths.dataset_directory) products= listdir(product_directory) semaphore = multiprocessing.Semaphore(n_processes) progress_bar = Progress_bar(len(products)) job_list = [] for product in products: semaphore.acquire() job = multiprocessing.Process(target=process, args=(product,semaphore,progress_bar, tmp)) job_list.append(job) job.start() for j in job_list: j.join() csv_index = open(paths.csv_index, 'wb+') csv_bands = open(paths.csv_band, 'wb+') csv_index.write(b'DATE;LONGITUDE;LATITUDE;NDVI;SLAVI;GVMI;NDWI;BSI;NPCRI;CLASS\n') csv_bands.write(b'DATE;LONGITUDE;LATITUDE;B01;B02;B03;B04;B05;B06;B07;B08;B8A;B09;B11;B12;CLASS\n')
# Obtain possible configuration from the environment, assuming 1 (on) # by default, upon 0 set to None. Should instructively fail if some non # 0/1 value is set. multiprocessing = int(os.environ.get('JOBLIB_MULTIPROCESSING', 1)) or None if multiprocessing: try: import multiprocessing except ImportError: multiprocessing = None # 2nd stage: validate that locking is available on the system and # issue a warning if not if multiprocessing: try: _sem = multiprocessing.Semaphore() del _sem # cleanup except (ImportError, OSError) as e: multiprocessing = None warnings.warn('%s. joblib will operate in serial mode' % (e, )) from .format_stack import format_exc, format_outer_frames from .logger import Logger, short_format_time from .my_exceptions import TransportableException, _mk_exception ############################################################################### # CPU that works also when multiprocessing is not installed (python2.5) def cpu_count(): """ Return the number of CPUs. """
import serial, time import threading, random import multiprocessing as mp import requests, json # CONSTANT SERIALPORT = "COM7" BAUDRATE = 115200 API_SENSOR_URL = "http://localhost:8081/api/sensors" # GLOBAL nValue, nEndTransmission = 0, 0 locationX, locationY = 0, 0 ser = serial.Serial() semaphoreSendingData = mp.Semaphore(1) def getFireHTTP(sLocationX, sLocationY): headers = {"Content-Type": "application/json"} result = requests.get(url=API_SENSOR_URL, headers=headers) jsonValue = json.loads(result.content.decode("utf-8")) for i in range(len(jsonValue)): if (jsonValue[i])["x"] == int(sLocationX) and ( jsonValue[i])["y"] == int(sLocationY): return ((jsonValue[i])["intensity"]) # Function in charge of initialize the serial port def initUART(): ser.port = SERIALPORT ser.baudrate = BAUDRATE ser.bytesize = serial.EIGHTBITS #number of bits per bytes
# -*- coding: utf-8 -*- """ Created on Fri Jun 1 10:35:45 2018 @author: antoine.bralet """ import multiprocessing as mp import sys import os import time verrou = mp.Semaphore(1) verrou2 = mp.Semaphore(0) verrou3 = mp.Semaphore(0) def A(): verrou.acquire() print("Vive le dodo de A") time.sleep(2) print("Coucou je suis le processus A et j'ai attendu 2 secondes") verrou.release() verrou.release() verrou.release() def B(): verrou.acquire() print("Vive le dodo de B") time.sleep(3) print("Coucou je suis le processus B et j'ai attendu 3 secondes")
# -*- coding: utf-8 -*- """ Created on Fri Jun 1 11:53:36 2018 @author: antoine.bralet """ import multiprocessing as mp import sys import os import time import random patates = mp.Semaphore(0) fromage = mp.Semaphore(0) menage = mp.Semaphore(0) def rdv1(): print("Je suis la ca y est") fromage.release() menage.release() patates.acquire() patates.acquire() print("Il faut que j'aille chercher des patates pour la raclette") time.sleep(4) def rdv2(): print("Je suis la ca y est") patates.release() menage.release() fromage.acquire()