def loadData(bot, update): c = StrictRedis() query = update.callback_query chat_id = query.message.chat_id regNumber = c.hget(chat_id, 'umsReg') umsPass = c.hget(chat_id, 'umsPass') count = 0 bot.send_chat_action(chat_id=chat_id, action=ChatAction.TYPING) try: homeUms = Verto(regNumber, umsPass) homeData = homeUms.initiater() if homeData != 'Incorrect Credentials': pass else: bot.edit_message_text(text='Incorrect Credentials', chat_id=chat_id, message_id=query.message.message_id) raise ValueError("Something Went Wrong") except ValueError as error: return else: def msg(): msgUms = Message(homeData) msgList = msgUms.initiater() no_msg = 0 for i, item in enumerate(msgList): if item != '': no_msg = no_msg + 1 item = ('|').join(item) c.hset(chat_id, 'msg' + str(i), item) c.hset(chat_id, '#msg', str(no_msg)) def course(): courseRes = Resources(homeUms.session) courseList = courseRes.course_codes() c.hset(chat_id, '#course', str(len(courseList))) for i, item in enumerate(courseList): c.hset(chat_id, 'c{}'.format(i), item) msg() pool = Pool(4) p1 = pool.Process(target=course) p1.start() p1.join() bot.edit_message_text(text="All set, Now what you want to know about?", chat_id=chat_id, message_id=query.message.message_id, reply_markup=markup_options) return
def EID_Attenuation_Sim(simDataFile, nThread): # Read simulation jobs f = open(simDataFile, "r") trials = f.readlines() nSimTrials = len(trials) # Submit simulations if nThread > cpu_count(): nThread = cpu_count() if nSimTrials < nThread: nThread = nSimTrials # Start a new parallel pool print("Starting parallel pool with {0} threads".format(nThread)) ctx = get_context("fork") pool = Pool(processes=nThread) max_queue_size = min(2 * nThread, nSimTrials) work_queue = Queue(maxsize=max_queue_size) jobs = [] remaining_jobs = nSimTrials # Kick off worker threads for _ in range(nThread): # Start a new job thread try: p = pool.Process(target=QueueWorker, args=(work_queue, )) except Exception: if ctx is not None: # Fallback to use context p = ctx.Process(target=QueueWorker, args=(work_queue, )) p.start() jobs.append(p) for it in range(nSimTrials): # Fill in work queue work_queue.put((trials[it].split(), remaining_jobs), block=True, timeout=None) remaining_jobs -= 1 print_color( f"[MasterNode-{getpid()}]: Adding new job {trials[it].split()[3]}, " f"remaining jobs {remaining_jobs}", color="green", ) # Unfortunately we need to wait briefly before adding new data into the queue. # This is because it takes some time for the object to get properly ingested. time.sleep(0.1) # Wait until all the active thread to finish for job in jobs: job.join()
def evaluate_command(command, attributes): manager = multiprocessing.Manager() return_list = manager.list() pool = Pool(processes=1) main_queue = Queue() main_event = Event() main_event.set() proc = pool.Process(target=handle_service, args=(main_queue, main_event, command, attributes, return_list)) proc.daemon = True proc.start() proc.join() print("from main: " + str(return_list))
def main(command, value): manager = multiprocessing.Manager() return_list = manager.list() pool = Pool(processes=1) value = 'test' + str(command) main_queue = Queue() main_event = Event() main_event.set() proc = pool.Process(target=eval(command), args=(main_queue, main_event, value, return_list)) proc.daemon = True proc.start() proc.join() print("from main: " + str(return_list))
def main(): try: limit = 256 sigint = signal.signal(signal.SIGINT, signal.SIG_IGN) pool = Pool(processes=limit) signal.signal(signal.SIGINT, sigint) eid_proc = {} main_queue = Queue() main_event = Event() main_event.set() for eid in range(limit): proc = pool.Process(target=echo_forever, args=(main_queue, main_event)) proc.daemon = True proc.start() eid_proc[eid] = proc while True: alive = len( [_proc for _proc in eid_proc.values() if _proc.is_alive()]) print('{}/{} alive'.format(alive, limit)) if alive == 0: print('All of the client processes are dead. Exiting loop...') break else: time.sleep(1) except (KeyboardInterrupt, Exception): main_event.clear() for proc in eid_proc.values(): proc.terminate() finally: res = [] while not main_queue.empty(): res.append(main_queue.get()) cdelta = [_max['cdelta'] for _max in res] delta = [_max['delta'] for _max in res] if cdelta: cdelta = max(cdelta) else: cdelta = "unknown" if delta: delta = max(delta) else: delta = "unknown" time.sleep(1) print("Max time to establish: {}".format(cdelta)) print("Max time echo reply: {}".format(delta)) print(cdelta, delta)
def exposed_echo(self, message, value): manager = multiprocessing.Manager() return_list = manager.list() if message == "Echo": print("received EchoService - forwarding to executing server") #sigint = signal.signal(signal.SIGINT, signal.SIG_IGN) pool = Pool(processes=1) #signal.signal(signal.SIGINT, sigint) server_res = [] main_queue = Queue() main_event = Event() main_event.set() proc = pool.Process(target=echo, args=(main_queue, main_event, value, return_list)) proc.daemon = True proc.start() proc.join() print(return_list) return str(return_list[0]) else: return "Parameter Problem"
def SimulationMainQueue(dataFiles, nThread=1): if type(dataFiles) is not list: dataFiles = [dataFiles] paramList = [] simParamList = [] nSimJobsList = [] for dat in dataFiles: # Load parameters paramList.append(loadParams(dat)) # Permutate conditions simParamList.append( list( product( paramList[-1]["SNR"], paramList[-1]["procNoiseSigma"], paramList[-1]["pLSigmaAmp"], paramList[-1]["Sigma"], paramList[-1]["objAmp"], paramList[-1]["dt"], paramList[-1]["wControl"], paramList[-1]["randSeed"], ) ) ) nSimJobsList.append(len(simParamList[-1])) nTrials = len(nSimJobsList) # additional wiggle attenuation sims with open("./SimParameters/SimJobList.txt", "r") as fp: attenuation_sim_trials = fp.readlines() attenuation_sim_trials.sort() nAttenuationSimTrials = len(attenuation_sim_trials) nRegularSimJobs = sum(nSimJobsList) nTotalJobs = nRegularSimJobs + nAttenuationSimTrials # Limit pool size when job size is smaller than total available threads if nTotalJobs < nThread: nThread = nTotalJobs # Start a new parallel pool print("Starting parallel pool with {0} threads".format(nThread)) ctx = get_context("fork") pool = Pool(processes=nThread) work_queue = JoinableQueue(maxsize=nTotalJobs) jobs = [] remaining_jobs = nTotalJobs job_id = 1 for it in range(nAttenuationSimTrials): job_path = f'{sim_jobs_path}/eih_sim_job_{job_id}.pkl' with open(job_path, 'wb') as fp: pkl.dump(attenuation_sim_trials[it].split(), fp, pkl.HIGHEST_PROTOCOL) # Fill in work queue work_queue.put(job_path, block=True, timeout=None) job_id += 1 remaining_jobs -= 1 print_color( f"[MasterNode-{getpid()}]: Adding new job {attenuation_sim_trials[it].split()[3]}", color="green", ) # Kick off worker threads for _ in range(nThread): # Start a new job thread try: p = pool.Process(target=QueueWorker, args=(work_queue,)) except Exception: if ctx is not None: # Fallback to use context p = ctx.Process(target=QueueWorker, args=(work_queue,)) p.start() jobs.append(p) for trial in range(nTrials): # Parse parameters param = paramList[trial] simParam = simParamList[trial] nJobs = nSimJobsList[trial] eidParam = param["eidParam"] ergParam = param["ergParam"] filename = param["filename"] # Check if saveDir exists, create the folder if not if not exists(eidParam.saveDir): print(f"Save folder {eidParam.saveDir} does not exist, creating...") makedirs(eidParam.saveDir, exist_ok=True) ergParam.time = None ergParam.eidTime = None for it in range(nJobs): eidParam.SNR = simParam[it][0] eidParam.procNoiseSigma = simParam[it][1] eidParam.pLSigmaAmp = simParam[it][2] eidParam.pLSigmaAmpBayesian = simParam[it][2] eidParam.pLSigmaAmpEID = simParam[it][2] eidParam.Sigma = simParam[it][3] eidParam.objAmp = simParam[it][4] ergParam.dt = simParam[it][5] eidParam.UpdateDeltaT(simParam[it][5]) if eidParam.simType == "IF": eidParam.maxIter = round(eidParam.maxT / ergParam.dt) ergParam.wControl = simParam[it][6] eidParam.randSeed = simParam[it][7] eidParam.filename = ( filename.replace("SNR", "SNR-" + str(eidParam.SNR)) .replace("wC", "wC-" + str(ergParam.wControl)) .replace("RandSeed", "RandSeed-" + str(eidParam.randSeed)) ) # Do the extra initialization here to speed up. if isinstance(eidParam.rawTraj, str) and "moth" in eidParam.rawTraj: eidParam.rawTraj = np.array( loadMothData( target="M300lux", trialID=0, nrmGain=eidParam.objAmp )[1] ) ergParam.time = linspace(0.0, ergParam.timeHorizon, ergParam.tRes) ergParam.eidTime = linspace(0.0, ergParam.timeHorizon, eidParam.res) # initialize multiple targets tracking if eidParam.multiTargetTracking == "dual": eidParam.multiTargetTracking = True eidParam.otherTargets = [RealTarget(eidParam.multiTargetInitialPos)] elif eidParam.multiTargetTracking == "distractor": eidParam.multiTargetTracking = True eidParam.otherTargets = [Distractor(eidParam.multiTargetInitialPos)] # Fill in work queue if it's not full job_path = f'{sim_jobs_path}/eih_sim_job_{job_id}.pkl' with open(job_path, 'wb') as fp: pkl.dump((ergParam, eidParam, False), fp, pkl.HIGHEST_PROTOCOL) work_queue.put(job_path, block=True, timeout=None) job_id += 1 remaining_jobs -= 1 print_color( f"[MasterNode-{getpid()}]: Adding new job {eidParam.filename}", color="green", ) # Wait until all the active thread to finish work_queue.join() for job in jobs: job.join()