def funcSharedFunction(): shared.setConst(myRemoteFunc=func4) result = True for _ in range(100): try: result &= futures.submit(funcUseSharedFunction).result() except AssertionError: result = False return result
def initialize(wd, rxns): global working_dir, reactions working_dir = wd assert os.path.isdir(working_dir) #set global variable here such that functions executed in the root worker have access to it. reactions = rxns reduce_reactions = [ReductionReaction(rxn) for rxn in rxns] shared.setConst(reactions=reduce_reactions)
def funcSharedFunction(): shared.setConst(myRemoteFunc=func4) result = True for _ in range(100): try: result &= futures.submit(funcUseSharedFunction).result() except AssertionError: result = False return result
def initialize(wd, rxns): global working_dir, reactions working_dir = wd assert os.path.isdir(working_dir) #set global variable here such that functions executed in the root worker have access to it. reactions = rxns reduce_reactions = [ReductionReaction(rxn) for rxn in rxns] shared.setConst(reactions = reduce_reactions)
def funcSharedConstant(): shared.setConst(myVar={1: 'Example 1', 2: 'Example 2', 3: 'Example 3', }) shared.setConst(secondVar="Hello World!") result = True for _ in range(100): try: result &= futures.submit(funcUseSharedConstant).result() except AssertionError: result = False return result
def funcSharedConstant(): shared.setConst(myVar={1: 'Example 1', 2: 'Example 2', 3: 'Example 3', }) shared.setConst(secondVar="Hello World!") result = True for _ in range(100): try: result &= futures.submit(funcUseSharedConstant).result() except AssertionError: result = False return result
def funcUseDeletedSharedConstant(): # Tries to retrieve a deleted variable, should timeout shared.deleteConst('myVar') shared.deleteConst('secondVar') timeout = 0.01 assert shared.getConst('myVar', timeout=timeout) is None assert shared.getConst('secondVar', timeout=timeout) is None try: # You should not be able to reset a deleted variable shared.setConst(myVar=42) return False except TypeError: pass return True
def simulate(config): config_data = json.load(config) config.close() config = config_data output_formatter = config["output_formatter"] subs = {a:b for a,b in config.get("range_substitutions", [["Z",{"min":0,"max":0.5,"step":1}]])} range_constraints = [eval("lambda {}:{}".format(",".join(subs.keys()),con)) for con in config["range_constraints"]] def switch_weighter(vector): return lambda x,i,j: x if vector[j] is None else vector[j].pop() if x else 0 switch = config['switch'] write_delay = config.get("write_delay",10) V = {} V['choice'] = matrix_map(config['choice'],lambda x,i,j:parse_expr(str(x))) V['i_ip'] = int(config['power_iterations']["super_iterations"]) V['i_i'] = int(config['power_iterations']["iterations"]) V['iterations'] = int(config['iterations']) if "starting_points" in config: origional_pops = [[a/sum(s) if sum(s)>0 else 1.0/len(s) for a in s] for s in config['starting_points']] V['origional_pops'] = [numpy.matrix([a]).transpose() for a in origional_pops] else: V['origional_pops'] = [numpy.matrix([[1.0/len(V['choice'])] for s in range(len(V['choice']))])] V['extrema'] = [numpy.matrix(matrix_map(switch,switch_weighter(e))) for e in generate_switch_extrema(switch)] V['pop_symbols'] = [sympy.symbols("s{}".format(i)) for i in range(len(V['choice']))] V['subs_symbols'] = {a:symbols(a) for a,b in subs.iteritems()} V['sequence'] = config.get('sequence',"1.0/(x+2)") V['weight_function'] = config['weight_function'] shared.setConst(V=V) output_json = [] big_list = [a for a in multi_iterate(subs) if False not in [con(**a) for con in range_constraints]] progress = tqdm(total=len(big_list)) last_write_time = time() def output_file(output_json): out_file = file(output_formatter, "w") out_file.write(json.dumps({"output":output_json},sort_keys=True).replace('{"parameters":','\n{"parameters":')) out_file.flush() out_file.close() for result in futures.map_as_completed(process, big_list): output_json.append(result) progress.update() if time()-last_write_time > write_delay: output_file(output_json) last_write_time = time() output_file(output_json) progress.close()
def broadcast(obj, key): """ Broadcasts the object across the workers using the key parameter as the key. """ kwargs = {key : obj} try: if shared.getConst(key): logger.debug('An object with the key {} was already broadcasted.'.format(key)) else: shared.setConst(**kwargs) except NameError, e: """ Name error will be caught when the SCOOP library is not imported properly. """ logger.debug('SCOOP not loaded. Not broadcasting the object {}'.format(obj))
def broadcast(obj, key): """ Broadcasts the object across the workers using the key parameter as the key. """ kwargs = {key : obj} try: if shared.getConst(key): logger.debug('An object with the key {} was already broadcasted.'.format(key)) else: shared.setConst(**kwargs) except NameError, e: """ Name error will be caught when the SCOOP library is not imported properly. """ logger.debug('SCOOP not loaded. Not broadcasting the object {}'.format(obj))
if sub: s += "<SUB> "+ sub +" </SUB>\n " if article_kwds: for kwd in article_kwds: if kwd: s += "<KWD> "+ kwd +" </KWD>\n " if article_title: s += "<ATITLE> "+ article_title +" </ATITLE>\n " if article_abstract: s += "<ABSTRACT> "+ article_abstract +" </ABSTRACT>\n " s += "<TEXT> "+ article_contents +" </TEXT>\n" s += "</DOC>\n" destination = shared.getConst('destination') with codecs.open(destination + "/" + doc_id, encoding="utf-8", mode="w") as f: f.write(s) return True if __name__ == "__main__": datapath = sys.argv[1] + "*.nxml" destination = sys.argv[2] print "Datapath: ", datapath nxmls = glob.iglob(datapath) shared.setConst(destination=destination) print "processed: ", len(list(futures.map(process_doc, nxmls)))
def main(): if len(sys.argv)!=4: print 'USAGE:' print 'python -m scoop devel.py [cloneID] [clusterDir] [outputDir]' print 'see devel_config.py' return cloneID = sys.argv[1] clusterDir = sys.argv[2]; assert clusterDir[-1]=='/',"should be ended with '/'" baseOutDir = sys.argv[3]; assert baseOutDir[-1]!='/',"should NOT be ended with '/'" clfParam = None method = cfg['method'] if method=='esvm': from esvm_config import config as clfParam elif method=='psvm': from psvm_config import config as clfParam else: print 'FATAL: unknown method' return outDir = os.path.join(baseOutDir,'devel-'+os.path.basename(baseOutDir)) if not(os.path.isdir(baseOutDir)): os.makedirs(baseOutDir) if not(os.path.isdir(outDir)): os.makedirs(outDir) ## Load data ################################################################################### dataLog = {}; dataLogFpath = os.path.join(outDir,'data_log_'+os.path.basename(baseOutDir)+'.json') dataset = clusterDir.split('/')[-2].split('-')[-1]; dataLog['dataset'] = dataset datasetParams = dataset.split('#') assert datasetParams[0]=='yamanishi' xyDevFpath = os.path.join(baseOutDir,'_'.join(['xdev','ydev','xrel','yrel']+datasetParams)+'.h5') if os.path.exists(xyDevFpath): print 'loading data from PREVIOUS...' with h5py.File(xyDevFpath,'r') as f: xdev = f['xdev'][:] ydev = f['ydev'][:] xrel = f['xrel'][:] yrel = f['yrel'][:] xrelraw = f['xrelraw'][:] with open(dataLogFpath,'r') as f: dataLog = yaml.load(f) else: print 'loading data FRESHLY...' print 'loading cluster result...' nUnlabels = [] statFnames = [i for i in os.listdir(clusterDir) if 'labels_stat.json' in i] for i in statFnames: with open(os.path.join(clusterDir,i),'r') as f: stat = yaml.load(f) nUnlabels.append(stat['0']) # use the cluster with minimum numbers of unlabeled samples metric = '_'.join(statFnames[ nUnlabels.index(min(nUnlabels)) ].split('_')[0:2]) dataLog['metric'] = metric connFpath = os.path.join(clusterDir,metric+'_labels.pkl') with open(connFpath,'r') as f: data = pickle.load(f) ## print 'getting devel and release data...' xraw = []; yraw = [] for k,v in data.iteritems(): for vv in v: xraw.append(vv) yraw.append(k) devIdx = [i for i in range(len(xraw)) if yraw[i]!=0] xdev = [xraw[i] for i in devIdx] ydev = [yraw[i] for i in devIdx] relIdx = [i for i in range(len(xraw)) if yraw[i]==0] xrel = [xraw[i] for i in relIdx] yrel = [yraw[i] for i in relIdx] dataLog['nDevel'] = len(devIdx); dataLog['nData'] = len(yraw) dataLog['rDevel:Data'] = dataLog['nDevel']/float(dataLog['nData']) dataLog['nDevel(+)'] = len( [i for i in ydev if i==1] ); assert dataLog['nDevel(+)']!=0 dataLog['nDevel(-)'] = len( [i for i in ydev if i==-1] ); assert dataLog['nDevel(-)']!=0 dataLog['rDevel(+):Devel'] = float(dataLog['nDevel(+)'])/dataLog['nDevel'] dataLog['rDevel(-):Devel'] = float(dataLog['nDevel(-)'])/dataLog['nDevel'] dataLog['rDevel(+):(-)'] = float(dataLog['nDevel(+)'])/float(dataLog['nDevel(-)']) dataLog['nRelease'] = len(relIdx); dataLog['rRelease:Data'] = dataLog['nRelease']/float(dataLog['nData']) ## print 'loading com, pro feature...' krFpath = os.path.join(cfg['datasetDir'],datasetParams[0],'feature', 'klekotaroth','klekotaroth-'+datasetParams[1]+'.h5') aacFpath = os.path.join(cfg['datasetDir'],datasetParams[0],'feature', 'amino-acid-composition','amino-acid-composition-'+datasetParams[1]+'.h5') krDict = {}; aacDict = {} with h5py.File(krFpath, 'r') as f: for com in [str(i) for i in f.keys()]: krDict[com] = f[com][:] with h5py.File(aacFpath, 'r') as f: for pro in [str(i) for i in f.keys()]: aacDict[pro] = f[pro][:] # aacDict[pro] = list( fu.map(lambda x: float('%.2f'%(x)),f[pro][:]) ) # rounding comFeaLenOri = len(krDict.values()[0]) proFeaLenOri = len(aacDict.values()[0]) ## msg = 'extract (com,pro) feature... dims: '+str(comFeaLenOri)+','+str(proFeaLenOri) msg += ' of '+str(len(ydev))+' and '+str(len(yrel)) print msg sh.setConst(krDict=krDict) sh.setConst(aacDict=aacDict) xdevf = list( fu.map(cutil.extractComProFea,xdev) ) xrelf = list( fu.map(cutil.extractComProFea,xrel) ) ## xyDevList = cutil.divideSamples(xdevf,ydev,cfg['smoteBatchSize']) if cfg['maxNumberOfSmoteBatch'] != 0: xyDevList = xyDevList[0:cfg['maxNumberOfSmoteBatch']] smoteSeed = util.seed(); dataLog['smoteSeed'] = smoteSeed sh.setConst(smoteSeed=smoteSeed) print 'resampling via Smote FRESHLY... '+str(len(xyDevList))+' smote(s)'+' on '+str(len(ydev)) smoteTic = time.time() xdevfr = []; ydevr = [] xydevfrList = list( fu.map(ensembleSmote,xyDevList) ) for xdevfri,ydevri in xydevfrList: for x in xdevfri: xdevfr.append(x.tolist()) for y in ydevri: ydevr.append(y) assert len(xdevfr)==len(ydevr),'len(xdevfr)!=len(ydevr)' dataLog['nSmote'] = len(xyDevList) dataLog['nDevelResampled'] = len(ydevr) dataLog['rDevelResampled:Data'] = dataLog['nDevelResampled']/float(dataLog['nData']) dataLog['nDevelResampled(+)'] = len( [i for i in ydevr if i==1] ) dataLog['nDevelResampled(-)'] = len( [i for i in ydevr if i==-1] ) dataLog['rDevelResampled(+):DevelResampled'] = dataLog['nDevelResampled(+)']/float(dataLog['nDevelResampled']) dataLog['rDevelResampled(-):DevelResampled'] = dataLog['nDevelResampled(-)']/float(dataLog['nDevelResampled']) dataLog['rDevelResampled(+):(-)'] = dataLog['nDevelResampled(+)']/float(dataLog['nDevelResampled(-)']) dataLog['timeSMOTE'] = str(time.time()-smoteTic) ## print 'update xdev,ydev,xrel... '+str(np.asarray(xdevfr).shape) xrelraw = xrel[:] # raw: feature is NOT extracted xrel = xrelf[:] xdev = xdevfr[:] ydev = ydevr[:] print 'writing updated xdev,ydev and xrel,yrel...' with h5py.File(xyDevFpath,'w') as f: f.create_dataset('xdev',data=xdev,dtype=np.float32) f.create_dataset('ydev',data=ydev,dtype=np.int8) f.create_dataset('xrel',data=xrel,dtype=np.float32) f.create_dataset('yrel',data=yrel,dtype=np.int8) f.create_dataset('xrelraw',data=xrelraw) print 'writing dataLog...' dataLog['nCom'] = len(krDict) dataLog['nPro'] = len(aacDict) with open(dataLogFpath,'w') as f: json.dump(dataLog,f,indent=2,sort_keys=True) ## TUNE+TRAIN+TEST ############################################################################# devLog = {} devSeed = util.seed(); dataLog['devSeed'] = devSeed tag = '_'.join([method+'#'+cloneID,dataset,util.tag()]) ## split devel dataset msg = ' '.join( ['devel',dataset,cloneID]) xtr,xte,ytr,yte = tts(xdev,ydev,test_size=cfg['testSize'], random_state=devSeed,stratify=ydev) if cfg['maxTestingSamples']>0: chosenIdx = np.random.randint(len(xte),size=cfg['maxTestingSamples']) xte = [xte[i] for i in chosenIdx]; yte = [yte[i] for i in chosenIdx] devLog['nTraining'] = len(xtr) devLog['nTraining(+)'] = len([i for i in ytr if i==1]) devLog['nTraining(-)'] = len([i for i in ytr if i==-1]) devLog['rTraining(+):(-)'] = devLog['nTraining(+)']/float(devLog['nTraining(-)']) devLog['rTraining:Devel'] = devLog['nTraining']/float(dataLog['nDevelResampled']) devLog['nTesting'] = len(xte) devLog['nTesting(+)'] = len([i for i in yte if i==1]) devLog['nTesting(-)'] = len([i for i in yte if i==-1]) devLog['rTesting(+):(-)'] = devLog['nTesting(+)']/float(devLog['nTesting(-)']) devLog['rTesting:Devel'] = devLog['nTesting']/float(dataLog['nDevelResampled']) ## tuning clf = None if method=='esvm': clf = eSVM(simMat=None) elif method=='psvm': clf = svm.SVC(kernel=clfParam['kernel'],probability=True) ## training print msg+': fitting nTr= '+str(len(ytr)) trTic = time.time() if method=='esvm': clf.fit(xtr,ytr) devLog['labels'] = clf.labels() devLog['nSVM'] = clf.nSVM() devLog['xtrDimAllBatches'] = clf.xtrDimAllBatches() elif method=='psvm': if cfg['method']['kernel']=='precomputed': assert False # simMatTr = cutil.makeComProKernelMatFromSimMat(xtr,xtr,simMat) # clf.fit(simMatTr,ytr) else: clf.fit(xtr,ytr) devLog['labels'] = clf.classes_.tolist() devLog['timeTraining'] = str(time.time()-trTic) ## testing print msg+': predicting nTe= '+str(len(yte)) teTic = time.time() if method=='esvm': ypred,yscore = clf.predict(xte) elif method=='psvm': if cfg['method']['kernel']=='precomputed': assert False # simMatTe = cutil.makeComProKernelMatFromSimMat(xte,xtr,simMat) # ypred = clf.predict(simMatTe) # yscore = clf.predict_proba(simMatTe) else: ypred = clf.predict(xte) yscore = clf.predict_proba(xte) yscore = [max(i.tolist()) for i in yscore] devLog['timeTesting'] = str(time.time()-teTic) ## TEST RELEASE ################################################################################ print msg+': predicting RELEASE n= '+str(len(yrel)) relTic = time.time() if method=='esvm': yrel,yrelscore = clf.predict(xrel) elif method=='psvm': if cfg['method']['kernel']=='precomputed': assert False # simMatTe = cutil.makeComProKernelMatFromSimMat(xrel,xtr,simMat) # yrel = clf.predict(simMatTe) # yrelscore = clf.predict_proba(simMatTe) else: yrel = clf.predict(xrel) yrelscore = clf.predict_proba(xrel) yrelscore = [max(i.tolist()) for i in yrelscore] devLog['timeRelease'] = str(time.time()-relTic) ## WRITE RESULT ################################################################################ result = {'yte':yte,'ypred':ypred,'yscore':yscore, 'xrelraw':xrelraw,'yrel':yrel,'yrelscore':yrelscore} print 'writing prediction...' with h5py.File(os.path.join(outDir,'result_'+tag+'.h5'),'w') as f: for k,v in result.iteritems(): if 'raw' in k: f.create_dataset(k,data=v) else: dt = np.int8 if 'score' in k: dt = np.float32 f.create_dataset(k,data=v,dtype=dt) ## print 'writing devLog...' devLog['clfParam'] = clfParam devLog['devParam'] = cfg with open(os.path.join(outDir,'devLog_'+tag+'.json'),'w') as f: json.dump(devLog,f,indent=2,sort_keys=True)
nb_runs, generations, mean_fit_mins, mean_fit_avg, mean_duration_mins, mean_duration_maxs = runs_results fig, ax1 = plt.subplots() fig.suptitle(f"Mean results over {nb_runs} runs, graph: '{graph_name}', duration constraint: {max_duration}") line1 = ax1.plot(generations, mean_fit_mins, "b-", label="Minimum Cost") line3 = ax1.plot(generations, mean_fit_avg, "g-", label= "Average Cost") ax1.set_xlabel("Generation") ax1.set_ylabel("Cost", color="b") for tl in ax1.get_yticklabels(): tl.set_color("b") ax2 = ax1.twinx() line2 = ax2.plot(generations, mean_duration_mins, "r-", label="Minimum Duration") line4 = ax2.plot(generations, mean_duration_maxs, "y-", label="Maximum Duration") ax2.set_ylabel("Duration", color="r") for tl in ax2.get_yticklabels(): tl.set_color("r") lns = line1 + line2 + line3 + line4 labs = [l.get_label() for l in lns] ax1.legend(lns, labs, loc="upper right") OUTPUT_DIR = os.environ.get("AZ_BATCH_TASK_DIR", ".") OUTPUT_FILE = os.path.join(OUTPUT_DIR, f"{graph_name}_{nb_runs}_runs.png") plt.savefig(OUTPUT_FILE) if __name__ == "__main__": graph_name = "MediumComplex" task_graph, MAXIMUM_DURATION = construct_graph(graph_name) shared.setConst(graph_name=graph_name, graph=task_graph, max_duration=MAXIMUM_DURATION) plot_runs_mean(multiple_runs_mean(10))
def main(): if len(sys.argv)!=6: print 'USAGE:' print 'python -m scoop cluster.py [method] [nIter] [dataset#x] [compound/protein] [outDir]' return method = sys.argv[1] nIter = int(sys.argv[2]) dataset = sys.argv[3] mode = sys.argv[4] outDir = sys.argv[5] outDir = os.path.join(outDir, '-'.join([method+'#'+str(nIter),dataset,mode,util.tag()])) os.makedirs(outDir) ## print 'loading data...' dParam = dataset.split('#') disMat = None; iList = None if dParam[0]=='yamanishi': dataDir = os.path.join(DATASET_DIR,dParam[0]) simDict = yam.loadKernel2(mode,dParam[1],os.path.join(dataDir,'similarity-mat')) simMat,iList = util.makeKernelMatrix(simDict) disMat = util.kernel2distanceMatrix('naive',simMat) else: assert False,'FATAL: unknown dataset' ## print 'clustering...' paramList = [] if method=='dbscan': epsMin,epsMax = [0.0,1.0] nMin,nMax = [1,len(iList)] for i in range(nIter): eps = np.random.uniform(epsMin,epsMax,1)[0] n = np.random.randint(nMin,nMax,1)[0] paramList.append( dict(eps=eps,min_samples=n) ) else: assert False sh.setConst(method=method) sh.setConst(mat=disMat) resList = list( fu.map(_cluster,paramList) ) bestResIdxCal = _getBestResultIdx(resList,'calinski_harabaz_score') bestResIdxSil = _getBestResultIdx(resList,'silhouette_score') resDictCal = dict( zip(iList,resList[bestResIdxCal][0]) ) resDictSil = dict( zip(iList,resList[bestResIdxSil][0]) ) bestParamCal = dict(param=paramList[bestResIdxCal], score=resList[bestResIdxCal][1]) bestParamSil = dict(param=paramList[bestResIdxSil], score=resList[bestResIdxSil][1]) ## print 'writing result...' def _writeLabelAndParam(metric,resDict,paramDict): resDict2 = defaultdict(list); resDict3 = defaultdict(list) for k,v in resDict.iteritems(): resDict2[v].append(k) for k,v in resDict2.iteritems(): resDict3[k].append(len(v)) summ = sum([v[0] for v in resDict3.values()]) for k,v in resDict3.iteritems(): resDict3[k].append(float(v[0])/summ) fname = '_'.join([mode,metric]) with open(os.path.join(outDir,fname+"_bestlabels.json"),'w') as f: json.dump(resDict,f,indent=2,sort_keys=True) with open(os.path.join(outDir,fname+"_bestlabels_stat.json"),'w') as f: json.dump(resDict3,f,indent=2,sort_keys=True) with open(os.path.join(outDir,fname+"_bestparams.json"),'w') as f: json.dump(paramDict,f,indent=2,sort_keys=True) _writeLabelAndParam('calinskiharabaz',resDictCal,bestParamCal) _writeLabelAndParam('silhouette',resDictSil,bestParamSil)
if self.right: return self.right.search(value) return None if __name__ == '__main__': import time print("Beginning Tree generation.") # Generate the same tree on every workers. random.seed(314159265) exampleTree = BinaryTreeNode(0) for _ in range(128000): exampleTree.insert(random.randint(-sys.maxsize - 1, sys.maxsize)) shared.setConst(myTree=exampleTree) print("Tree generation done.") # Splits the tree in two and process the left and right branches parallely ts = time.time() presult = max( futures.map( maxTreeDepthDivide, [exampleTree.payload], parallelLevel=2, )) pts = time.time() - ts # Serial computation of tree depth ts = time.time()
shared.setConst( lettersValue={ "a": 1, "b": 3, "c": 3, "d": 2, "e": 1, "f": 4, "g": 2, "h": 4, "i": 1, "j": 8, "k": 10, "l": 1, "m": 2, "n": 1, "o": 1, "p": 3, "q": 8, "r": 1, "r": 1, "s": 1, "t": 1, "u": 1, "v": 4, "w": 10, "x": 10, "y": 10, "z": 10, } )
parser.add_argument('AdaptionParamSet') parser.add_argument('--listindex', type=int, help="index of value list") parser.add_argument( '--outputFileFolder', type=str, help="where to store the output, default is working directory") parser.add_argument('--fileName', type=str, help="name of vesselfile to optimize") goodArguments, otherArguments = parser.parse_known_args() print("running with %s" % goodArguments.AdaptionParamSet) if (str(goodArguments.AdaptionParamSet).startswith('value_li')): bla = int(goodArguments.listindex) print("list index: %i chosen" % bla) shared.setConst(parameterListIndex=bla) #adaption parameters shared.setConst(adaptionParams=goodArguments.AdaptionParamSet) #file if (goodArguments.fileName): vfile_name = goodArguments.fileName vessel_grp = "vessels" #default place for vesselgenerator else: if 0: # 2d set vfile_name = "/localdisk/thierry/vessel_trees_better/my_chosen/PSO_data_vessels-large_2d-typeE-17x1L600-sample05_adption_p_human_guess.h5" if 1: # 3d set vfile_name = "/localdisk/thierry/vessel_trees_better/my_chosen/PSO_data_vessels-large_2d-typeE-9x11L600-sample13_adption_p_human_guess.h5" vessel_grp = "adaption/vessels_after_adaption" shared.setConst(vesselInputFile=vfile_name) #group shared.setConst(vessel_grp=vessel_grp)
value = 0 for word in words: for letter in word: # shared.getConst will evaluate to the dictionary broadcasted by # the root Future value += shared.getConst('lettersValue')[letter] return value if __name__ == "__main__": # Set the values of the letters according to the language and broadcast it # This list is set at runtime import sys if len(sys.argv) > 1 and sys.argv[1] == 'francais': shared.setConst(lettersValue={'a': 1, 'b': 3, 'c': 3, 'd': 2, 'e': 1, 'f': 4, 'g': 2, 'h': 4, 'i': 1, 'j': 8, 'k':10, 'l': 1, 'm': 2, 'n': 1, 'o': 1, 'p': 3, 'q': 8, 'r': 1, 'r': 1, 's': 1, 't': 1, 'u': 1, 'v': 4, 'w':10, 'x':10, 'y':10, 'z': 10}) print("French letter values used.") else: shared.setConst(lettersValue={'a': 1, 'b': 3, 'c': 3, 'd': 2, 'e': 1, 'f': 4, 'g': 2, 'h': 4, 'i': 1, 'j': 8, 'k': 5, 'l': 1, 'm': 3, 'n': 1, 'o': 1, 'p': 3, 'q':10, 'r': 1, 'r': 1, 's': 1, 't': 1, 'u': 1, 'v': 4, 'w': 4, 'x': 8, 'y': 4, 'z': 10}) print("English letter values used.") # Get the player words (generate a list of random letters import random import string random.seed(3141592653) words = [] player_quantity = 4
def myFunc(parameter): """This function will be executed on the remote host even if it was not available at launch.""" print('Hello World from {0}!'.format(scoop.worker)) # It is possible to get a constant anywhere print(shared.getConst('myVar')[2]) # Parameters are handled as usual return parameter + 1 if __name__ == "__main__": # Populate the shared constants shared.setConst(myVar={ 1: 'First element', 2: 'Second element', 3: 'Third element', }) shared.setConst(secondVar="Hello World!") shared.setConst(myFunc=myFunc) # Use the previously defined shared function print(list(futures.map(myFunc, range(10)))) # Un-commenting the following line will give a TypeError # since re-defining a constant is not allowed. #shared.setConst(myVar="Variable re-assignation")
def configureMaster(): global CONFIGURED if CONFIGURED: return # default parameters global MYSYSTEM, NGEN, N_RANDOM_PROBLEMS, RANDOM_SEED, FILE_POPULATION, USE_RECAST, BASELINE, RUN_OPT, OUTPUT_FOLDER MYSYSTEM = 'anymal' NGEN = 20 N_RANDOM_PROBLEMS = 0 RANDOM_SEED = 5489 FILE_POPULATION = '' USE_RECAST = False BASELINE = '' RUN_OPT = True # get arguments try: opts, args = getopt.getopt(sys.argv[1:],"s:n:g:e:b:l:rc") except getopt.GetoptError: print("deap-arch-es.py") print(" -s <SYSTEM>: the sytem we plan on 'anymal', 'starleth', or 'dummy'") print(" -n <NGEN> : how many generations to run the optimization") print(" -g <NPROBS>: how many random problems to generate") print(" -e <RSEED> : random seed for problems to generate") print(" -b <NAME> : use baseline method with this name") print(" -l <FILE> : load the initial population from a file") print(" -r : whether to use recast") print(" -c : only check/evaluate population (skip optimization)") sys.exit(2) # parse arguments (normal) for opt, arg in opts: if opt == "-s": MYSYSTEM = arg elif opt == "-n": NGEN = int(arg) elif opt == "-g": N_RANDOM_PROBLEMS = int(arg) elif opt == "-e": RANDOM_SEED = int(arg) elif opt == "-b": BASELINE = arg elif opt == "-l": FILE_POPULATION = arg elif opt == "-r": USE_RECAST = True elif opt == "-c": RUN_OPT = False # output folder OUTPUT_FOLDER = "%s-%s-%s-randprobs%s-%s/" % (BASELINE if BASELINE != '' else "opt", MYSYSTEM, "recast" if USE_RECAST else "norecast", N_RANDOM_PROBLEMS, time.strftime("%Y%m%d-%H%M%S")) if not os.path.exists(OUTPUT_FOLDER): os.mkdir(OUTPUT_FOLDER) # share parameters with SCOOP workers if SCOOP_IS_RUNNING: shared.setConst(MYSYSTEM=MYSYSTEM) shared.setConst(N_RANDOM_PROBLEMS=N_RANDOM_PROBLEMS) shared.setConst(RANDOM_SEED=RANDOM_SEED) shared.setConst(USE_RECAST=USE_RECAST) shared.setConst(BASELINE=BASELINE) shared.setConst(OUTPUT_FOLDER=OUTPUT_FOLDER) # finish configureEvaluator() CONFIGURED = True
else: if self.right: return self.right.search(value) return None if __name__ == '__main__': import time print("Beginning Tree generation.") # Generate the same tree on every workers. random.seed(314159265) exampleTree = BinaryTreeNode(0) for _ in range(128000): exampleTree.insert(random.randint(-sys.maxsize - 1, sys.maxsize)) shared.setConst(myTree=exampleTree) print("Tree generation done.") # Splits the tree in two and process the left and right branches parallely ts = time.time() presult = max( futures.map( maxTreeDepthDivide, [exampleTree.payload], parallelLevel=2, ) ) pts = time.time() - ts # Serial computation of tree depth
lst[:middle], lst[middle:], ], repeat(current_depth+1), repeat(parallel_depth), ) ) else: results = [] results.append(mergesort(lst[:middle])) results.append(mergesort(lst[middle:])) return merge(*results) if __name__ == "__main__": the_list = [random.randint(-sys.maxsize - 1, sys.maxsize) for r in range(10000)] shared.setConst(the_list=the_list) ts = time.time() parallel_result = mergesort(the_list, parallel_depth=1) pts = time.time() - ts ts = time.time() serial_result = mergesort(the_list) sts = time.time() - ts print("Parallel time: {0:.5f}s".format(pts)) print("Serial time: {0:.5f}s".format(sts)) assert serial_result == parallel_result
lst[middle:], ], repeat(current_depth + 1), repeat(parallel_depth), )) else: results = [] results.append(mergesort(lst[:middle])) results.append(mergesort(lst[middle:])) return merge(*results) if __name__ == "__main__": the_list = [ random.randint(-sys.maxsize - 1, sys.maxsize) for r in range(10000) ] shared.setConst(the_list=the_list) ts = time.time() parallel_result = mergesort(the_list, parallel_depth=1) pts = time.time() - ts ts = time.time() serial_result = mergesort(the_list) sts = time.time() - ts print("Parallel time: {0:.5f}s".format(pts)) print("Serial time: {0:.5f}s".format(sts)) assert serial_result == parallel_result
version='%(prog)s version 0.9') args = parser.parse_args() return args if __name__ == "__main__": try: args = cmdlineparse() if not args.PROCESSED_ALIGNMENT_FILE: progbar = ProgressBar(100) aln_index = 0 shared.setConst(PROGBAR=progbar) shared.setConst(ALN_INDEX=aln_index) shared.setConst(ARGS=args) shared.setConst(chains_bin_dir=CHAINS_BIN_DIR) fasta_file_list = [ args.peptide_fasta_file + ".chunk" + str(num) for num in range(1, args.CPUs + 1) ] if args.COMPRESS: list_file_list = [ args.peptide_list_file + ".chunk" + str(num) + ".bz2" for num in range(1, args.CPUs + 1) ] else: list_file_list = [ args.peptide_list_file + ".chunk" + str(num)
logging.info('run time: %d sec\n', elapsed_time) logging.info('max possible fitness for the nonograms ran: %d\n') logging.info('log: %s\n', log) logging.info('pop: %s\n', utils.individual_lst_to_str(pop)) logging.info('hof: %s\n', utils.individual_lst_to_str(hof)) logging.info('stats: %s\n', stats) fitnesses = [ind.fitness.values for ind in pop] plot = Plotter(log, fitnesses) if plot_d3_fitness: plot.plot_population_tuples_3d() if plot_fitness_stats: plot.plot_fitness_stats_from_logbook() if plot_min_max_stats: plot.plot_min_max_counts() if plot_fitness_distribution_2d: plot.plot_fitness_distribution_2d() from scoop import shared from utils import load_train_and_test_sets if __name__ == '__main__': train_test_sets = load_train_and_test_sets() train_dicts = train_test_sets['train'] train_nonograms = [(d['unsolved'], d['solved']) for d in train_dicts] test_dicts = train_test_sets['test'] test_nonograms = [(d['unsolved'], d['solved']) for d in test_dicts] shared.setConst(train_nonograms=train_nonograms) main()