def funcSharedFunction():
shared.setConst(myRemoteFunc=func4)
result = True
for _ in range(100):
try:
result &= futures.submit(funcUseSharedFunction).result()
except AssertionError:
result = False
return result
def initialize(wd, rxns):
global working_dir, reactions
working_dir = wd
assert os.path.isdir(working_dir)
#set global variable here such that functions executed in the root worker have access to it.
reactions = rxns
reduce_reactions = [ReductionReaction(rxn) for rxn in rxns]
shared.setConst(reactions=reduce_reactions)
def funcSharedFunction():
shared.setConst(myRemoteFunc=func4)
result = True
for _ in range(100):
try:
result &= futures.submit(funcUseSharedFunction).result()
except AssertionError:
result = False
return result
def initialize(wd, rxns):
global working_dir, reactions
working_dir = wd
assert os.path.isdir(working_dir)
#set global variable here such that functions executed in the root worker have access to it.
reactions = rxns
reduce_reactions = [ReductionReaction(rxn) for rxn in rxns]
shared.setConst(reactions = reduce_reactions)
def funcSharedConstant():
shared.setConst(myVar={1: 'Example 1',
2: 'Example 2',
3: 'Example 3',
})
shared.setConst(secondVar="Hello World!")
result = True
for _ in range(100):
try:
result &= futures.submit(funcUseSharedConstant).result()
except AssertionError:
result = False
return result
def funcSharedConstant():
shared.setConst(myVar={1: 'Example 1',
2: 'Example 2',
3: 'Example 3',
})
shared.setConst(secondVar="Hello World!")
result = True
for _ in range(100):
try:
result &= futures.submit(funcUseSharedConstant).result()
except AssertionError:
result = False
return result
def funcUseDeletedSharedConstant():
# Tries to retrieve a deleted variable, should timeout
shared.deleteConst('myVar')
shared.deleteConst('secondVar')
timeout = 0.01
assert shared.getConst('myVar', timeout=timeout) is None
assert shared.getConst('secondVar', timeout=timeout) is None
try:
# You should not be able to reset a deleted variable
shared.setConst(myVar=42)
return False
except TypeError:
pass
return True
def simulate(config):
config_data = json.load(config)
config.close()
config = config_data
output_formatter = config["output_formatter"]
subs = {a:b for a,b in config.get("range_substitutions", [["Z",{"min":0,"max":0.5,"step":1}]])}
range_constraints = [eval("lambda {}:{}".format(",".join(subs.keys()),con)) for con in config["range_constraints"]]
def switch_weighter(vector):
return lambda x,i,j: x if vector[j] is None else vector[j].pop() if x else 0
switch = config['switch']
write_delay = config.get("write_delay",10)
V = {}
V['choice'] = matrix_map(config['choice'],lambda x,i,j:parse_expr(str(x)))
V['i_ip'] = int(config['power_iterations']["super_iterations"])
V['i_i'] = int(config['power_iterations']["iterations"])
V['iterations'] = int(config['iterations'])
if "starting_points" in config:
origional_pops = [[a/sum(s) if sum(s)>0 else 1.0/len(s) for a in s] for s in config['starting_points']]
V['origional_pops'] = [numpy.matrix([a]).transpose() for a in origional_pops]
else:
V['origional_pops'] = [numpy.matrix([[1.0/len(V['choice'])] for s in range(len(V['choice']))])]
V['extrema'] = [numpy.matrix(matrix_map(switch,switch_weighter(e))) for e in generate_switch_extrema(switch)]
V['pop_symbols'] = [sympy.symbols("s{}".format(i)) for i in range(len(V['choice']))]
V['subs_symbols'] = {a:symbols(a) for a,b in subs.iteritems()}
V['sequence'] = config.get('sequence',"1.0/(x+2)")
V['weight_function'] = config['weight_function']
shared.setConst(V=V)
output_json = []
big_list = [a for a in multi_iterate(subs) if False not in [con(**a) for con in range_constraints]]
progress = tqdm(total=len(big_list))
last_write_time = time()
def output_file(output_json):
out_file = file(output_formatter, "w")
out_file.write(json.dumps({"output":output_json},sort_keys=True).replace('{"parameters":','\n{"parameters":'))
out_file.flush()
out_file.close()
for result in futures.map_as_completed(process, big_list):
output_json.append(result)
progress.update()
if time()-last_write_time > write_delay:
output_file(output_json)
last_write_time = time()
output_file(output_json)
progress.close()
def broadcast(obj, key):
"""
Broadcasts the object across the workers using the key parameter as the key.
"""
kwargs = {key : obj}
try:
if shared.getConst(key):
logger.debug('An object with the key {} was already broadcasted.'.format(key))
else:
shared.setConst(**kwargs)
except NameError, e:
"""
Name error will be caught when the SCOOP library is not imported properly.
"""
logger.debug('SCOOP not loaded. Not broadcasting the object {}'.format(obj))
def broadcast(obj, key):
"""
Broadcasts the object across the workers using the key parameter as the key.
"""
kwargs = {key : obj}
try:
if shared.getConst(key):
logger.debug('An object with the key {} was already broadcasted.'.format(key))
else:
shared.setConst(**kwargs)
except NameError, e:
"""
Name error will be caught when the SCOOP library is not imported properly.
"""
logger.debug('SCOOP not loaded. Not broadcasting the object {}'.format(obj))
if sub:
s += "<SUB> "+ sub +" </SUB>\n "
if article_kwds:
for kwd in article_kwds:
if kwd:
s += "<KWD> "+ kwd +" </KWD>\n "
if article_title:
s += "<ATITLE> "+ article_title +" </ATITLE>\n "
if article_abstract:
s += "<ABSTRACT> "+ article_abstract +" </ABSTRACT>\n "
s += "<TEXT> "+ article_contents +" </TEXT>\n"
s += "</DOC>\n"
destination = shared.getConst('destination')
with codecs.open(destination + "/" + doc_id, encoding="utf-8", mode="w") as f:
f.write(s)
return True
if __name__ == "__main__":
datapath = sys.argv[1] + "*.nxml"
destination = sys.argv[2]
print "Datapath: ", datapath
nxmls = glob.iglob(datapath)
shared.setConst(destination=destination)
print "processed: ", len(list(futures.map(process_doc, nxmls)))
def main():
if len(sys.argv)!=4:
print 'USAGE:'
print 'python -m scoop devel.py [cloneID] [clusterDir] [outputDir]'
print 'see devel_config.py'
return
cloneID = sys.argv[1]
clusterDir = sys.argv[2]; assert clusterDir[-1]=='/',"should be ended with '/'"
baseOutDir = sys.argv[3]; assert baseOutDir[-1]!='/',"should NOT be ended with '/'"
clfParam = None
method = cfg['method']
if method=='esvm':
from esvm_config import config as clfParam
elif method=='psvm':
from psvm_config import config as clfParam
else:
print 'FATAL: unknown method'
return
outDir = os.path.join(baseOutDir,'devel-'+os.path.basename(baseOutDir))
if not(os.path.isdir(baseOutDir)): os.makedirs(baseOutDir)
if not(os.path.isdir(outDir)): os.makedirs(outDir)
## Load data ###################################################################################
dataLog = {}; dataLogFpath = os.path.join(outDir,'data_log_'+os.path.basename(baseOutDir)+'.json')
dataset = clusterDir.split('/')[-2].split('-')[-1]; dataLog['dataset'] = dataset
datasetParams = dataset.split('#')
assert datasetParams[0]=='yamanishi'
xyDevFpath = os.path.join(baseOutDir,'_'.join(['xdev','ydev','xrel','yrel']+datasetParams)+'.h5')
if os.path.exists(xyDevFpath):
print 'loading data from PREVIOUS...'
with h5py.File(xyDevFpath,'r') as f:
xdev = f['xdev'][:]
ydev = f['ydev'][:]
xrel = f['xrel'][:]
yrel = f['yrel'][:]
xrelraw = f['xrelraw'][:]
with open(dataLogFpath,'r') as f:
dataLog = yaml.load(f)
else:
print 'loading data FRESHLY...'
print 'loading cluster result...'
nUnlabels = []
statFnames = [i for i in os.listdir(clusterDir) if 'labels_stat.json' in i]
for i in statFnames:
with open(os.path.join(clusterDir,i),'r') as f: stat = yaml.load(f)
nUnlabels.append(stat['0'])
# use the cluster with minimum numbers of unlabeled samples
metric = '_'.join(statFnames[ nUnlabels.index(min(nUnlabels)) ].split('_')[0:2])
dataLog['metric'] = metric
connFpath = os.path.join(clusterDir,metric+'_labels.pkl')
with open(connFpath,'r') as f:
data = pickle.load(f)
##
print 'getting devel and release data...'
xraw = []; yraw = []
for k,v in data.iteritems():
for vv in v:
xraw.append(vv)
yraw.append(k)
devIdx = [i for i in range(len(xraw)) if yraw[i]!=0]
xdev = [xraw[i] for i in devIdx]
ydev = [yraw[i] for i in devIdx]
relIdx = [i for i in range(len(xraw)) if yraw[i]==0]
xrel = [xraw[i] for i in relIdx]
yrel = [yraw[i] for i in relIdx]
dataLog['nDevel'] = len(devIdx); dataLog['nData'] = len(yraw)
dataLog['rDevel:Data'] = dataLog['nDevel']/float(dataLog['nData'])
dataLog['nDevel(+)'] = len( [i for i in ydev if i==1] ); assert dataLog['nDevel(+)']!=0
dataLog['nDevel(-)'] = len( [i for i in ydev if i==-1] ); assert dataLog['nDevel(-)']!=0
dataLog['rDevel(+):Devel'] = float(dataLog['nDevel(+)'])/dataLog['nDevel']
dataLog['rDevel(-):Devel'] = float(dataLog['nDevel(-)'])/dataLog['nDevel']
dataLog['rDevel(+):(-)'] = float(dataLog['nDevel(+)'])/float(dataLog['nDevel(-)'])
dataLog['nRelease'] = len(relIdx);
dataLog['rRelease:Data'] = dataLog['nRelease']/float(dataLog['nData'])
##
print 'loading com, pro feature...'
krFpath = os.path.join(cfg['datasetDir'],datasetParams[0],'feature',
'klekotaroth','klekotaroth-'+datasetParams[1]+'.h5')
aacFpath = os.path.join(cfg['datasetDir'],datasetParams[0],'feature',
'amino-acid-composition','amino-acid-composition-'+datasetParams[1]+'.h5')
krDict = {}; aacDict = {}
with h5py.File(krFpath, 'r') as f:
for com in [str(i) for i in f.keys()]:
krDict[com] = f[com][:]
with h5py.File(aacFpath, 'r') as f:
for pro in [str(i) for i in f.keys()]:
aacDict[pro] = f[pro][:]
# aacDict[pro] = list( fu.map(lambda x: float('%.2f'%(x)),f[pro][:]) ) # rounding
comFeaLenOri = len(krDict.values()[0])
proFeaLenOri = len(aacDict.values()[0])
##
msg = 'extract (com,pro) feature... dims: '+str(comFeaLenOri)+','+str(proFeaLenOri)
msg += ' of '+str(len(ydev))+' and '+str(len(yrel))
print msg
sh.setConst(krDict=krDict)
sh.setConst(aacDict=aacDict)
xdevf = list( fu.map(cutil.extractComProFea,xdev) )
xrelf = list( fu.map(cutil.extractComProFea,xrel) )
##
xyDevList = cutil.divideSamples(xdevf,ydev,cfg['smoteBatchSize'])
if cfg['maxNumberOfSmoteBatch'] != 0:
xyDevList = xyDevList[0:cfg['maxNumberOfSmoteBatch']]
smoteSeed = util.seed(); dataLog['smoteSeed'] = smoteSeed
sh.setConst(smoteSeed=smoteSeed)
print 'resampling via Smote FRESHLY... '+str(len(xyDevList))+' smote(s)'+' on '+str(len(ydev))
smoteTic = time.time()
xdevfr = []; ydevr = []
xydevfrList = list( fu.map(ensembleSmote,xyDevList) )
for xdevfri,ydevri in xydevfrList:
for x in xdevfri: xdevfr.append(x.tolist())
for y in ydevri: ydevr.append(y)
assert len(xdevfr)==len(ydevr),'len(xdevfr)!=len(ydevr)'
dataLog['nSmote'] = len(xyDevList)
dataLog['nDevelResampled'] = len(ydevr)
dataLog['rDevelResampled:Data'] = dataLog['nDevelResampled']/float(dataLog['nData'])
dataLog['nDevelResampled(+)'] = len( [i for i in ydevr if i==1] )
dataLog['nDevelResampled(-)'] = len( [i for i in ydevr if i==-1] )
dataLog['rDevelResampled(+):DevelResampled'] = dataLog['nDevelResampled(+)']/float(dataLog['nDevelResampled'])
dataLog['rDevelResampled(-):DevelResampled'] = dataLog['nDevelResampled(-)']/float(dataLog['nDevelResampled'])
dataLog['rDevelResampled(+):(-)'] = dataLog['nDevelResampled(+)']/float(dataLog['nDevelResampled(-)'])
dataLog['timeSMOTE'] = str(time.time()-smoteTic)
##
print 'update xdev,ydev,xrel... '+str(np.asarray(xdevfr).shape)
xrelraw = xrel[:] # raw: feature is NOT extracted
xrel = xrelf[:]
xdev = xdevfr[:]
ydev = ydevr[:]
print 'writing updated xdev,ydev and xrel,yrel...'
with h5py.File(xyDevFpath,'w') as f:
f.create_dataset('xdev',data=xdev,dtype=np.float32)
f.create_dataset('ydev',data=ydev,dtype=np.int8)
f.create_dataset('xrel',data=xrel,dtype=np.float32)
f.create_dataset('yrel',data=yrel,dtype=np.int8)
f.create_dataset('xrelraw',data=xrelraw)
print 'writing dataLog...'
dataLog['nCom'] = len(krDict)
dataLog['nPro'] = len(aacDict)
with open(dataLogFpath,'w') as f:
json.dump(dataLog,f,indent=2,sort_keys=True)
## TUNE+TRAIN+TEST #############################################################################
devLog = {}
devSeed = util.seed(); dataLog['devSeed'] = devSeed
tag = '_'.join([method+'#'+cloneID,dataset,util.tag()])
## split devel dataset
msg = ' '.join( ['devel',dataset,cloneID])
xtr,xte,ytr,yte = tts(xdev,ydev,test_size=cfg['testSize'],
random_state=devSeed,stratify=ydev)
if cfg['maxTestingSamples']>0:
chosenIdx = np.random.randint(len(xte),size=cfg['maxTestingSamples'])
xte = [xte[i] for i in chosenIdx]; yte = [yte[i] for i in chosenIdx]
devLog['nTraining'] = len(xtr)
devLog['nTraining(+)'] = len([i for i in ytr if i==1])
devLog['nTraining(-)'] = len([i for i in ytr if i==-1])
devLog['rTraining(+):(-)'] = devLog['nTraining(+)']/float(devLog['nTraining(-)'])
devLog['rTraining:Devel'] = devLog['nTraining']/float(dataLog['nDevelResampled'])
devLog['nTesting'] = len(xte)
devLog['nTesting(+)'] = len([i for i in yte if i==1])
devLog['nTesting(-)'] = len([i for i in yte if i==-1])
devLog['rTesting(+):(-)'] = devLog['nTesting(+)']/float(devLog['nTesting(-)'])
devLog['rTesting:Devel'] = devLog['nTesting']/float(dataLog['nDevelResampled'])
## tuning
clf = None
if method=='esvm':
clf = eSVM(simMat=None)
elif method=='psvm':
clf = svm.SVC(kernel=clfParam['kernel'],probability=True)
## training
print msg+': fitting nTr= '+str(len(ytr))
trTic = time.time()
if method=='esvm':
clf.fit(xtr,ytr)
devLog['labels'] = clf.labels()
devLog['nSVM'] = clf.nSVM()
devLog['xtrDimAllBatches'] = clf.xtrDimAllBatches()
elif method=='psvm':
if cfg['method']['kernel']=='precomputed':
assert False
# simMatTr = cutil.makeComProKernelMatFromSimMat(xtr,xtr,simMat)
# clf.fit(simMatTr,ytr)
else:
clf.fit(xtr,ytr)
devLog['labels'] = clf.classes_.tolist()
devLog['timeTraining'] = str(time.time()-trTic)
## testing
print msg+': predicting nTe= '+str(len(yte))
teTic = time.time()
if method=='esvm':
ypred,yscore = clf.predict(xte)
elif method=='psvm':
if cfg['method']['kernel']=='precomputed':
assert False
# simMatTe = cutil.makeComProKernelMatFromSimMat(xte,xtr,simMat)
# ypred = clf.predict(simMatTe)
# yscore = clf.predict_proba(simMatTe)
else:
ypred = clf.predict(xte)
yscore = clf.predict_proba(xte)
yscore = [max(i.tolist()) for i in yscore]
devLog['timeTesting'] = str(time.time()-teTic)
## TEST RELEASE ################################################################################
print msg+': predicting RELEASE n= '+str(len(yrel))
relTic = time.time()
if method=='esvm':
yrel,yrelscore = clf.predict(xrel)
elif method=='psvm':
if cfg['method']['kernel']=='precomputed':
assert False
# simMatTe = cutil.makeComProKernelMatFromSimMat(xrel,xtr,simMat)
# yrel = clf.predict(simMatTe)
# yrelscore = clf.predict_proba(simMatTe)
else:
yrel = clf.predict(xrel)
yrelscore = clf.predict_proba(xrel)
yrelscore = [max(i.tolist()) for i in yrelscore]
devLog['timeRelease'] = str(time.time()-relTic)
## WRITE RESULT ################################################################################
result = {'yte':yte,'ypred':ypred,'yscore':yscore,
'xrelraw':xrelraw,'yrel':yrel,'yrelscore':yrelscore}
print 'writing prediction...'
with h5py.File(os.path.join(outDir,'result_'+tag+'.h5'),'w') as f:
for k,v in result.iteritems():
if 'raw' in k:
f.create_dataset(k,data=v)
else:
dt = np.int8
if 'score' in k: dt = np.float32
f.create_dataset(k,data=v,dtype=dt)
##
print 'writing devLog...'
devLog['clfParam'] = clfParam
devLog['devParam'] = cfg
with open(os.path.join(outDir,'devLog_'+tag+'.json'),'w') as f:
json.dump(devLog,f,indent=2,sort_keys=True)
nb_runs, generations, mean_fit_mins, mean_fit_avg, mean_duration_mins, mean_duration_maxs = runs_results
fig, ax1 = plt.subplots()
fig.suptitle(f"Mean results over {nb_runs} runs, graph: '{graph_name}', duration constraint: {max_duration}")
line1 = ax1.plot(generations, mean_fit_mins, "b-", label="Minimum Cost")
line3 = ax1.plot(generations, mean_fit_avg, "g-", label= "Average Cost")
ax1.set_xlabel("Generation")
ax1.set_ylabel("Cost", color="b")
for tl in ax1.get_yticklabels():
tl.set_color("b")
ax2 = ax1.twinx()
line2 = ax2.plot(generations, mean_duration_mins, "r-", label="Minimum Duration")
line4 = ax2.plot(generations, mean_duration_maxs, "y-", label="Maximum Duration")
ax2.set_ylabel("Duration", color="r")
for tl in ax2.get_yticklabels():
tl.set_color("r")
lns = line1 + line2 + line3 + line4
labs = [l.get_label() for l in lns]
ax1.legend(lns, labs, loc="upper right")
OUTPUT_DIR = os.environ.get("AZ_BATCH_TASK_DIR", ".")
OUTPUT_FILE = os.path.join(OUTPUT_DIR, f"{graph_name}_{nb_runs}_runs.png")
plt.savefig(OUTPUT_FILE)
if __name__ == "__main__":
graph_name = "MediumComplex"
task_graph, MAXIMUM_DURATION = construct_graph(graph_name)
shared.setConst(graph_name=graph_name, graph=task_graph, max_duration=MAXIMUM_DURATION)
plot_runs_mean(multiple_runs_mean(10))
def main():
if len(sys.argv)!=6:
print 'USAGE:'
print 'python -m scoop cluster.py [method] [nIter] [dataset#x] [compound/protein] [outDir]'
return
method = sys.argv[1]
nIter = int(sys.argv[2])
dataset = sys.argv[3]
mode = sys.argv[4]
outDir = sys.argv[5]
outDir = os.path.join(outDir,
'-'.join([method+'#'+str(nIter),dataset,mode,util.tag()]))
os.makedirs(outDir)
##
print 'loading data...'
dParam = dataset.split('#')
disMat = None; iList = None
if dParam[0]=='yamanishi':
dataDir = os.path.join(DATASET_DIR,dParam[0])
simDict = yam.loadKernel2(mode,dParam[1],os.path.join(dataDir,'similarity-mat'))
simMat,iList = util.makeKernelMatrix(simDict)
disMat = util.kernel2distanceMatrix('naive',simMat)
else:
assert False,'FATAL: unknown dataset'
##
print 'clustering...'
paramList = []
if method=='dbscan':
epsMin,epsMax = [0.0,1.0]
nMin,nMax = [1,len(iList)]
for i in range(nIter):
eps = np.random.uniform(epsMin,epsMax,1)[0]
n = np.random.randint(nMin,nMax,1)[0]
paramList.append( dict(eps=eps,min_samples=n) )
else:
assert False
sh.setConst(method=method)
sh.setConst(mat=disMat)
resList = list( fu.map(_cluster,paramList) )
bestResIdxCal = _getBestResultIdx(resList,'calinski_harabaz_score')
bestResIdxSil = _getBestResultIdx(resList,'silhouette_score')
resDictCal = dict( zip(iList,resList[bestResIdxCal][0]) )
resDictSil = dict( zip(iList,resList[bestResIdxSil][0]) )
bestParamCal = dict(param=paramList[bestResIdxCal],
score=resList[bestResIdxCal][1])
bestParamSil = dict(param=paramList[bestResIdxSil],
score=resList[bestResIdxSil][1])
##
print 'writing result...'
def _writeLabelAndParam(metric,resDict,paramDict):
resDict2 = defaultdict(list); resDict3 = defaultdict(list)
for k,v in resDict.iteritems(): resDict2[v].append(k)
for k,v in resDict2.iteritems(): resDict3[k].append(len(v))
summ = sum([v[0] for v in resDict3.values()])
for k,v in resDict3.iteritems(): resDict3[k].append(float(v[0])/summ)
fname = '_'.join([mode,metric])
with open(os.path.join(outDir,fname+"_bestlabels.json"),'w') as f:
json.dump(resDict,f,indent=2,sort_keys=True)
with open(os.path.join(outDir,fname+"_bestlabels_stat.json"),'w') as f:
json.dump(resDict3,f,indent=2,sort_keys=True)
with open(os.path.join(outDir,fname+"_bestparams.json"),'w') as f:
json.dump(paramDict,f,indent=2,sort_keys=True)
_writeLabelAndParam('calinskiharabaz',resDictCal,bestParamCal)
_writeLabelAndParam('silhouette',resDictSil,bestParamSil)
if self.right:
return self.right.search(value)
return None
if __name__ == '__main__':
import time
print("Beginning Tree generation.")
# Generate the same tree on every workers.
random.seed(314159265)
exampleTree = BinaryTreeNode(0)
for _ in range(128000):
exampleTree.insert(random.randint(-sys.maxsize - 1, sys.maxsize))
shared.setConst(myTree=exampleTree)
print("Tree generation done.")
# Splits the tree in two and process the left and right branches parallely
ts = time.time()
presult = max(
futures.map(
maxTreeDepthDivide,
[exampleTree.payload],
parallelLevel=2,
))
pts = time.time() - ts
# Serial computation of tree depth
ts = time.time()
shared.setConst(
lettersValue={
"a": 1,
"b": 3,
"c": 3,
"d": 2,
"e": 1,
"f": 4,
"g": 2,
"h": 4,
"i": 1,
"j": 8,
"k": 10,
"l": 1,
"m": 2,
"n": 1,
"o": 1,
"p": 3,
"q": 8,
"r": 1,
"r": 1,
"s": 1,
"t": 1,
"u": 1,
"v": 4,
"w": 10,
"x": 10,
"y": 10,
"z": 10,
}
)
parser.add_argument('AdaptionParamSet')
parser.add_argument('--listindex', type=int, help="index of value list")
parser.add_argument(
'--outputFileFolder',
type=str,
help="where to store the output, default is working directory")
parser.add_argument('--fileName',
type=str,
help="name of vesselfile to optimize")
goodArguments, otherArguments = parser.parse_known_args()
print("running with %s" % goodArguments.AdaptionParamSet)
if (str(goodArguments.AdaptionParamSet).startswith('value_li')):
bla = int(goodArguments.listindex)
print("list index: %i chosen" % bla)
shared.setConst(parameterListIndex=bla)
#adaption parameters
shared.setConst(adaptionParams=goodArguments.AdaptionParamSet)
#file
if (goodArguments.fileName):
vfile_name = goodArguments.fileName
vessel_grp = "vessels" #default place for vesselgenerator
else:
if 0: # 2d set
vfile_name = "/localdisk/thierry/vessel_trees_better/my_chosen/PSO_data_vessels-large_2d-typeE-17x1L600-sample05_adption_p_human_guess.h5"
if 1: # 3d set
vfile_name = "/localdisk/thierry/vessel_trees_better/my_chosen/PSO_data_vessels-large_2d-typeE-9x11L600-sample13_adption_p_human_guess.h5"
vessel_grp = "adaption/vessels_after_adaption"
shared.setConst(vesselInputFile=vfile_name)
#group
shared.setConst(vessel_grp=vessel_grp)
value = 0
for word in words:
for letter in word:
# shared.getConst will evaluate to the dictionary broadcasted by
# the root Future
value += shared.getConst('lettersValue')[letter]
return value
if __name__ == "__main__":
# Set the values of the letters according to the language and broadcast it
# This list is set at runtime
import sys
if len(sys.argv) > 1 and sys.argv[1] == 'francais':
shared.setConst(lettersValue={'a': 1, 'b': 3, 'c': 3, 'd': 2, 'e': 1,
'f': 4, 'g': 2, 'h': 4, 'i': 1, 'j': 8, 'k':10, 'l': 1, 'm': 2, 'n': 1,
'o': 1, 'p': 3, 'q': 8, 'r': 1, 'r': 1, 's': 1, 't': 1, 'u': 1, 'v': 4,
'w':10, 'x':10, 'y':10, 'z': 10})
print("French letter values used.")
else:
shared.setConst(lettersValue={'a': 1, 'b': 3, 'c': 3, 'd': 2, 'e': 1,
'f': 4, 'g': 2, 'h': 4, 'i': 1, 'j': 8, 'k': 5, 'l': 1, 'm': 3, 'n': 1,
'o': 1, 'p': 3, 'q':10, 'r': 1, 'r': 1, 's': 1, 't': 1, 'u': 1, 'v': 4,
'w': 4, 'x': 8, 'y': 4, 'z': 10})
print("English letter values used.")
# Get the player words (generate a list of random letters
import random
import string
random.seed(3141592653)
words = []
player_quantity = 4
def myFunc(parameter):
"""This function will be executed on the remote host even if it was not
available at launch."""
print('Hello World from {0}!'.format(scoop.worker))
# It is possible to get a constant anywhere
print(shared.getConst('myVar')[2])
# Parameters are handled as usual
return parameter + 1
if __name__ == "__main__":
# Populate the shared constants
shared.setConst(myVar={
1: 'First element',
2: 'Second element',
3: 'Third element',
})
shared.setConst(secondVar="Hello World!")
shared.setConst(myFunc=myFunc)
# Use the previously defined shared function
print(list(futures.map(myFunc, range(10))))
# Un-commenting the following line will give a TypeError
# since re-defining a constant is not allowed.
#shared.setConst(myVar="Variable re-assignation")
def configureMaster():
global CONFIGURED
if CONFIGURED:
return
# default parameters
global MYSYSTEM, NGEN, N_RANDOM_PROBLEMS, RANDOM_SEED, FILE_POPULATION, USE_RECAST, BASELINE, RUN_OPT, OUTPUT_FOLDER
MYSYSTEM = 'anymal'
NGEN = 20
N_RANDOM_PROBLEMS = 0
RANDOM_SEED = 5489
FILE_POPULATION = ''
USE_RECAST = False
BASELINE = ''
RUN_OPT = True
# get arguments
try:
opts, args = getopt.getopt(sys.argv[1:],"s:n:g:e:b:l:rc")
except getopt.GetoptError:
print("deap-arch-es.py")
print(" -s <SYSTEM>: the sytem we plan on 'anymal', 'starleth', or 'dummy'")
print(" -n <NGEN> : how many generations to run the optimization")
print(" -g <NPROBS>: how many random problems to generate")
print(" -e <RSEED> : random seed for problems to generate")
print(" -b <NAME> : use baseline method with this name")
print(" -l <FILE> : load the initial population from a file")
print(" -r : whether to use recast")
print(" -c : only check/evaluate population (skip optimization)")
sys.exit(2)
# parse arguments (normal)
for opt, arg in opts:
if opt == "-s":
MYSYSTEM = arg
elif opt == "-n":
NGEN = int(arg)
elif opt == "-g":
N_RANDOM_PROBLEMS = int(arg)
elif opt == "-e":
RANDOM_SEED = int(arg)
elif opt == "-b":
BASELINE = arg
elif opt == "-l":
FILE_POPULATION = arg
elif opt == "-r":
USE_RECAST = True
elif opt == "-c":
RUN_OPT = False
# output folder
OUTPUT_FOLDER = "%s-%s-%s-randprobs%s-%s/" % (BASELINE if BASELINE != '' else "opt", MYSYSTEM, "recast" if USE_RECAST else "norecast", N_RANDOM_PROBLEMS, time.strftime("%Y%m%d-%H%M%S"))
if not os.path.exists(OUTPUT_FOLDER):
os.mkdir(OUTPUT_FOLDER)
# share parameters with SCOOP workers
if SCOOP_IS_RUNNING:
shared.setConst(MYSYSTEM=MYSYSTEM)
shared.setConst(N_RANDOM_PROBLEMS=N_RANDOM_PROBLEMS)
shared.setConst(RANDOM_SEED=RANDOM_SEED)
shared.setConst(USE_RECAST=USE_RECAST)
shared.setConst(BASELINE=BASELINE)
shared.setConst(OUTPUT_FOLDER=OUTPUT_FOLDER)
# finish
configureEvaluator()
CONFIGURED = True
else:
if self.right:
return self.right.search(value)
return None
if __name__ == '__main__':
import time
print("Beginning Tree generation.")
# Generate the same tree on every workers.
random.seed(314159265)
exampleTree = BinaryTreeNode(0)
for _ in range(128000):
exampleTree.insert(random.randint(-sys.maxsize - 1, sys.maxsize))
shared.setConst(myTree=exampleTree)
print("Tree generation done.")
# Splits the tree in two and process the left and right branches parallely
ts = time.time()
presult = max(
futures.map(
maxTreeDepthDivide,
[exampleTree.payload],
parallelLevel=2,
)
)
pts = time.time() - ts
# Serial computation of tree depth
lst[:middle],
lst[middle:],
],
repeat(current_depth+1),
repeat(parallel_depth),
)
)
else:
results = []
results.append(mergesort(lst[:middle]))
results.append(mergesort(lst[middle:]))
return merge(*results)
if __name__ == "__main__":
the_list = [random.randint(-sys.maxsize - 1, sys.maxsize) for r in range(10000)]
shared.setConst(the_list=the_list)
ts = time.time()
parallel_result = mergesort(the_list, parallel_depth=1)
pts = time.time() - ts
ts = time.time()
serial_result = mergesort(the_list)
sts = time.time() - ts
print("Parallel time: {0:.5f}s".format(pts))
print("Serial time: {0:.5f}s".format(sts))
assert serial_result == parallel_result
lst[middle:],
],
repeat(current_depth + 1),
repeat(parallel_depth),
))
else:
results = []
results.append(mergesort(lst[:middle]))
results.append(mergesort(lst[middle:]))
return merge(*results)
if __name__ == "__main__":
the_list = [
random.randint(-sys.maxsize - 1, sys.maxsize) for r in range(10000)
]
shared.setConst(the_list=the_list)
ts = time.time()
parallel_result = mergesort(the_list, parallel_depth=1)
pts = time.time() - ts
ts = time.time()
serial_result = mergesort(the_list)
sts = time.time() - ts
print("Parallel time: {0:.5f}s".format(pts))
print("Serial time: {0:.5f}s".format(sts))
assert serial_result == parallel_result
version='%(prog)s version 0.9')
args = parser.parse_args()
return args
if __name__ == "__main__":
try:
args = cmdlineparse()
if not args.PROCESSED_ALIGNMENT_FILE:
progbar = ProgressBar(100)
aln_index = 0
shared.setConst(PROGBAR=progbar)
shared.setConst(ALN_INDEX=aln_index)
shared.setConst(ARGS=args)
shared.setConst(chains_bin_dir=CHAINS_BIN_DIR)
fasta_file_list = [
args.peptide_fasta_file + ".chunk" + str(num)
for num in range(1, args.CPUs + 1)
]
if args.COMPRESS:
list_file_list = [
args.peptide_list_file + ".chunk" + str(num) + ".bz2"
for num in range(1, args.CPUs + 1)
]
else:
list_file_list = [
args.peptide_list_file + ".chunk" + str(num)
logging.info('run time: %d sec\n', elapsed_time)
logging.info('max possible fitness for the nonograms ran: %d\n')
logging.info('log: %s\n', log)
logging.info('pop: %s\n', utils.individual_lst_to_str(pop))
logging.info('hof: %s\n', utils.individual_lst_to_str(hof))
logging.info('stats: %s\n', stats)
fitnesses = [ind.fitness.values for ind in pop]
plot = Plotter(log, fitnesses)
if plot_d3_fitness:
plot.plot_population_tuples_3d()
if plot_fitness_stats:
plot.plot_fitness_stats_from_logbook()
if plot_min_max_stats:
plot.plot_min_max_counts()
if plot_fitness_distribution_2d:
plot.plot_fitness_distribution_2d()
from scoop import shared
from utils import load_train_and_test_sets
if __name__ == '__main__':
train_test_sets = load_train_and_test_sets()
train_dicts = train_test_sets['train']
train_nonograms = [(d['unsolved'], d['solved']) for d in train_dicts]
test_dicts = train_test_sets['test']
test_nonograms = [(d['unsolved'], d['solved']) for d in test_dicts]
shared.setConst(train_nonograms=train_nonograms)
main()
