def scatter_processor (configs, pair, rank):
log, xlim, ylim, num_instances = configs['worker_log'], 0, 0, 0
if sum([plot['processing_bit'] for plot in pair]) >= 1:
xlim, ylim, num_instances = master.calculate_Axis_limits (pair, configs)
for plot in pair:
if plot['processing_bit'] == 1:
mywrite(log, "\n\t\t\t\tworker #"+str(rank)+": applying scatter on "+plot['file_path'].split('/')[-1])
plot['title'] = plot['title']+", "+util.pf(float(num_instances))+" instances]"
mode, cbar_label = 'percentage' , "Frequency (%)" # defaults
if configs['mode'].strip() == 'count':
cbar_label = "Frequency (log_2)"
mode = 'count'
#-------------------------------------
df = open (plot['file_path'], 'r')
#-------------------------------------
instances_so_far, fraction_freq = 0, {}
while instances_so_far < num_instances:
Bs_Ds = {}
next(df) # skip over objects
Bs = [int(s) for s in next(df).split()]
Ds = [int(s) for s in next(df).split()]
next(df) #skip over Xs (solution vector)
next(df) #skip over coresize/exec_time
instances_so_far += 1
assert len(Bs) == len(Ds)
if len(Bs)>0:
Bs_Ds = Counter ([bd for bd in zip(Bs, Ds)])
for obj in Bs_Ds.keys():
if obj not in fraction_freq.keys():
fraction_freq[obj] = {'avg_frac_sofar':0.0,'count_sofar':0.0}
fraction = sdiv(float(Bs_Ds[obj]), len(Bs))
fraction_freq[obj]['avg_frac_sofar'] = ((fraction_freq[obj]['avg_frac_sofar']*fraction_freq[obj]['count_sofar']) + fraction ) / (fraction_freq[obj]['count_sofar'] + 1)
fraction_freq[obj]['count_sofar'] = fraction_freq[obj]['count_sofar'] + 1
del Bs_Ds
Bs, Ds, frequency = [],[],[]
for obj in fraction_freq.keys():
Bs.append(obj[0])
Ds.append(obj[1])
if mode == 'percentage':
frequency.append(fraction_freq[obj]['avg_frac_sofar']*100) # convert it to %
else:
frequency.append(math.log (fraction_freq[obj]['count_sofar'], 2))
if rank == 1 and mode == 'percentage':
master.print_freqs(fraction_freq, plot['file_path'], configs['worker_log'])
plot['data'], plot['worker_configs'], plot['rank'] = {'Bs':Bs, 'Ds':Ds, 'frequency':frequency, 'xlim':xlim, 'ylim':ylim, 'cbar_label':cbar_label}, configs, rank
##########################
master.dump(plot)
##########################
else:
mywrite(log,"\n\t\t\t\tworker #"+str(rank)+": skipping: "+plot['file_path'].split('/')[-1])
def ExecStats_processor(configs, pair, rank):
log = configs['worker_log']
for plot in pair:
if plot['processing_bit'] == 1:
##############################################################
cruncher, data, num_instances = ExecStats_init(plot, configs)
##############################################################
mywrite(log, "\n\t\t\t\tworker #"+str(rank)+": applying "+cruncher.__name__+" on "+plot['file_path'][0:90]+"...")
for PT in data.keys():
####################################################################################
num_instances = min(sum([1 for line in open (data[PT]['csv'], 'r')]), num_instances)
####################################################################################
df = open (data[PT]['csv'], 'r')
instances_so_far = 0
optimized_over_instances = 0
while instances_so_far < num_instances:
stats = [float(s) for s in next(df).strip().split()]
# each line in csv (=stats) is: 0-num_white_genes, 1-num_black_genes, 2-num_grey_genes, 3-TOTAL_Bin, 4-TOTAL_Din, 5-TOTAL_Bou, 6-TOTAL_Dou, 7-core_size, 8-Ctime_s, 9-Ctime_ms, 10-PythonTime
# we only consider instances that were optimized over, i.e. grey genes>0, coresize & exec_time > 0 (see minknap_execstats.c)
if stats[8]>=0:#stats[7]>0 and stats[8] >0:
cruncher(stats, data[PT]['STATS'])
optimized_over_instances += 1
instances_so_far += 1
cruncher(None, data[PT]['STATS'], True)
data[PT]['num_instances']=int(instances_so_far)
print('\n'+data[PT]['csv'].split('/')[-1]+': optimized_over_instances = '+str(optimized_over_instances))
data[PT]['bar_text'] = '$'+util.pf(optimized_over_instances)+'/'+util.pf(float(instances_so_far))+'$'
plot['title'] = plot['title']+", ACTUAL: "+util.pf(float(instances_so_far))+" instances]"
plot['data'], plot['worker_configs'], plot['rank'] = data , configs, rank
#with open('dump'+str(rank)+'-'+str(plot['pos']),'wb') as f:
# import pickle
# pickle.dump(plot,f)
# f.close()
##########################
master.dump(plot)
##########################
else:
mywrite(log,"\n\t\t\t\tworker #"+str(rank)+": skipping "+plot['file_path'][0:90])
def wheel_processor (configs, pair, rank):
log, num_instances = configs['worker_log'], 0
if sum(plot['processing_bit'] for plot in pair) >= 1:
num_instances = master.max_instances(pair, configs, 5)
for plot in pair:
if plot['processing_bit'] == 1:
cruncher = None
if 'cruncher' in configs.keys():
if configs['cruncher'] in globals().keys():
cruncher = globals()[configs['cruncher']]
else:
cruncher = cruncher1 # default
else:
cruncher = cruncher1 # default
mywrite(log, "\n\t\t\t\tworker #"+str(rank)+": applying "+cruncher.__name__+" on "+plot['file_path'].split('/')[-1])
plot['title'] = plot['title']+", "+util.pf(float(num_instances))+" instances]"
#file_no, coords, pos, prefix, file_path, title, processing_bit = plot['file_no'], ['coords'], plot['pos'], plot['prefix'], plot['file_path'], plot['title'], plot['processing_bit']
slices = copy.deepcopy(configs['slices'])
for key in slices['segments'].keys(): #assert slices
if cruncher==cruncher4:
assert slices['segments'][key]['degree_freq'] == {}
else:
assert slices['segments'][key]['count'] == 0
assert slices['segments'][key]['fractions'] == []
#-------------------------------------
df = open (plot['file_path'], 'r')
#-------------------------------------
instances_so_far = 0
while instances_so_far < num_instances:
Gs = [g for g in next(df).strip().split()]
Bs = [int(s) for s in next(df).strip().split()]
Ds = [int(s) for s in next(df).strip().split()]
Xs = [int(x) for x in next(df).strip().split()] #skip over Xs (solution vector)
next(df) #skip over coresize/exec_time
instances_so_far += 1
assert len(Bs) == len(Ds)
if len(Bs)>0:
cruncher(Gs,Bs,Ds,Xs,slices)
plot['data'], plot['worker_configs'], plot['rank'] = slices, configs, rank
##########################
master.dump(plot)
##########################
else:
mywrite(log,"\n\t\t\t\tworker #"+str(rank)+": skipping: "+plot['file_path'].split('/')[-1])
def ETX_processor (configs, pair, rank):
log = configs['worker_log']
for plot in pair:
if plot['processing_bit'] == 1:
############################################
cruncher, data = ETX_init(plot, configs)
############################################
mywrite(log, "\n\t\t\t\tworker #"+str(rank)+": applying "+cruncher.__name__+" on "+plot['file_path'][0:90]+"...")
for PT in data.keys():
df = open (data[PT]['csv'], 'r')
num_instances = min(int(sum([1 for line in df])/5), configs['max_instances'])
df.close()
df = open (data[PT]['csv'], 'r')
instances_so_far = 0
while instances_so_far < num_instances:
Gs = [g for g in next(df).strip().split()]
Bs = [int(s) for s in next(df).strip().split()]
Ds = [int(s) for s in next(df).strip().split()]
Xs = [int(x) for x in next(df).strip().split()] #skip over Xs (solution vector)
next(df) #skip over coresize/exec_time
instances_so_far += 1
assert len(Bs) == len(Ds)
if len(Bs)>0:
cruncher(Gs,Bs,Ds,Xs,data[PT]['STATS'])
norm = False
try:
if str(configs['normalize2unity']).strip().lower() == 'true':
norm = True
except:
pass # 'normalize2unity' was not provided
if norm: # default = True
cruncher(None,None,None,None,data[PT]['STATS'],final=True) # normalize collected stats to add up to unity (100$)
data[PT]['num_instances']=int(instances_so_far)
plot['title'] = plot['title']+", "+util.pf(float(data[PT]['num_instances']))+" instances]"
plot['data'], plot['worker_configs'], plot['rank'] = data , configs, rank
#with open('dump'+str(rank)+'-'+str(plot['pos']),'wb') as f:
# import pickle
# pickle.dump(plot,f)
# f.close()
##########################
master.dump(plot)
##########################
else:
mywrite(log,"\n\t\t\t\tworker #"+str(rank)+": skipping "+plot['file_path'][0:90])
def sSpace_processor(configs, pair, rank):
log = configs['worker_log']
# you can't do this cos max_instances() expects files not dirs
#if sum(plot['processing_bit'] for plot in pair) >= 1:
# num_instances = master.max_instances(pair, configs, 5)
for plot in pair:
if plot['processing_bit'] == 1:
############################################
cruncher, data = sSpace_init(plot, configs)
############################################
mywrite(log, "\n\t\t\t\tworker #"+str(rank)+": applying "+cruncher.__name__+" on "+plot['file_path'][0:90]+"...")
for PT in data.keys():
df = open (data[PT]['csv'], 'r')
num_instances = min(int(sum([1 for line in df])/5), configs['max_instances'])
df.close()
df = open (data[PT]['csv'], 'r')
instances_so_far = 0
sSpace = copy.deepcopy(configs['slices'])
while instances_so_far < num_instances:
try:
#next(df) #skip over Gs
Gs = [str(g).strip() for g in next(df).strip().split()]
Bs = [int(s) for s in next(df).strip().split()]
Ds = [int(s) for s in next(df).strip().split()]
next(df) #skip over Xs
next(df) #skip over coresize/exec_time
instances_so_far += 1
assert len(Bs) == len(Ds)
if len(Bs)>0:
cruncher(Gs,Bs,Ds,sSpace)
except:
print ("\nError in sSpace_processor() inside crunchers.py ("+str(instances_so_far)+"): "+str(data[PT]['csv']))
data[PT]['num_instances']=int(instances_so_far)
data[PT]['STATS'] = sSpace
del sSpace
plot['title'] = plot['title']+", "+util.pf(float(instances_so_far))+" instances]"
plot['data'], plot['worker_configs'], plot['rank'] = data , configs, rank
##########################
master.dump(plot)
##########################
else:
mywrite(log,"\n\t\t\t\tworker #"+str(rank)+": skipping "+plot['file_path'][0:90])