def main():
print('test')
bar = progressbar.progbar(10,clear_display=True)
for i in range(10):
time.sleep(0.1)
bar.update(i)
del bar
print('test2')
bar = progressbar.progbar(10,clear_display=False)
for i in range(10):
time.sleep(0.1)
bar.update(i)
del bar
print('test3')
bar = progressbar.progbar(10,clear_display=True)
for j in range(10):
prog = progressbar.progbar(10,clear_display=True)
for i in range(10):
time.sleep(0.1)
prog.update(i)
del prog
bar.update(j)
del bar
def update_card_status(self, cardref=-1, number_of_event=-1):
"""creates the directory for standard run
number_of_event[-1]:-1:automatic (number of event of cardref if !=-1, highest existing event in other case
"""
#check if all card have a directory in P_
list_card = [num for num in self.MWparam.actif_param]
#progress bar
pbar = progressbar.progbar(
'update Pdir',
len(list_card) * len(self.MWparam.P_listdir))
for directory in self.MWparam.P_listdir:
self.dir_type = directory[0]
self.Sdir_pos = './SubProcesses/' + directory
for card in list_card:
self.create_one_P_dir(card, remove_old=0)
pbar.update()
pbar.finish()
#find cardref and number_of_event if not defined
if number_of_event == -1 and cardref == -1: #search the card with the maximum of event defined
self.dir_type = 'M'
self.Sdir_pos = './SubProcesses/' + self.MWparam.MW_listdir[0]
for card in self.MWparam.actif_param:
event = self.find_exist_event(card)
if event >= number_of_event:
cardref = card
number_of_event = event
elif cardref == -1:
ref_num = number_of_event - 1 #-1 due to the starting at zero
for card in self.MWparam.actif_param:
if self.find_exist_event(card) >= ref_num:
cardref = card
if cardref == -1 or number_of_event == -1:
sys.exit(
'impossible to update card and/or number of event not defined')
else:
self.ref_card = cardref
#update events
#progress bar
list_card = self.MWparam.actif_param
list_event = range(0, number_of_event)
if (len(list_card)) * (len(list_event)) * len(
self.MWparam.MW_listdir) > 0:
pbar = progressbar.progbar('update MWdir',
(len(list_card)) * (len(list_event)) *
len(self.MWparam.P_listdir))
for dir in self.MWparam.MW_listdir:
self.dir_type = dir[0]
self.Sdir_pos = './SubProcesses/' + dir
#then pass in update mode for all the card
for card in list_card:
for event in list_event:
self.create_one_M_dir(card, event, remove_old=0)
pbar.update()
pbar.finish()
def update_card_status(self,cardref=-1,number_of_event=-1):
"""creates the directory for standard run
number_of_event[-1]:-1:automatic (number of event of cardref if !=-1, highest existing event in other case
"""
#check if all card have a directory in P_
list_card=[num for num in self.MWparam.actif_param]
#progress bar
pbar = progressbar.progbar('update Pdir',len(list_card)*len(self.MWparam.P_listdir))
for directory in self.MWparam.P_listdir:
self.dir_type=directory[0]
self.Sdir_pos='./SubProcesses/'+directory
for card in list_card:
self.create_one_P_dir(card,remove_old=0)
pbar.update()
pbar.finish()
#find cardref and number_of_event if not defined
if number_of_event==-1 and cardref==-1: #search the card with the maximum of event defined
self.dir_type='M'
self.Sdir_pos='./SubProcesses/'+self.MWparam.MW_listdir[0]
for card in self.MWparam.actif_param:
event=self.find_exist_event(card)
if event>=number_of_event:
cardref=card
number_of_event=event
elif cardref==-1:
ref_num=number_of_event-1 #-1 due to the starting at zero
for card in self.MWparam.actif_param:
if self.find_exist_event(card)>=ref_num:
cardref=card
if cardref==-1 or number_of_event==-1:
sys.exit('impossible to update card and/or number of event not defined')
else:
self.ref_card=cardref
#update events
#progress bar
list_card=self.MWparam.actif_param
list_event=range(0,number_of_event)
if (len(list_card))*(len(list_event))*len(self.MWparam.MW_listdir)>0:
pbar = progressbar.progbar('update MWdir',(len(list_card))*(len(list_event))*len(self.MWparam.P_listdir))
for dir in self.MWparam.MW_listdir:
self.dir_type=dir[0]
self.Sdir_pos='./SubProcesses/'+dir
#then pass in update mode for all the card
for card in list_card:
for event in list_event:
self.create_one_M_dir(card,event,remove_old=0)
pbar.update()
pbar.finish()
def update_card_status(self, cardref=-1):
"""creates the directory for standard run """
#check if all card have a directory in P_
list_card = [num - 1 for num in self.MWparam.actif_param]
#progress bar
pbar = progressbar.progbar(
'update Pdir',
len(list_card) * len(self.MWparam.P_listdir))
for directory in self.MWparam.P_listdir:
self.dir_type = directory[0]
self.Sdir_pos = './SubProcesses/' + directory
for card in list_card:
self.create_one_P_dir(card, remove_old=0)
pbar.update()
pbar.finish()
#update events
#progress bar
list_card = self.MWparam.actif_param
list_event = range(0, self.MWparam.nb_event)
pbar = progressbar.progbar('update MWdir',
(len(list_card)) * (len(list_event)) *
len(self.MWparam.P_listdir))
for dir in self.MWparam.MW_listdir:
self.dir_type = dir[0]
self.Sdir_pos = './SubProcesses/' + dir
#find an uptodate card
if cardref == -1:
ref_num = self.MWparam.nb_event - 1 #-1 due to the starting at zero
self.ref_card = -1
for card in self.MWparam.actif_param:
print self.find_exist_event(card)
if self.find_exist_event(card) >= ref_num:
self.ref_card = card
break
else:
self.card_ref = cardref
if self.ref_card == -1:
break
#then pass in update mode for all the card
for card in list_card:
for event in list_event:
self.create_one_M_dir(card, event, remove_old=0)
pbar.update()
pbar.finish()
def monte_carlo_V_function(self, policy, runs_per_start=10, show_progress=True):
'''
Calcuates the value-function under the given policy for the mountaincar environment
via Monte Carlo roll-outs.
This functions sets via a loop the start state runs several times sufficiently long
trajecotries with the policy.
The expected reward for each starting state is obtained via averaging the rewards of all roll-outs.
'''
logger.info("Creating MC value-function")
V_MC = np.empty((self.binsX, self.binsY))
iter = self.iterate_state_space()
if show_progress:
iter = progbar()(iter, max_value = V_MC.size)
for (i,j), o in iter:
reward_rollouts = np.empty((runs_per_start,))
for t in xrange(runs_per_start):
_,_,rewards,_,_,_ = self.roll_out(policy, o)
reward_rollouts[t] = EnvironmentWrapper.apply_discount(rewards, self.gamma)
#end for
V_MC[i,j] = reward_rollouts.mean()
#end for
return V_MC
def create_M_dir(self):
self.file_event=open(self.Sdir_pos+'/'+self.MWparam.name+'/verif.lhco')
self.line_event=self.file_event.readline()
dirname = self.Sdir_pos.split('/')[-1]
self.del_old_dir()
list_card=self.MWparam.actif_param
#progress bar
pbar = progressbar.progbar('create_dir',(len(list_card))*(self.MWparam.nb_event_MW[dirname]))
for card in list_card:
list_event=range(0, self.MWparam.nb_event_MW[dirname])
try:
os.mkdir(self.Sdir_pos+'/'+self.MWparam.name+'/card_'+str(card))
except:
pass
for event in list_event:
try:
self.create_one_M_dir(card,event)
except NoMoreEvent:
os.system('rm -rf %s' % self.Sdir_pos+'/'+self.MWparam.name+'/card_'+str(card))
self.MWparam.nb_event_MW[dirname] = event
break
pbar.update()
pbar.finish()
def collect_dir(self, path):
""" collect_one_directory """
# Pierre: change the tag MW -> P
print[name for name in path.split('/') if name.startswith('P')]
dir_name = [name for name in path.split('/')
if name.startswith('P')][-1]
pbar = progressbar.progbar(
'Collect %s' % dir_name,
len(self.MWparam.actif_param) *
(self.MWparam.nb_event_MW[dir_name] - self.MWparam.startevent))
for nb_card in self.MWparam.actif_param:
for nb_event in range(self.MWparam.startevent,
self.MWparam.nb_event_MW[dir_name]):
cur_path = path + '/card_' + str(nb_card) + '/event_' + str(
nb_event)
pbar.update()
if not os.path.exists(cur_path):
print 'WARNING: no directory: ', cur_path
print 'stop to collect for this SubProcesses'
pbar.finish()
return
self.collect_one(cur_path,
nb_card,
nb_event,
mode=self.MWparam['mw_perm']['combine_mode'])
pbar.finish()
def create_M_dir(self):
self.file_event = open(self.Sdir_pos + '/' + self.MWparam.name +
'/verif.lhco')
self.line_event = self.file_event.readline()
dirname = self.Sdir_pos.split('/')[-1]
self.del_old_dir()
list_card = self.MWparam.actif_param
#progress bar
pbar = progressbar.progbar('create_dir', (len(list_card)) *
(self.MWparam.nb_event_MW[dirname]))
for card in list_card:
list_event = range(0, self.MWparam.nb_event_MW[dirname])
try:
os.mkdir(self.Sdir_pos + '/' + self.MWparam.name + '/card_' +
str(card))
except:
pass
for event in list_event:
try:
self.create_one_M_dir(card, event)
except NoMoreEvent:
os.system('rm -rf %s' % self.Sdir_pos + '/' +
self.MWparam.name + '/card_' + str(card))
self.MWparam.nb_event_MW[dirname] = event
break
pbar.update()
pbar.finish()
def get_list_of_track_objects(self):
""" Ronseal. """
result = []
list_of_tracks = list(self.set_of_tracks)
print("Gathering data for all tracks by " + self.name + "...")
for track_title in progbar(list_of_tracks):
track = Track(self.name, track_title)
result.append(track)
return result
def update_card_status(self,cardref=-1):
"""creates the directory for standard run """
#check if all card have a directory in P_
list_card=[num-1 for num in self.MWparam.actif_param]
#progress bar
pbar = progressbar.progbar('update Pdir',len(list_card)*len(self.MWparam.P_listdir))
for directory in self.MWparam.P_listdir:
self.dir_type=directory[0]
self.Sdir_pos='./SubProcesses/'+directory
for card in list_card:
self.create_one_P_dir(card,remove_old=0)
pbar.update()
pbar.finish()
#update events
#progress bar
list_card=self.MWparam.actif_param
list_event=range(0,self.MWparam.nb_event)
pbar = progressbar.progbar('update MWdir',(len(list_card))*(len(list_event))*len(self.MWparam.P_listdir))
for dir in self.MWparam.MW_listdir:
self.dir_type=dir[0]
self.Sdir_pos='./SubProcesses/'+dir
#find an uptodate card
if cardref==-1:
ref_num=self.MWparam.nb_event-1 #-1 due to the starting at zero
self.ref_card=-1
for card in self.MWparam.actif_param:
print self.find_exist_event(card)
if self.find_exist_event(card)>=ref_num:
self.ref_card=card
break
else:
self.card_ref=cardref
if self.ref_card==-1:
break
#then pass in update mode for all the card
for card in list_card:
for event in list_event:
self.create_one_M_dir(card,event,remove_old=0)
pbar.update()
pbar.finish()
def train(params, non_linearities, data_wrapper, run, iterations,
plotting_rate, show_progress, **kwargs):
''' The main training loop, returns the lists with parameters and errors over time '''
t0 = time.time()
errors_all = []
params_all = []
it = xrange(iterations)
if show_progress:
it = progbar()(it)
for i in it:
params, error = descent_step(params, non_linearities, data_wrapper,
**kwargs)
errors_all.append(error)
params_all.append(params)
if plotting_rate > 0 and (i + 1) % plotting_rate == 0:
plot_error(run, errors_all)
plot_weight_distance(run, params_all)
plt.show()
#end if
# Relative change requires more than one error
if len(errors_all) < 2:
continue
relative_change = np.abs(errors_all[-1] - errors_all[-2]) / np.abs(
errors_all[-2])
threshold = 1e-25
if relative_change < threshold:
logger.info(
"The relative change of the error is smaller than {} after {} iterations, terminating."
.format(threshold, i))
break
#end
threshold = 1e100
if relative_change > threshold:
logger.info(
"The relative change of the error is larger than {} after {} iterations, terminating."
.format(threshold, i))
break
#end
#end for
t1 = time.time()
return params_all, errors_all, t1 - t0
def create_P_dir(self):
self.del_old_dir()
list_card = [num - 1 for num in self.MWparam.actif_param]
#progress bar
pbar = progressbar.progbar('create_dir', len(list_card))
for card in list_card:
self.create_one_P_dir(card)
pbar.update()
pbar.finish()
def create_P_dir(self):
self.del_old_dir()
list_card=[num for num in self.MWparam.actif_param]
#progress bar
pbar = progressbar.progbar('create_dir',len(list_card))
for card in list_card:
self.create_one_P_dir(card)
pbar.update()
pbar.finish()
def all(self):
""" control all the submition routines: Default launches MW and ME routines """
if self.mother.submission_file.nbfile:
self.pbar=progressbar.progbar('Submission',self.mother.submission_file.nbfile)
self.all_MadWeight()
if self.MWparam.norm_with_cross:
self.all_MadEvent()
if self.pbar:
self.pbar.finish()
def add_events(self):
self.file_event=open(self.Sdir_pos+'/'+self.MWparam.name+'/verif.lhco')
self.line_event=self.file_event.readline()
nb_exist_event=self.find_exist_event()
list_card=self.MWparam.actif_param
list_event=range(nb_exist_event,nb_exist_event+self.MWparam.nb_event)
pbar = progressbar.progbar('create_dir',(len(list_card))*(len(list_event)))
for card in list_card:
for event in list_event:
self.create_one_M_dir(card,event)
pbar.update()
pbar.finish()
def create_M_dir(self):
self.file_event=open(self.Sdir_pos+'/verif.lhco')
self.line_event=self.file_event.readline()
self.del_old_dir()
list_card=self.MWparam.actif_param
list_event=range(0,self.MWparam.nb_event)
#progress bar
pbar = progressbar.progbar('create_dir',(len(list_card))*(len(list_event)))
for card in list_card:
for event in list_event:
self.create_one_M_dir(card,event)
pbar.update()
pbar.finish()
def relaunch(self,failed={}):
""" relaunch all the failed job for the directory """
# use the submision file if they are at least one submision file
if self.mother.submission_file.nbfile:
self.pbar=progressbar.progbar('Submission',self.mother.submission_file.nbfile)
self.launch_submission_file()
self.pbar.finish()
return
#else use the self.one method
for directory in failed.keys():
dir_type=directory[0]
for nb1,nb2 in failed[directory]:
self.one(directory,nb1,nb2,dir_type)
def create_M_dir(self):
self.file_event = open(self.Sdir_pos + '/verif.lhco')
self.line_event = self.file_event.readline()
self.del_old_dir()
list_card = self.MWparam.actif_param
list_event = range(0, self.MWparam.nb_event)
#progress bar
pbar = progressbar.progbar('create_dir',
(len(list_card)) * (len(list_event)))
for card in list_card:
for event in list_event:
self.create_one_M_dir(card, event)
pbar.update()
pbar.finish()
def add_events(self):
self.file_event = open(self.Sdir_pos + '/verif.lhco')
self.line_event = self.file_event.readline()
nb_exist_event = self.find_exist_event()
list_card = self.MWparam.actif_param
print nb_exist_event, self.MWparam.nb_event
list_event = range(nb_exist_event + 1,
nb_exist_event + self.MWparam.nb_event + 1)
pbar = progressbar.progbar('create_dir',
(len(list_card)) * (len(list_event)))
for card in list_card:
for event in list_event:
self.create_one_M_dir(card, event)
pbar.update()
pbar.finish()
def collect_dir(self, path):
""" collect_one_directory """
#Pierre
dir_name = [name for name in path.split('/') if name.startswith('XXX')][-1]
pbar = progressbar.progbar('Collect %s' % dir_name,
len(self.MWparam.actif_param))
for nb_card in self.MWparam.actif_param:
cur_path = path + '/card_'+str(nb_card)
pbar.update()
if not os.path.exists(path):
print 'WARNING: no directory: ',path
print 'stop to collect for this SubProcesses'
pbar.finish()
break
self.collect_one(cur_path, nb_card)
pbar.finish()
def collect_dir(self, path):
""" collect_one_directory """
#Pierre
dir_name = [
name for name in path.split('/') if name.startswith('XXX')
][-1]
pbar = progressbar.progbar('Collect %s' % dir_name,
len(self.MWparam.actif_param))
for nb_card in self.MWparam.actif_param:
cur_path = path + '/card_' + str(nb_card)
pbar.update()
if not os.path.exists(path):
print 'WARNING: no directory: ', path
print 'stop to collect for this SubProcesses'
pbar.finish()
break
self.collect_one(cur_path, nb_card)
pbar.finish()
def collect_dir(self, path):
""" collect_one_directory """
# Pierre: change the tag MW -> P
print [name for name in path.split('/') if name.startswith('P')]
dir_name = [name for name in path.split('/') if name.startswith('P')][-1]
pbar = progressbar.progbar('Collect %s' % dir_name,
len(self.MWparam.actif_param) *
(self.MWparam.nb_event_MW[dir_name] - self.MWparam.startevent))
for nb_card in self.MWparam.actif_param:
for nb_event in range(self.MWparam.startevent, self.MWparam.nb_event_MW[dir_name]):
cur_path = path + '/card_'+str(nb_card)+'/event_'+str(nb_event)
pbar.update()
if not os.path.exists(cur_path):
print 'WARNING: no directory: ',cur_path
print 'stop to collect for this SubProcesses'
pbar.finish()
return
self.collect_one(cur_path, nb_card, nb_event, mode=self.MWparam['mw_perm']['combine_mode'])
pbar.finish()
def evaluate_distance_V_and_V_MC(V_MC,
params,
non_linearities,
data_wrapper,
steps=10,
show_progress=False):
'''
Calculates the distance from the ground truth for all parameters
Currently only every 10-th parameter vector is used.
'''
dist = []
it = params[::steps]
if show_progress:
it = progbar()(it)
for param in it:
V_eval = evaluate_V(param, non_linearities, data_wrapper)
d = np.linalg.norm(V_eval - V_MC)
dist.append(d)
return dist
def train(params, non_linearities, data_wrapper, mu, mu_a_given_s,
pi_a_given_s, run, iterations, plotting_rate, semi_online,
show_progress, **kwargs):
''' The main training loop for an agent, returns the lists with parameters and errors over time and the required time '''
t0 = time.time()
errors_all = []
params_all = []
# Before the semi online training starts the wrapper has to be in a fresh state.
if semi_online is True:
data_wrapper.reset()
it = xrange(iterations)
if show_progress:
it = progbar()(it)
for i in it:
# For semi online training after every step new data has to be collected, starting in the last state of the last round
if semi_online is True:
data_wrapper.create_data_set(policy=mu, reset=False)
# A single gradient descent step
params, error = descent_step(params, non_linearities, data_wrapper,
mu_a_given_s, pi_a_given_s, **kwargs)
errors_all.append(error)
params_all.append(params)
if plotting_rate > 0 and (i + 1) % plotting_rate == 0:
plot_error(run, errors_all)
#plot_weight_distance(run, params_all)
plt.show()
#end if
how_many = 10
err_mean = np.array(errors_all[-how_many:]).mean()
threshold = 1e-15
if err_mean < threshold:
logger.info(
"Mean error of the last {} steps smaller than {} after {} iterations, terminating."
.format(how_many, threshold, i))
break
#end
threshold = 1e150
if err_mean > threshold:
logger.info(
"Mean error of the last {} steps larger than {} after {} iterations, terminating."
.format(how_many, threshold, i))
break
#end
#end for
t1 = time.time()
return params_all, errors_all, t1 - t0
betas, axis=1).sum(1)
df_slice['z'] = np.exp(df_slice['betax'])
df_slice['gt1'] = np.exp(df_slice['t1'] * gamma)
df_slice['gt2'] = np.exp(df_slice['t2'] * gamma)
for chrn in range(1, 23):
# chrn = '22'
filename = 'calls/chr%02d' % chrn
print filename
(bim, fam, G) = read_plink(filename)
mask = [True if elem in df_slice.index else False for elem in fam['fid']]
xs = fam['fid'].values[mask]
dfxs = df_slice.loc[xs, ['event', 't1', 't2', 'betax', 'z', 'gt1', 'gt2']]
g = G.rechunk((1000, -1))[:, mask].astype(float)
pbar = progbar(widgets=[Bar(), Counter(), AdaptiveETA()])
mass = []
for block in pbar(list(g.blocks)):
mass.append(pd.concat(map(getbeta, block.compute()), axis=1).T)
result_table = pd.DataFrame(pd.concat(mass).values,
index=bim['snp'],
columns=['beta', 'sigma', 'Nd'])
outfile = join('data', prefix, basename(filename) + '.pkl')
result_table.to_pickle(outfile)
