def main(): print('test') bar = progressbar.progbar(10,clear_display=True) for i in range(10): time.sleep(0.1) bar.update(i) del bar print('test2') bar = progressbar.progbar(10,clear_display=False) for i in range(10): time.sleep(0.1) bar.update(i) del bar print('test3') bar = progressbar.progbar(10,clear_display=True) for j in range(10): prog = progressbar.progbar(10,clear_display=True) for i in range(10): time.sleep(0.1) prog.update(i) del prog bar.update(j) del bar
def update_card_status(self, cardref=-1, number_of_event=-1): """creates the directory for standard run number_of_event[-1]:-1:automatic (number of event of cardref if !=-1, highest existing event in other case """ #check if all card have a directory in P_ list_card = [num for num in self.MWparam.actif_param] #progress bar pbar = progressbar.progbar( 'update Pdir', len(list_card) * len(self.MWparam.P_listdir)) for directory in self.MWparam.P_listdir: self.dir_type = directory[0] self.Sdir_pos = './SubProcesses/' + directory for card in list_card: self.create_one_P_dir(card, remove_old=0) pbar.update() pbar.finish() #find cardref and number_of_event if not defined if number_of_event == -1 and cardref == -1: #search the card with the maximum of event defined self.dir_type = 'M' self.Sdir_pos = './SubProcesses/' + self.MWparam.MW_listdir[0] for card in self.MWparam.actif_param: event = self.find_exist_event(card) if event >= number_of_event: cardref = card number_of_event = event elif cardref == -1: ref_num = number_of_event - 1 #-1 due to the starting at zero for card in self.MWparam.actif_param: if self.find_exist_event(card) >= ref_num: cardref = card if cardref == -1 or number_of_event == -1: sys.exit( 'impossible to update card and/or number of event not defined') else: self.ref_card = cardref #update events #progress bar list_card = self.MWparam.actif_param list_event = range(0, number_of_event) if (len(list_card)) * (len(list_event)) * len( self.MWparam.MW_listdir) > 0: pbar = progressbar.progbar('update MWdir', (len(list_card)) * (len(list_event)) * len(self.MWparam.P_listdir)) for dir in self.MWparam.MW_listdir: self.dir_type = dir[0] self.Sdir_pos = './SubProcesses/' + dir #then pass in update mode for all the card for card in list_card: for event in list_event: self.create_one_M_dir(card, event, remove_old=0) pbar.update() pbar.finish()
def update_card_status(self,cardref=-1,number_of_event=-1): """creates the directory for standard run number_of_event[-1]:-1:automatic (number of event of cardref if !=-1, highest existing event in other case """ #check if all card have a directory in P_ list_card=[num for num in self.MWparam.actif_param] #progress bar pbar = progressbar.progbar('update Pdir',len(list_card)*len(self.MWparam.P_listdir)) for directory in self.MWparam.P_listdir: self.dir_type=directory[0] self.Sdir_pos='./SubProcesses/'+directory for card in list_card: self.create_one_P_dir(card,remove_old=0) pbar.update() pbar.finish() #find cardref and number_of_event if not defined if number_of_event==-1 and cardref==-1: #search the card with the maximum of event defined self.dir_type='M' self.Sdir_pos='./SubProcesses/'+self.MWparam.MW_listdir[0] for card in self.MWparam.actif_param: event=self.find_exist_event(card) if event>=number_of_event: cardref=card number_of_event=event elif cardref==-1: ref_num=number_of_event-1 #-1 due to the starting at zero for card in self.MWparam.actif_param: if self.find_exist_event(card)>=ref_num: cardref=card if cardref==-1 or number_of_event==-1: sys.exit('impossible to update card and/or number of event not defined') else: self.ref_card=cardref #update events #progress bar list_card=self.MWparam.actif_param list_event=range(0,number_of_event) if (len(list_card))*(len(list_event))*len(self.MWparam.MW_listdir)>0: pbar = progressbar.progbar('update MWdir',(len(list_card))*(len(list_event))*len(self.MWparam.P_listdir)) for dir in self.MWparam.MW_listdir: self.dir_type=dir[0] self.Sdir_pos='./SubProcesses/'+dir #then pass in update mode for all the card for card in list_card: for event in list_event: self.create_one_M_dir(card,event,remove_old=0) pbar.update() pbar.finish()
def update_card_status(self, cardref=-1): """creates the directory for standard run """ #check if all card have a directory in P_ list_card = [num - 1 for num in self.MWparam.actif_param] #progress bar pbar = progressbar.progbar( 'update Pdir', len(list_card) * len(self.MWparam.P_listdir)) for directory in self.MWparam.P_listdir: self.dir_type = directory[0] self.Sdir_pos = './SubProcesses/' + directory for card in list_card: self.create_one_P_dir(card, remove_old=0) pbar.update() pbar.finish() #update events #progress bar list_card = self.MWparam.actif_param list_event = range(0, self.MWparam.nb_event) pbar = progressbar.progbar('update MWdir', (len(list_card)) * (len(list_event)) * len(self.MWparam.P_listdir)) for dir in self.MWparam.MW_listdir: self.dir_type = dir[0] self.Sdir_pos = './SubProcesses/' + dir #find an uptodate card if cardref == -1: ref_num = self.MWparam.nb_event - 1 #-1 due to the starting at zero self.ref_card = -1 for card in self.MWparam.actif_param: print self.find_exist_event(card) if self.find_exist_event(card) >= ref_num: self.ref_card = card break else: self.card_ref = cardref if self.ref_card == -1: break #then pass in update mode for all the card for card in list_card: for event in list_event: self.create_one_M_dir(card, event, remove_old=0) pbar.update() pbar.finish()
def monte_carlo_V_function(self, policy, runs_per_start=10, show_progress=True): ''' Calcuates the value-function under the given policy for the mountaincar environment via Monte Carlo roll-outs. This functions sets via a loop the start state runs several times sufficiently long trajecotries with the policy. The expected reward for each starting state is obtained via averaging the rewards of all roll-outs. ''' logger.info("Creating MC value-function") V_MC = np.empty((self.binsX, self.binsY)) iter = self.iterate_state_space() if show_progress: iter = progbar()(iter, max_value = V_MC.size) for (i,j), o in iter: reward_rollouts = np.empty((runs_per_start,)) for t in xrange(runs_per_start): _,_,rewards,_,_,_ = self.roll_out(policy, o) reward_rollouts[t] = EnvironmentWrapper.apply_discount(rewards, self.gamma) #end for V_MC[i,j] = reward_rollouts.mean() #end for return V_MC
def create_M_dir(self): self.file_event=open(self.Sdir_pos+'/'+self.MWparam.name+'/verif.lhco') self.line_event=self.file_event.readline() dirname = self.Sdir_pos.split('/')[-1] self.del_old_dir() list_card=self.MWparam.actif_param #progress bar pbar = progressbar.progbar('create_dir',(len(list_card))*(self.MWparam.nb_event_MW[dirname])) for card in list_card: list_event=range(0, self.MWparam.nb_event_MW[dirname]) try: os.mkdir(self.Sdir_pos+'/'+self.MWparam.name+'/card_'+str(card)) except: pass for event in list_event: try: self.create_one_M_dir(card,event) except NoMoreEvent: os.system('rm -rf %s' % self.Sdir_pos+'/'+self.MWparam.name+'/card_'+str(card)) self.MWparam.nb_event_MW[dirname] = event break pbar.update() pbar.finish()
def collect_dir(self, path): """ collect_one_directory """ # Pierre: change the tag MW -> P print[name for name in path.split('/') if name.startswith('P')] dir_name = [name for name in path.split('/') if name.startswith('P')][-1] pbar = progressbar.progbar( 'Collect %s' % dir_name, len(self.MWparam.actif_param) * (self.MWparam.nb_event_MW[dir_name] - self.MWparam.startevent)) for nb_card in self.MWparam.actif_param: for nb_event in range(self.MWparam.startevent, self.MWparam.nb_event_MW[dir_name]): cur_path = path + '/card_' + str(nb_card) + '/event_' + str( nb_event) pbar.update() if not os.path.exists(cur_path): print 'WARNING: no directory: ', cur_path print 'stop to collect for this SubProcesses' pbar.finish() return self.collect_one(cur_path, nb_card, nb_event, mode=self.MWparam['mw_perm']['combine_mode']) pbar.finish()
def create_M_dir(self): self.file_event = open(self.Sdir_pos + '/' + self.MWparam.name + '/verif.lhco') self.line_event = self.file_event.readline() dirname = self.Sdir_pos.split('/')[-1] self.del_old_dir() list_card = self.MWparam.actif_param #progress bar pbar = progressbar.progbar('create_dir', (len(list_card)) * (self.MWparam.nb_event_MW[dirname])) for card in list_card: list_event = range(0, self.MWparam.nb_event_MW[dirname]) try: os.mkdir(self.Sdir_pos + '/' + self.MWparam.name + '/card_' + str(card)) except: pass for event in list_event: try: self.create_one_M_dir(card, event) except NoMoreEvent: os.system('rm -rf %s' % self.Sdir_pos + '/' + self.MWparam.name + '/card_' + str(card)) self.MWparam.nb_event_MW[dirname] = event break pbar.update() pbar.finish()
def get_list_of_track_objects(self): """ Ronseal. """ result = [] list_of_tracks = list(self.set_of_tracks) print("Gathering data for all tracks by " + self.name + "...") for track_title in progbar(list_of_tracks): track = Track(self.name, track_title) result.append(track) return result
def update_card_status(self,cardref=-1): """creates the directory for standard run """ #check if all card have a directory in P_ list_card=[num-1 for num in self.MWparam.actif_param] #progress bar pbar = progressbar.progbar('update Pdir',len(list_card)*len(self.MWparam.P_listdir)) for directory in self.MWparam.P_listdir: self.dir_type=directory[0] self.Sdir_pos='./SubProcesses/'+directory for card in list_card: self.create_one_P_dir(card,remove_old=0) pbar.update() pbar.finish() #update events #progress bar list_card=self.MWparam.actif_param list_event=range(0,self.MWparam.nb_event) pbar = progressbar.progbar('update MWdir',(len(list_card))*(len(list_event))*len(self.MWparam.P_listdir)) for dir in self.MWparam.MW_listdir: self.dir_type=dir[0] self.Sdir_pos='./SubProcesses/'+dir #find an uptodate card if cardref==-1: ref_num=self.MWparam.nb_event-1 #-1 due to the starting at zero self.ref_card=-1 for card in self.MWparam.actif_param: print self.find_exist_event(card) if self.find_exist_event(card)>=ref_num: self.ref_card=card break else: self.card_ref=cardref if self.ref_card==-1: break #then pass in update mode for all the card for card in list_card: for event in list_event: self.create_one_M_dir(card,event,remove_old=0) pbar.update() pbar.finish()
def train(params, non_linearities, data_wrapper, run, iterations, plotting_rate, show_progress, **kwargs): ''' The main training loop, returns the lists with parameters and errors over time ''' t0 = time.time() errors_all = [] params_all = [] it = xrange(iterations) if show_progress: it = progbar()(it) for i in it: params, error = descent_step(params, non_linearities, data_wrapper, **kwargs) errors_all.append(error) params_all.append(params) if plotting_rate > 0 and (i + 1) % plotting_rate == 0: plot_error(run, errors_all) plot_weight_distance(run, params_all) plt.show() #end if # Relative change requires more than one error if len(errors_all) < 2: continue relative_change = np.abs(errors_all[-1] - errors_all[-2]) / np.abs( errors_all[-2]) threshold = 1e-25 if relative_change < threshold: logger.info( "The relative change of the error is smaller than {} after {} iterations, terminating." .format(threshold, i)) break #end threshold = 1e100 if relative_change > threshold: logger.info( "The relative change of the error is larger than {} after {} iterations, terminating." .format(threshold, i)) break #end #end for t1 = time.time() return params_all, errors_all, t1 - t0
def create_P_dir(self): self.del_old_dir() list_card = [num - 1 for num in self.MWparam.actif_param] #progress bar pbar = progressbar.progbar('create_dir', len(list_card)) for card in list_card: self.create_one_P_dir(card) pbar.update() pbar.finish()
def create_P_dir(self): self.del_old_dir() list_card=[num for num in self.MWparam.actif_param] #progress bar pbar = progressbar.progbar('create_dir',len(list_card)) for card in list_card: self.create_one_P_dir(card) pbar.update() pbar.finish()
def all(self): """ control all the submition routines: Default launches MW and ME routines """ if self.mother.submission_file.nbfile: self.pbar=progressbar.progbar('Submission',self.mother.submission_file.nbfile) self.all_MadWeight() if self.MWparam.norm_with_cross: self.all_MadEvent() if self.pbar: self.pbar.finish()
def add_events(self): self.file_event=open(self.Sdir_pos+'/'+self.MWparam.name+'/verif.lhco') self.line_event=self.file_event.readline() nb_exist_event=self.find_exist_event() list_card=self.MWparam.actif_param list_event=range(nb_exist_event,nb_exist_event+self.MWparam.nb_event) pbar = progressbar.progbar('create_dir',(len(list_card))*(len(list_event))) for card in list_card: for event in list_event: self.create_one_M_dir(card,event) pbar.update() pbar.finish()
def create_M_dir(self): self.file_event=open(self.Sdir_pos+'/verif.lhco') self.line_event=self.file_event.readline() self.del_old_dir() list_card=self.MWparam.actif_param list_event=range(0,self.MWparam.nb_event) #progress bar pbar = progressbar.progbar('create_dir',(len(list_card))*(len(list_event))) for card in list_card: for event in list_event: self.create_one_M_dir(card,event) pbar.update() pbar.finish()
def relaunch(self,failed={}): """ relaunch all the failed job for the directory """ # use the submision file if they are at least one submision file if self.mother.submission_file.nbfile: self.pbar=progressbar.progbar('Submission',self.mother.submission_file.nbfile) self.launch_submission_file() self.pbar.finish() return #else use the self.one method for directory in failed.keys(): dir_type=directory[0] for nb1,nb2 in failed[directory]: self.one(directory,nb1,nb2,dir_type)
def create_M_dir(self): self.file_event = open(self.Sdir_pos + '/verif.lhco') self.line_event = self.file_event.readline() self.del_old_dir() list_card = self.MWparam.actif_param list_event = range(0, self.MWparam.nb_event) #progress bar pbar = progressbar.progbar('create_dir', (len(list_card)) * (len(list_event))) for card in list_card: for event in list_event: self.create_one_M_dir(card, event) pbar.update() pbar.finish()
def add_events(self): self.file_event = open(self.Sdir_pos + '/verif.lhco') self.line_event = self.file_event.readline() nb_exist_event = self.find_exist_event() list_card = self.MWparam.actif_param print nb_exist_event, self.MWparam.nb_event list_event = range(nb_exist_event + 1, nb_exist_event + self.MWparam.nb_event + 1) pbar = progressbar.progbar('create_dir', (len(list_card)) * (len(list_event))) for card in list_card: for event in list_event: self.create_one_M_dir(card, event) pbar.update() pbar.finish()
def collect_dir(self, path): """ collect_one_directory """ #Pierre dir_name = [name for name in path.split('/') if name.startswith('XXX')][-1] pbar = progressbar.progbar('Collect %s' % dir_name, len(self.MWparam.actif_param)) for nb_card in self.MWparam.actif_param: cur_path = path + '/card_'+str(nb_card) pbar.update() if not os.path.exists(path): print 'WARNING: no directory: ',path print 'stop to collect for this SubProcesses' pbar.finish() break self.collect_one(cur_path, nb_card) pbar.finish()
def collect_dir(self, path): """ collect_one_directory """ #Pierre dir_name = [ name for name in path.split('/') if name.startswith('XXX') ][-1] pbar = progressbar.progbar('Collect %s' % dir_name, len(self.MWparam.actif_param)) for nb_card in self.MWparam.actif_param: cur_path = path + '/card_' + str(nb_card) pbar.update() if not os.path.exists(path): print 'WARNING: no directory: ', path print 'stop to collect for this SubProcesses' pbar.finish() break self.collect_one(cur_path, nb_card) pbar.finish()
def collect_dir(self, path): """ collect_one_directory """ # Pierre: change the tag MW -> P print [name for name in path.split('/') if name.startswith('P')] dir_name = [name for name in path.split('/') if name.startswith('P')][-1] pbar = progressbar.progbar('Collect %s' % dir_name, len(self.MWparam.actif_param) * (self.MWparam.nb_event_MW[dir_name] - self.MWparam.startevent)) for nb_card in self.MWparam.actif_param: for nb_event in range(self.MWparam.startevent, self.MWparam.nb_event_MW[dir_name]): cur_path = path + '/card_'+str(nb_card)+'/event_'+str(nb_event) pbar.update() if not os.path.exists(cur_path): print 'WARNING: no directory: ',cur_path print 'stop to collect for this SubProcesses' pbar.finish() return self.collect_one(cur_path, nb_card, nb_event, mode=self.MWparam['mw_perm']['combine_mode']) pbar.finish()
def evaluate_distance_V_and_V_MC(V_MC, params, non_linearities, data_wrapper, steps=10, show_progress=False): ''' Calculates the distance from the ground truth for all parameters Currently only every 10-th parameter vector is used. ''' dist = [] it = params[::steps] if show_progress: it = progbar()(it) for param in it: V_eval = evaluate_V(param, non_linearities, data_wrapper) d = np.linalg.norm(V_eval - V_MC) dist.append(d) return dist
def train(params, non_linearities, data_wrapper, mu, mu_a_given_s, pi_a_given_s, run, iterations, plotting_rate, semi_online, show_progress, **kwargs): ''' The main training loop for an agent, returns the lists with parameters and errors over time and the required time ''' t0 = time.time() errors_all = [] params_all = [] # Before the semi online training starts the wrapper has to be in a fresh state. if semi_online is True: data_wrapper.reset() it = xrange(iterations) if show_progress: it = progbar()(it) for i in it: # For semi online training after every step new data has to be collected, starting in the last state of the last round if semi_online is True: data_wrapper.create_data_set(policy=mu, reset=False) # A single gradient descent step params, error = descent_step(params, non_linearities, data_wrapper, mu_a_given_s, pi_a_given_s, **kwargs) errors_all.append(error) params_all.append(params) if plotting_rate > 0 and (i + 1) % plotting_rate == 0: plot_error(run, errors_all) #plot_weight_distance(run, params_all) plt.show() #end if how_many = 10 err_mean = np.array(errors_all[-how_many:]).mean() threshold = 1e-15 if err_mean < threshold: logger.info( "Mean error of the last {} steps smaller than {} after {} iterations, terminating." .format(how_many, threshold, i)) break #end threshold = 1e150 if err_mean > threshold: logger.info( "Mean error of the last {} steps larger than {} after {} iterations, terminating." .format(how_many, threshold, i)) break #end #end for t1 = time.time() return params_all, errors_all, t1 - t0
betas, axis=1).sum(1) df_slice['z'] = np.exp(df_slice['betax']) df_slice['gt1'] = np.exp(df_slice['t1'] * gamma) df_slice['gt2'] = np.exp(df_slice['t2'] * gamma) for chrn in range(1, 23): # chrn = '22' filename = 'calls/chr%02d' % chrn print filename (bim, fam, G) = read_plink(filename) mask = [True if elem in df_slice.index else False for elem in fam['fid']] xs = fam['fid'].values[mask] dfxs = df_slice.loc[xs, ['event', 't1', 't2', 'betax', 'z', 'gt1', 'gt2']] g = G.rechunk((1000, -1))[:, mask].astype(float) pbar = progbar(widgets=[Bar(), Counter(), AdaptiveETA()]) mass = [] for block in pbar(list(g.blocks)): mass.append(pd.concat(map(getbeta, block.compute()), axis=1).T) result_table = pd.DataFrame(pd.concat(mass).values, index=bim['snp'], columns=['beta', 'sigma', 'Nd']) outfile = join('data', prefix, basename(filename) + '.pkl') result_table.to_pickle(outfile)