def load_bag_file(bag_path,BagFolder_dic,bag_img_dic,skip_bag_dic,bag_names_dic,meta_dir,rgb_1to4_dir): bn = bag_path run_name = bn.split('/')[0] if bn in skip_bag_dic: print(d2n('\t',bn,' in skip_bag_dic')) return False bf = fname(bn) if bag_names_dic[bn] == False: #cprint(BagFolder_dic.keys(),'blue') if run_name not in BagFolder_dic: cprint('loading '+opj(run_name,'Bag_Folder.pkl'),'yellow','on_red') BagFolder_dic[run_name] = load_obj(opj(meta_dir,run_name,'Bag_Folder.pkl')) bag_img_dic[bn] = Bag_File.load_images(opj(rgb_1to4_dir,bn),color_mode="rgb8",include_flip=True) bag_names_dic[bn] == True good_bag_timestamps = list(set(BagFolder_dic[run_name]['data']['good_start_timestamps']) & set(bag_img_dic[bn]['left'].keys())) if len(good_bag_timestamps) < 100: if verbose: print(d2n('\t',bn,' len(good_bag_timestamps) < 100')) skip_bag_dic[bn] = True #skip_bag_dic[run_name] = True #del bag_img_dic[bn] #del bag_names_dic[bn] #del BagFolder_dic[run_name] return False binned_timestamps = [[],[]] binned_steers = [[],[]] for t in good_bag_timestamps: steer = BagFolder_dic[run_name]['left_image_bound_to_data'][t]['steer'] if steer < 43 or steer > 55: binned_timestamps[0].append(t) binned_steers[0].append(steer) else: binned_timestamps[1].append(t) binned_steers[1].append(steer) dic_keys = ['bag_file_image_data','good_bag_timestamps','binned_timestamps','binned_steers','bid_timestamps'] for dk in dic_keys: if dk not in BagFolder_dic[run_name]: BagFolder_dic[run_name][dk] = {} BagFolder_dic[run_name]['good_bag_timestamps'][bf] = good_bag_timestamps BagFolder_dic[run_name]['binned_timestamps'][bf] = binned_timestamps BagFolder_dic[run_name]['binned_steers'][bf] = binned_steers BagFolder_dic[run_name]['bid_timestamps'][bf] = sorted(bag_img_dic[bn]['left'].keys()) return True assert(False)
def load_bag_file(bag_names_dic, BagFolder_dic, bag_img_dic, skip_bag_dic, meta_dir, rgb_1to4_dir): timer = Timer(10) while True: bn = a_key(bag_names_dic) if bn in skip_bag_dic: continue bf = fname(bn) if bag_names_dic[bn] == False: run_name = bn.split('/')[0] if run_name not in BagFolder_dic: cprint('Loading ' + opj(run_name, 'Bag_Folder.pkl'), 'yellow', 'on_red') BagFolder_dic[run_name] = load_obj( opj(meta_dir, run_name, 'Bag_Folder.pkl')) bag_img_dic[bn] = Bag_File.load_images(opj(rgb_1to4_dir, bn), color_mode="rgb8", include_flip=True) bag_names_dic[bn] == True good_bag_timestamps = list( set(BagFolder_dic[run_name]['data']['good_start_timestamps']) & set(bag_img_dic[bn]['left'].keys())) if len(good_bag_timestamps) < 100: skip_bag_dic[bn] = True del bag_img_dic[bn] del bag_names_dic[bn] continue binned_timestamps = [[], []] binned_steers = [[], []] for t in good_bag_timestamps: steer = BagFolder_dic[run_name]['left_image_bound_to_data'][t][ 'steer'] if steer < 43 or steer > 55: binned_timestamps[0].append(t) binned_steers[0].append(steer) else: binned_timestamps[1].append(t) binned_steers[1].append(steer) dic_keys = [ 'bag_file_image_data', 'good_bag_timestamps', 'binned_timestamps', 'binned_steers', 'bid_timestamps' ] for dk in dic_keys: if dk not in BagFolder_dic[run_name]: BagFolder_dic[run_name][dk] = {} BagFolder_dic[run_name]['good_bag_timestamps'][ bf] = good_bag_timestamps BagFolder_dic[run_name]['binned_timestamps'][ bf] = binned_timestamps BagFolder_dic[run_name]['binned_steers'][bf] = binned_steers BagFolder_dic[run_name]['bid_timestamps'][bf] = sorted( bag_img_dic[bn]['left'].keys()) print bn return if timer: assert (False)
def bag_file_loader_thread(thread_id,command_dic,data_path,BagFolder_dic,BagFolders_weighted,delay_before_delete,loaded_bag_files_names,played_bagfile_dic): while True: state = command_dic[thread_id] command = command_dic[thread_id] if command == 'pause': if state == 'pause': pass else: state = 'pause' cprint(d2s('Pausing thread ',thread_id),'yellow','on_blue') time.sleep(1) continue if command == 'start' and state == 'pause': state = 'running' cprint(d2s('Unpausing thread ',thread_id),'yellow','on_blue') if command == 'stop': cprint(d2s('Stopping thread ',thread_id),'yellow','on_blue') return total_num_bag_files = 0 for bf in BagFolder_dic.keys(): total_num_bag_files += len(BagFolder_dic[bf]['bag_files']) #print((len(loaded_bag_files_names),total_num_bag_files)) if len(loaded_bag_files_names) >= 0.9*total_num_bag_files: cprint("90% of bag files loaded.") time.sleep(10) if len(loaded_bag_files_names) > 1000: cprint('\n\nTHREAD:: pause before deleting '+bf+'\n\n,,','blue','on_red') time.sleep(delay_before_delete) played_bagfile_dic_keys = [] played_bagfile_dic_values = [] for b in played_bagfile_dic.keys(): played_bagfile_dic_keys.append(b) played_bagfile_dic_values.append(played_bagfile_dic[b]) indicies = [i[0] for i in sorted(enumerate(played_bagfile_dic_values),key=lambda x:x[1])] indicies.reverse() ctr = 0 for i in indicies: if if ctr >= 25: #ctr >= 0.25*len(indicies): # if ctr >= 25: break bf = played_bagfile_dic_keys[i] if bf in loaded_bag_files_names: cprint('THREAD:: deleting '+bf,'blue','on_red') r = loaded_bag_files_names[bf] loaded_bag_files_names.pop(bf) BagFolder = BagFolder_dic[r] BagFolder['bag_file_image_data'].pop(bf) ctr += 1 if True: #try: run = random.choice(BagFolders_weighted) BagFolder = BagFolder_dic[run] if type(BagFolder) != dict: continue dic_keys = ['bag_file_image_data','good_bag_timestamps','binned_timestamps','binned_steers','bid_timestamps'] for dk in dic_keys: if dk not in BagFolder: BagFolder[dk] = {} if len(BagFolder['bag_file_num_dic']) > 0: try: bf = fname(random.choice(BagFolder['bag_file_num_dic'])) except: continue if bf in BagFolder['bag_file_image_data']: continue #cprint('bf = ' + bf,'red','on_white') bag_file_path = opj(data_path,'rgb_1to4',run,bf) #print bag_file_path BagFolder['bag_file_image_data'][bf] = Bag_File.load_images(bag_file_path,color_mode="rgb8",include_flip=True) loaded_bag_files_names[bf] = run bid = BagFolder['bag_file_image_data'][bf] bag_left_timestamps = sorted(bid['left'].keys()) good_bag_timestamps = list(set(BagFolder['data']['good_start_timestamps']) & set(bag_left_timestamps)) binned_timestamps = [[],[]] binned_steers = [[],[]] for t in good_bag_timestamps: steer = BagFolder['left_image_bound_to_data'][t]['steer'] if steer < 43 or steer > 55: binned_timestamps[0].append(t) binned_steers[0].append(steer) else: binned_timestamps[1].append(t) binned_steers[1].append(steer) BagFolder['good_bag_timestamps'][bf] = good_bag_timestamps BagFolder['binned_timestamps'][bf] = binned_timestamps BagFolder['binned_steers'][bf] = binned_steers BagFolder['bid_timestamps'][bf] = sorted(bid['left'].keys()) else: #except Exception as e: cprint("THREAD:: ********** Exception ***********************",'red') print(e.message, e.args)
if False: BagFolder_dic = {} bag_img_dic = {} bn = 'direct_local_21Nov16_17h39m00s_Mr_Yellow/bair_car_2016-11-21-18-03-39_30.bag.pkl' run_name = bn.split('/')[0] bf = fname(bn) cprint('loading ' + opj(run_name, 'Bag_Folder.pkl'), 'yellow', 'on_red') BagFolder_dic[run_name] = load_obj( opj(meta_dir, run_name, 'Bag_Folder.pkl')) bag_img_dic[bn] = Bag_File.load_images(opj(rgb_1to4_dir, bn), color_mode="rgb8", include_flip=True) bag_names_dic[bn] == True good_bag_timestamps = list( set(BagFolder_dic[run_name]['data']['good_start_timestamps']) & set(bag_img_dic[bn]['left'].keys())) if len(good_bag_timestamps) < 100: if verbose: print(d2n('\t', bn, ' len(good_bag_timestamps) < 100')) def load_bag_file(bag_path, BagFolder_dic, bag_img_dic, skip_bag_dic, bag_names_dic, meta_dir, rgb_1to4_dir): bn = bag_path
def bag_file_loader_thread(thread_id,command_dic,data_path,BagFolder_dic,BagFolders_weighted,delay_before_delete,loaded_bag_files_names,played_bagfile_dic): while True: state = command_dic[thread_id] command = command_dic[thread_id] if command == 'pause': if state == 'pause': pass else: state = 'pause' cprint(d2s('Pausing thread ',thread_id),'yellow','on_blue') time.sleep(1) continue if command == 'start' and state == 'pause': state = 'running' cprint(d2s('Unpausing thread ',thread_id),'yellow','on_blue') if command == 'stop': cprint(d2s('Stopping thread ',thread_id),'yellow','on_blue') return total_num_bag_files = 0 for bf in BagFolder_dic.keys(): total_num_bag_files += len(BagFolder_dic[bf]['bag_files']) #print((len(loaded_bag_files_names),total_num_bag_files)) if len(loaded_bag_files_names) >= 0.9*total_num_bag_files: cprint("90% of bag files loaded.") time.sleep(10) if len(loaded_bag_files_names) > 1000: cprint('\n\nTHREAD:: pause before deleting '+bf+'\n\n,,','blue','on_red') time.sleep(delay_before_delete) played_bagfile_dic_keys = [] played_bagfile_dic_values = [] for b in played_bagfile_dic.keys(): played_bagfile_dic_keys.append(b) played_bagfile_dic_values.append(played_bagfile_dic[b]) indicies = [i[0] for i in sorted(enumerate(played_bagfile_dic_values),key=lambda x:x[1])] indicies.reverse() ctr = 0 for i in indicies: if ctr >= 25: #ctr >= 0.25*len(indicies): # if ctr >= 25: break bf = played_bagfile_dic_keys[i] if bf in loaded_bag_files_names: cprint('THREAD:: deleting '+bf,'blue','on_red') r = loaded_bag_files_names[bf] loaded_bag_files_names.pop(bf) BagFolder = BagFolder_dic[r] BagFolder['bag_file_image_data'].pop(bf) ctr += 1 if True: #try: run = random.choice(BagFolders_weighted) BagFolder = BagFolder_dic[run] if type(BagFolder) != dict: continue dic_keys = ['bag_file_image_data','good_bag_timestamps','binned_timestamps','binned_steers','bid_timestamps'] for dk in dic_keys: if dk not in BagFolder: BagFolder[dk] = {} if len(BagFolder['bag_file_num_dic']) > 0: try: bf = fname(random.choice(BagFolder['bag_file_num_dic'])) except: continue if bf in BagFolder['bag_file_image_data']: continue #cprint('bf = ' + bf,'red','on_white') bag_file_path = opj(data_path,'rgb_1to4',run,bf) #print bag_file_path BagFolder['bag_file_image_data'][bf] = Bag_File.load_images(bag_file_path,color_mode="rgb8",include_flip=True) loaded_bag_files_names[bf] = run bid = BagFolder['bag_file_image_data'][bf] bag_left_timestamps = sorted(bid['left'].keys()) good_bag_timestamps = list(set(BagFolder['data']['good_start_timestamps']) & set(bag_left_timestamps)) binned_timestamps = [[],[]] binned_steers = [[],[]] for t in good_bag_timestamps: steer = BagFolder['left_image_bound_to_data'][t]['steer'] if steer < 43 or steer > 55: binned_timestamps[0].append(t) binned_steers[0].append(steer) else: binned_timestamps[1].append(t) binned_steers[1].append(steer) BagFolder['good_bag_timestamps'][bf] = good_bag_timestamps BagFolder['binned_timestamps'][bf] = binned_timestamps BagFolder['binned_steers'][bf] = binned_steers BagFolder['bid_timestamps'][bf] = sorted(bid['left'].keys()) else: #except Exception as e: cprint("THREAD:: ********** Exception ***********************",'red') print(e.message, e.args)
def bag_file_loader_thread(BF_dic, BF_dic_keys_weights, delay_before_delete, loaded_bag_files_names, played_bagfile_dic): while True: if bag_file_loader_thread_please_exit: cprint('THREAD:: exiting bag_file_loader_thread()') return elif not thread_please_load_data: time.sleep(1) else: if len(loaded_bag_files_names) > 1000: cprint('\n\nTHREAD:: pause before deleting ' + bf + '\n\n,,', 'blue', 'on_red') time.sleep(delay_before_delete) played_bagfile_dic_keys = [] played_bagfile_dic_values = [] for b in played_bagfile_dic.keys(): played_bagfile_dic_keys.append(b) played_bagfile_dic_values.append(played_bagfile_dic[b]) indicies = [ i[0] for i in sorted(enumerate(played_bagfile_dic_values), key=lambda x: x[1]) ] indicies.reverse() ctr = 0 for i in indicies: if ctr >= 25: break bf = played_bagfile_dic_keys[i] if bf in loaded_bag_files_names: #bf = a_key(loaded_bag_files_names) cprint('THREAD:: deleting ' + bf, 'blue', 'on_red') r = loaded_bag_files_names[bf] loaded_bag_files_names.pop(bf) BF = BF_dic[r] BF['bag_file_image_data'].pop(bf) ctr += 1 if True: #try: r = random.choice(BF_dic.keys()) BF = BF_dic[r] if type(BF) != dict: continue dic_keys = [ 'bag_file_image_data', 'good_bag_timestamps', 'binned_timestamps', 'binned_steers', 'bid_timestamps' ] for dk in dic_keys: if dk not in BF: BF[dk] = {} #print BF['bag_file_num_dic'] if len(BF['bag_file_num_dic']) > 0: try: bf = random.choice(BF['bag_file_num_dic']) except: continue if bf in BF['bag_file_image_data']: continue BF['bag_file_image_data'][bf] = Bag_File.load_images( bf, color_mode="rgb8", include_flip=True) loaded_bag_files_names[bf] = r bid = BF['bag_file_image_data'][bf] bag_left_timestamps = sorted(bid['left'].keys()) good_bag_timestamps = list( set(BF['data']['good_start_timestamps']) & set(bag_left_timestamps)) #cprint(d2s('THREAD:: ',bf.split('/')[-1],'len(good_bag_timestamps) =',len(good_bag_timestamps)),'blue') binned_timestamps = [[], []] binned_steers = [[], []] for t in good_bag_timestamps: steer = BF['left_image_bound_to_data'][t]['steer'] if steer < 43 or steer > 55: binned_timestamps[0].append(t) binned_steers[0].append(steer) else: binned_timestamps[1].append(t) binned_steers[1].append(steer) BF['good_bag_timestamps'][bf] = good_bag_timestamps BF['binned_timestamps'][bf] = binned_timestamps BF['binned_steers'][bf] = binned_steers BF['bid_timestamps'][bf] = sorted(bid['left'].keys()) else: #except Exception as e: cprint("THREAD:: ********** Exception ***********************", 'red') print(e.message, e.args)
def bag_file_loader_thread( BF_dic, BF_dic_keys_weights, delay_before_delete, loaded_bag_files_names, played_bagfile_dic ): while True: if bag_file_loader_thread_please_exit: cprint("THREAD:: exiting bag_file_loader_thread()") return elif not thread_please_load_data: time.sleep(1) else: if len(loaded_bag_files_names) > 1000: cprint("\n\nTHREAD:: pause before deleting " + bf + "\n\n,,", "blue", "on_red") time.sleep(delay_before_delete) played_bagfile_dic_keys = [] played_bagfile_dic_values = [] for b in played_bagfile_dic.keys(): played_bagfile_dic_keys.append(b) played_bagfile_dic_values.append(played_bagfile_dic[b]) indicies = [i[0] for i in sorted(enumerate(played_bagfile_dic_values), key=lambda x: x[1])] indicies.reverse() ctr = 0 for i in indicies: if ctr >= 25: break bf = played_bagfile_dic_keys[i] if bf in loaded_bag_files_names: # bf = a_key(loaded_bag_files_names) cprint("THREAD:: deleting " + bf, "blue", "on_red") r = loaded_bag_files_names[bf] loaded_bag_files_names.pop(bf) BF = BF_dic[r] BF["bag_file_image_data"].pop(bf) ctr += 1 if True: # try: r = random.choice(BF_dic.keys()) BF = BF_dic[r] if type(BF) != dict: continue dic_keys = [ "bag_file_image_data", "good_bag_timestamps", "binned_timestamps", "binned_steers", "bid_timestamps", ] for dk in dic_keys: if dk not in BF: BF[dk] = {} # print BF['bag_file_num_dic'] if len(BF["bag_file_num_dic"]) > 0: try: bf = random.choice(BF["bag_file_num_dic"]) except: continue if bf in BF["bag_file_image_data"]: continue BF["bag_file_image_data"][bf] = Bag_File.load_images(bf, color_mode="rgb8", include_flip=True) loaded_bag_files_names[bf] = r bid = BF["bag_file_image_data"][bf] bag_left_timestamps = sorted(bid["left"].keys()) good_bag_timestamps = list(set(BF["data"]["good_start_timestamps"]) & set(bag_left_timestamps)) # cprint(d2s('THREAD:: ',bf.split('/')[-1],'len(good_bag_timestamps) =',len(good_bag_timestamps)),'blue') binned_timestamps = [[], []] binned_steers = [[], []] for t in good_bag_timestamps: steer = BF["left_image_bound_to_data"][t]["steer"] if steer < 43 or steer > 55: binned_timestamps[0].append(t) binned_steers[0].append(steer) else: binned_timestamps[1].append(t) binned_steers[1].append(steer) BF["good_bag_timestamps"][bf] = good_bag_timestamps BF["binned_timestamps"][bf] = binned_timestamps BF["binned_steers"][bf] = binned_steers BF["bid_timestamps"][bf] = sorted(bid["left"].keys()) else: # except Exception as e: cprint("THREAD:: ********** Exception ***********************", "red") print (e.message, e.args)