def load_bag_file(bag_path,BagFolder_dic,bag_img_dic,skip_bag_dic,bag_names_dic,meta_dir,rgb_1to4_dir):
bn = bag_path
run_name = bn.split('/')[0]
if bn in skip_bag_dic:
print(d2n('\t',bn,' in skip_bag_dic'))
return False
bf = fname(bn)
if bag_names_dic[bn] == False:
#cprint(BagFolder_dic.keys(),'blue')
if run_name not in BagFolder_dic:
cprint('loading '+opj(run_name,'Bag_Folder.pkl'),'yellow','on_red')
BagFolder_dic[run_name] = load_obj(opj(meta_dir,run_name,'Bag_Folder.pkl'))
bag_img_dic[bn] = Bag_File.load_images(opj(rgb_1to4_dir,bn),color_mode="rgb8",include_flip=True)
bag_names_dic[bn] == True
good_bag_timestamps = list(set(BagFolder_dic[run_name]['data']['good_start_timestamps']) & set(bag_img_dic[bn]['left'].keys()))
if len(good_bag_timestamps) < 100:
if verbose:
print(d2n('\t',bn,' len(good_bag_timestamps) < 100'))
skip_bag_dic[bn] = True
#skip_bag_dic[run_name] = True
#del bag_img_dic[bn]
#del bag_names_dic[bn]
#del BagFolder_dic[run_name]
return False
binned_timestamps = [[],[]]
binned_steers = [[],[]]
for t in good_bag_timestamps:
steer = BagFolder_dic[run_name]['left_image_bound_to_data'][t]['steer']
if steer < 43 or steer > 55:
binned_timestamps[0].append(t)
binned_steers[0].append(steer)
else:
binned_timestamps[1].append(t)
binned_steers[1].append(steer)
dic_keys = ['bag_file_image_data','good_bag_timestamps','binned_timestamps','binned_steers','bid_timestamps']
for dk in dic_keys:
if dk not in BagFolder_dic[run_name]:
BagFolder_dic[run_name][dk] = {}
BagFolder_dic[run_name]['good_bag_timestamps'][bf] = good_bag_timestamps
BagFolder_dic[run_name]['binned_timestamps'][bf] = binned_timestamps
BagFolder_dic[run_name]['binned_steers'][bf] = binned_steers
BagFolder_dic[run_name]['bid_timestamps'][bf] = sorted(bag_img_dic[bn]['left'].keys())
return True
assert(False)
def load_bag_file(bag_names_dic, BagFolder_dic, bag_img_dic, skip_bag_dic,
meta_dir, rgb_1to4_dir):
timer = Timer(10)
while True:
bn = a_key(bag_names_dic)
if bn in skip_bag_dic:
continue
bf = fname(bn)
if bag_names_dic[bn] == False:
run_name = bn.split('/')[0]
if run_name not in BagFolder_dic:
cprint('Loading ' + opj(run_name, 'Bag_Folder.pkl'), 'yellow',
'on_red')
BagFolder_dic[run_name] = load_obj(
opj(meta_dir, run_name, 'Bag_Folder.pkl'))
bag_img_dic[bn] = Bag_File.load_images(opj(rgb_1to4_dir, bn),
color_mode="rgb8",
include_flip=True)
bag_names_dic[bn] == True
good_bag_timestamps = list(
set(BagFolder_dic[run_name]['data']['good_start_timestamps'])
& set(bag_img_dic[bn]['left'].keys()))
if len(good_bag_timestamps) < 100:
skip_bag_dic[bn] = True
del bag_img_dic[bn]
del bag_names_dic[bn]
continue
binned_timestamps = [[], []]
binned_steers = [[], []]
for t in good_bag_timestamps:
steer = BagFolder_dic[run_name]['left_image_bound_to_data'][t][
'steer']
if steer < 43 or steer > 55:
binned_timestamps[0].append(t)
binned_steers[0].append(steer)
else:
binned_timestamps[1].append(t)
binned_steers[1].append(steer)
dic_keys = [
'bag_file_image_data', 'good_bag_timestamps',
'binned_timestamps', 'binned_steers', 'bid_timestamps'
]
for dk in dic_keys:
if dk not in BagFolder_dic[run_name]:
BagFolder_dic[run_name][dk] = {}
BagFolder_dic[run_name]['good_bag_timestamps'][
bf] = good_bag_timestamps
BagFolder_dic[run_name]['binned_timestamps'][
bf] = binned_timestamps
BagFolder_dic[run_name]['binned_steers'][bf] = binned_steers
BagFolder_dic[run_name]['bid_timestamps'][bf] = sorted(
bag_img_dic[bn]['left'].keys())
print bn
return
if timer:
assert (False)
def bag_file_loader_thread(thread_id,command_dic,data_path,BagFolder_dic,BagFolders_weighted,delay_before_delete,loaded_bag_files_names,played_bagfile_dic):
while True:
state = command_dic[thread_id]
command = command_dic[thread_id]
if command == 'pause':
if state == 'pause':
pass
else:
state = 'pause'
cprint(d2s('Pausing thread ',thread_id),'yellow','on_blue')
time.sleep(1)
continue
if command == 'start' and state == 'pause':
state = 'running'
cprint(d2s('Unpausing thread ',thread_id),'yellow','on_blue')
if command == 'stop':
cprint(d2s('Stopping thread ',thread_id),'yellow','on_blue')
return
total_num_bag_files = 0
for bf in BagFolder_dic.keys():
total_num_bag_files += len(BagFolder_dic[bf]['bag_files'])
#print((len(loaded_bag_files_names),total_num_bag_files))
if len(loaded_bag_files_names) >= 0.9*total_num_bag_files:
cprint("90% of bag files loaded.")
time.sleep(10)
if len(loaded_bag_files_names) > 1000:
cprint('\n\nTHREAD:: pause before deleting '+bf+'\n\n,,','blue','on_red')
time.sleep(delay_before_delete)
played_bagfile_dic_keys = []
played_bagfile_dic_values = []
for b in played_bagfile_dic.keys():
played_bagfile_dic_keys.append(b)
played_bagfile_dic_values.append(played_bagfile_dic[b])
indicies = [i[0] for i in sorted(enumerate(played_bagfile_dic_values),key=lambda x:x[1])]
indicies.reverse()
ctr = 0
for i in indicies:
if if ctr >= 25: #ctr >= 0.25*len(indicies): # if ctr >= 25:
break
bf = played_bagfile_dic_keys[i]
if bf in loaded_bag_files_names:
cprint('THREAD:: deleting '+bf,'blue','on_red')
r = loaded_bag_files_names[bf]
loaded_bag_files_names.pop(bf)
BagFolder = BagFolder_dic[r]
BagFolder['bag_file_image_data'].pop(bf)
ctr += 1
if True: #try:
run = random.choice(BagFolders_weighted)
BagFolder = BagFolder_dic[run]
if type(BagFolder) != dict:
continue
dic_keys = ['bag_file_image_data','good_bag_timestamps','binned_timestamps','binned_steers','bid_timestamps']
for dk in dic_keys:
if dk not in BagFolder:
BagFolder[dk] = {}
if len(BagFolder['bag_file_num_dic']) > 0:
try:
bf = fname(random.choice(BagFolder['bag_file_num_dic']))
except:
continue
if bf in BagFolder['bag_file_image_data']:
continue
#cprint('bf = ' + bf,'red','on_white')
bag_file_path = opj(data_path,'rgb_1to4',run,bf)
#print bag_file_path
BagFolder['bag_file_image_data'][bf] = Bag_File.load_images(bag_file_path,color_mode="rgb8",include_flip=True)
loaded_bag_files_names[bf] = run
bid = BagFolder['bag_file_image_data'][bf]
bag_left_timestamps = sorted(bid['left'].keys())
good_bag_timestamps = list(set(BagFolder['data']['good_start_timestamps']) & set(bag_left_timestamps))
binned_timestamps = [[],[]]
binned_steers = [[],[]]
for t in good_bag_timestamps:
steer = BagFolder['left_image_bound_to_data'][t]['steer']
if steer < 43 or steer > 55:
binned_timestamps[0].append(t)
binned_steers[0].append(steer)
else:
binned_timestamps[1].append(t)
binned_steers[1].append(steer)
BagFolder['good_bag_timestamps'][bf] = good_bag_timestamps
BagFolder['binned_timestamps'][bf] = binned_timestamps
BagFolder['binned_steers'][bf] = binned_steers
BagFolder['bid_timestamps'][bf] = sorted(bid['left'].keys())
else: #except Exception as e:
cprint("THREAD:: ********** Exception ***********************",'red')
print(e.message, e.args)
if False:
BagFolder_dic = {}
bag_img_dic = {}
bn = 'direct_local_21Nov16_17h39m00s_Mr_Yellow/bair_car_2016-11-21-18-03-39_30.bag.pkl'
run_name = bn.split('/')[0]
bf = fname(bn)
cprint('loading ' + opj(run_name, 'Bag_Folder.pkl'), 'yellow', 'on_red')
BagFolder_dic[run_name] = load_obj(
opj(meta_dir, run_name, 'Bag_Folder.pkl'))
bag_img_dic[bn] = Bag_File.load_images(opj(rgb_1to4_dir, bn),
color_mode="rgb8",
include_flip=True)
bag_names_dic[bn] == True
good_bag_timestamps = list(
set(BagFolder_dic[run_name]['data']['good_start_timestamps'])
& set(bag_img_dic[bn]['left'].keys()))
if len(good_bag_timestamps) < 100:
if verbose:
print(d2n('\t', bn, ' len(good_bag_timestamps) < 100'))
def load_bag_file(bag_path, BagFolder_dic, bag_img_dic, skip_bag_dic,
bag_names_dic, meta_dir, rgb_1to4_dir):
bn = bag_path
def bag_file_loader_thread(thread_id,command_dic,data_path,BagFolder_dic,BagFolders_weighted,delay_before_delete,loaded_bag_files_names,played_bagfile_dic):
while True:
state = command_dic[thread_id]
command = command_dic[thread_id]
if command == 'pause':
if state == 'pause':
pass
else:
state = 'pause'
cprint(d2s('Pausing thread ',thread_id),'yellow','on_blue')
time.sleep(1)
continue
if command == 'start' and state == 'pause':
state = 'running'
cprint(d2s('Unpausing thread ',thread_id),'yellow','on_blue')
if command == 'stop':
cprint(d2s('Stopping thread ',thread_id),'yellow','on_blue')
return
total_num_bag_files = 0
for bf in BagFolder_dic.keys():
total_num_bag_files += len(BagFolder_dic[bf]['bag_files'])
#print((len(loaded_bag_files_names),total_num_bag_files))
if len(loaded_bag_files_names) >= 0.9*total_num_bag_files:
cprint("90% of bag files loaded.")
time.sleep(10)
if len(loaded_bag_files_names) > 1000:
cprint('\n\nTHREAD:: pause before deleting '+bf+'\n\n,,','blue','on_red')
time.sleep(delay_before_delete)
played_bagfile_dic_keys = []
played_bagfile_dic_values = []
for b in played_bagfile_dic.keys():
played_bagfile_dic_keys.append(b)
played_bagfile_dic_values.append(played_bagfile_dic[b])
indicies = [i[0] for i in sorted(enumerate(played_bagfile_dic_values),key=lambda x:x[1])]
indicies.reverse()
ctr = 0
for i in indicies:
if ctr >= 25: #ctr >= 0.25*len(indicies): # if ctr >= 25:
break
bf = played_bagfile_dic_keys[i]
if bf in loaded_bag_files_names:
cprint('THREAD:: deleting '+bf,'blue','on_red')
r = loaded_bag_files_names[bf]
loaded_bag_files_names.pop(bf)
BagFolder = BagFolder_dic[r]
BagFolder['bag_file_image_data'].pop(bf)
ctr += 1
if True: #try:
run = random.choice(BagFolders_weighted)
BagFolder = BagFolder_dic[run]
if type(BagFolder) != dict:
continue
dic_keys = ['bag_file_image_data','good_bag_timestamps','binned_timestamps','binned_steers','bid_timestamps']
for dk in dic_keys:
if dk not in BagFolder:
BagFolder[dk] = {}
if len(BagFolder['bag_file_num_dic']) > 0:
try:
bf = fname(random.choice(BagFolder['bag_file_num_dic']))
except:
continue
if bf in BagFolder['bag_file_image_data']:
continue
#cprint('bf = ' + bf,'red','on_white')
bag_file_path = opj(data_path,'rgb_1to4',run,bf)
#print bag_file_path
BagFolder['bag_file_image_data'][bf] = Bag_File.load_images(bag_file_path,color_mode="rgb8",include_flip=True)
loaded_bag_files_names[bf] = run
bid = BagFolder['bag_file_image_data'][bf]
bag_left_timestamps = sorted(bid['left'].keys())
good_bag_timestamps = list(set(BagFolder['data']['good_start_timestamps']) & set(bag_left_timestamps))
binned_timestamps = [[],[]]
binned_steers = [[],[]]
for t in good_bag_timestamps:
steer = BagFolder['left_image_bound_to_data'][t]['steer']
if steer < 43 or steer > 55:
binned_timestamps[0].append(t)
binned_steers[0].append(steer)
else:
binned_timestamps[1].append(t)
binned_steers[1].append(steer)
BagFolder['good_bag_timestamps'][bf] = good_bag_timestamps
BagFolder['binned_timestamps'][bf] = binned_timestamps
BagFolder['binned_steers'][bf] = binned_steers
BagFolder['bid_timestamps'][bf] = sorted(bid['left'].keys())
else: #except Exception as e:
cprint("THREAD:: ********** Exception ***********************",'red')
print(e.message, e.args)
def bag_file_loader_thread(BF_dic, BF_dic_keys_weights, delay_before_delete,
loaded_bag_files_names, played_bagfile_dic):
while True:
if bag_file_loader_thread_please_exit:
cprint('THREAD:: exiting bag_file_loader_thread()')
return
elif not thread_please_load_data:
time.sleep(1)
else:
if len(loaded_bag_files_names) > 1000:
cprint('\n\nTHREAD:: pause before deleting ' + bf + '\n\n,,',
'blue', 'on_red')
time.sleep(delay_before_delete)
played_bagfile_dic_keys = []
played_bagfile_dic_values = []
for b in played_bagfile_dic.keys():
played_bagfile_dic_keys.append(b)
played_bagfile_dic_values.append(played_bagfile_dic[b])
indicies = [
i[0] for i in sorted(enumerate(played_bagfile_dic_values),
key=lambda x: x[1])
]
indicies.reverse()
ctr = 0
for i in indicies:
if ctr >= 25:
break
bf = played_bagfile_dic_keys[i]
if bf in loaded_bag_files_names:
#bf = a_key(loaded_bag_files_names)
cprint('THREAD:: deleting ' + bf, 'blue', 'on_red')
r = loaded_bag_files_names[bf]
loaded_bag_files_names.pop(bf)
BF = BF_dic[r]
BF['bag_file_image_data'].pop(bf)
ctr += 1
if True: #try:
r = random.choice(BF_dic.keys())
BF = BF_dic[r]
if type(BF) != dict:
continue
dic_keys = [
'bag_file_image_data', 'good_bag_timestamps',
'binned_timestamps', 'binned_steers', 'bid_timestamps'
]
for dk in dic_keys:
if dk not in BF:
BF[dk] = {}
#print BF['bag_file_num_dic']
if len(BF['bag_file_num_dic']) > 0:
try:
bf = random.choice(BF['bag_file_num_dic'])
except:
continue
if bf in BF['bag_file_image_data']:
continue
BF['bag_file_image_data'][bf] = Bag_File.load_images(
bf, color_mode="rgb8", include_flip=True)
loaded_bag_files_names[bf] = r
bid = BF['bag_file_image_data'][bf]
bag_left_timestamps = sorted(bid['left'].keys())
good_bag_timestamps = list(
set(BF['data']['good_start_timestamps'])
& set(bag_left_timestamps))
#cprint(d2s('THREAD:: ',bf.split('/')[-1],'len(good_bag_timestamps) =',len(good_bag_timestamps)),'blue')
binned_timestamps = [[], []]
binned_steers = [[], []]
for t in good_bag_timestamps:
steer = BF['left_image_bound_to_data'][t]['steer']
if steer < 43 or steer > 55:
binned_timestamps[0].append(t)
binned_steers[0].append(steer)
else:
binned_timestamps[1].append(t)
binned_steers[1].append(steer)
BF['good_bag_timestamps'][bf] = good_bag_timestamps
BF['binned_timestamps'][bf] = binned_timestamps
BF['binned_steers'][bf] = binned_steers
BF['bid_timestamps'][bf] = sorted(bid['left'].keys())
else: #except Exception as e:
cprint("THREAD:: ********** Exception ***********************",
'red')
print(e.message, e.args)
def bag_file_loader_thread(
BF_dic, BF_dic_keys_weights, delay_before_delete, loaded_bag_files_names, played_bagfile_dic
):
while True:
if bag_file_loader_thread_please_exit:
cprint("THREAD:: exiting bag_file_loader_thread()")
return
elif not thread_please_load_data:
time.sleep(1)
else:
if len(loaded_bag_files_names) > 1000:
cprint("\n\nTHREAD:: pause before deleting " + bf + "\n\n,,", "blue", "on_red")
time.sleep(delay_before_delete)
played_bagfile_dic_keys = []
played_bagfile_dic_values = []
for b in played_bagfile_dic.keys():
played_bagfile_dic_keys.append(b)
played_bagfile_dic_values.append(played_bagfile_dic[b])
indicies = [i[0] for i in sorted(enumerate(played_bagfile_dic_values), key=lambda x: x[1])]
indicies.reverse()
ctr = 0
for i in indicies:
if ctr >= 25:
break
bf = played_bagfile_dic_keys[i]
if bf in loaded_bag_files_names:
# bf = a_key(loaded_bag_files_names)
cprint("THREAD:: deleting " + bf, "blue", "on_red")
r = loaded_bag_files_names[bf]
loaded_bag_files_names.pop(bf)
BF = BF_dic[r]
BF["bag_file_image_data"].pop(bf)
ctr += 1
if True: # try:
r = random.choice(BF_dic.keys())
BF = BF_dic[r]
if type(BF) != dict:
continue
dic_keys = [
"bag_file_image_data",
"good_bag_timestamps",
"binned_timestamps",
"binned_steers",
"bid_timestamps",
]
for dk in dic_keys:
if dk not in BF:
BF[dk] = {}
# print BF['bag_file_num_dic']
if len(BF["bag_file_num_dic"]) > 0:
try:
bf = random.choice(BF["bag_file_num_dic"])
except:
continue
if bf in BF["bag_file_image_data"]:
continue
BF["bag_file_image_data"][bf] = Bag_File.load_images(bf, color_mode="rgb8", include_flip=True)
loaded_bag_files_names[bf] = r
bid = BF["bag_file_image_data"][bf]
bag_left_timestamps = sorted(bid["left"].keys())
good_bag_timestamps = list(set(BF["data"]["good_start_timestamps"]) & set(bag_left_timestamps))
# cprint(d2s('THREAD:: ',bf.split('/')[-1],'len(good_bag_timestamps) =',len(good_bag_timestamps)),'blue')
binned_timestamps = [[], []]
binned_steers = [[], []]
for t in good_bag_timestamps:
steer = BF["left_image_bound_to_data"][t]["steer"]
if steer < 43 or steer > 55:
binned_timestamps[0].append(t)
binned_steers[0].append(steer)
else:
binned_timestamps[1].append(t)
binned_steers[1].append(steer)
BF["good_bag_timestamps"][bf] = good_bag_timestamps
BF["binned_timestamps"][bf] = binned_timestamps
BF["binned_steers"][bf] = binned_steers
BF["bid_timestamps"][bf] = sorted(bid["left"].keys())
else: # except Exception as e:
cprint("THREAD:: ********** Exception ***********************", "red")
print (e.message, e.args)
