def CopyConfigCacheAndLogs(data_dir, segment_list):
print(getTime() +
"\033[1;32m [INFO]\033[0m copying config and log .........\n")
for seg_point in segment_list:
output_dir_path = seg_point["output_dir"]
log_dir_list = ["cache", "config", "versions", "simulator_scenario"]
for log_dir in log_dir_list:
try:
log_dir_path = os.path.join(output_dir_path, log_dir)
if os.path.isdir(log_dir_path):
shutil.rmtree(log_dir_path)
if os.path.exists(os.path.join(data_dir, log_dir)):
shutil.copytree(os.path.join(data_dir, log_dir),
log_dir_path)
except Exception as e:
print(getTime() +
"\033[1;31m [ERROR]\033[0m copy cache or config failed ")
cv22_path = os.path.join(data_dir, 'cv22')
cv22_output_path = os.path.join(output_dir_path, 'cv22')
cv22_copy_list = ["canlog", "config"]
for cv22_dir in cv22_copy_list:
try:
cv22_dir_path = os.path.join(cv22_output_path, cv22_dir)
if os.path.isdir(cv22_dir_path):
shutil.rmtree(cv22_dir_path)
if os.path.exists(os.path.join(cv22_path, cv22_dir)):
shutil.copytree(os.path.join(cv22_path, cv22_dir),
cv22_dir_path)
except Exception as e:
print(getTime() +
"\033[1;31m [ERROR]\033[0m copy cv22 failed ")
print(getTime() +
"\033[1;32m [INFO]\033[0m copying config and log completly\n")
def CutRecs(rec_files, segment_list):
print(getTime() + "\033[1;32m [INFO]\033[0m Cuting rec files......")
for rec_file in rec_files:
print("checkpoint58")
point_list = []
print('\n.................', rec_file)
sd_index_list = JudgeTheStartAndEndOfRec(rec_file, segment_list)
print(len(sd_index_list), len(segment_list))
print(sd_index_list)
print(segment_list)
if sd_index_list == [0, 0] or len(sd_index_list) != len(segment_list):
continue
for i in range(len(segment_list)):
seg_point = segment_list[i]
output_dir = seg_point["output_dir"]
start_index = sd_index_list[i][0]
end_index = sd_index_list[i][1]
if start_index == 0 or end_index == 0:
continue
point_list.append({
"start_index": start_index,
"end_index": end_index,
"output_dir": output_dir
})
CutRec(rec_file, point_list)
print(getTime() + "\033[1;32m [INFO]\033[0m Cuting rec files completly\n")
def CutDpcs(dpc_files, segment_list):
print(getTime() + "\033[1;32m [INFO]\033[0m Cuting dpc bag......")
point_list = []
print("checkpoint1008f")
print(dpc_files)
for i in range(len(dpc_files)):
dpc_file = dpc_files[i]
print("checkpoint1007")
print(dpc_file)
print(segment_list)
# judge the start and end
for seg_point in segment_list:
time_point = seg_point["time_point"]
front_duration = seg_point["front_duration"]
behind_duration = seg_point["behind_duration"]
output_dir = seg_point["output_dir"]
begin_secs = time_point / 1000000 - front_duration
end_secs = time_point / 1000000 + behind_duration
point_list.append({
"begin_secs": begin_secs,
"end_secs": end_secs,
"output_dir": output_dir
})
CutDpcbag(dpc_file, point_list)
print("checkpoint17")
print(getTime() + "\033[1;32m [INFO]\033[0m Cuting dpc bag completly\n")
def cutVideoAndTxt(video_list, segment_list, timestamp_bin_file):
print("test1")
print(segment_list)
print(getTime()+"\033[1;32m [INFO]\033[0m Cutting video files......")
start_time = time.time()
new_bin = TimeReader(str(timestamp_bin_file))
timestamp_file = new_bin.get_time_list()
point_list = []
timestamp_list = []
timebin_list = []
video_tmp_path = 'cv22/Sliced_ADAS'
for seg_point in segment_list:
print("checkpoint1.0.1")
print(seg_point)
time_point = seg_point["time_point"]
front_duration = seg_point["front_duration"]
behind_duration = seg_point["behind_duration"]
output_dir = seg_point["output_dir"]
print("\033[1;32m checkpoint1.1.\033[0m ")
print((time_point, front_duration, behind_duration))
start_index, end_index,b,c = cut_rec_multiprocess.JudgeTheStartAndEndOfVideo(
timestamp_file, time_point, front_duration, behind_duration)
print("\033[1;32m checkpoint1.1.3\033[0m ")
print(start_index, end_index, b, c)
output_video_path = os.path.join(output_dir, video_tmp_path, "Sliced_ADAS.mp4")
print("checkpoint1.1")
if not os.path.exists(os.path.join(output_dir, video_tmp_path)):
os.makedirs(os.path.join(output_dir, video_tmp_path))
print(output_video_path)
point_list.append({"start_index": start_index, "end_index": end_index, "output_dir": output_video_path})
# timestamp_list.append({"start_index": start_index,
# "end_index": end_index,
# "output_dir": os.path.join(output_dir, video_tmp_path)})
timebin_list.append({"start_index": start_index - front_duration * 60,
"end_index": end_index + behind_duration * 60,
"output_dir": os.path.join(output_dir, video_tmp_path)})
print("\033[1;32m checkpoint1.1.4\033[0m ")
print(point_list)
print(timebin_list)
# shutil.copyfile(timestamp_file,os.path.join(merged_path,"NOR_stream0.txt"))
print("\033[1;32m checkpoint1.1.5\033[0m ")
print(video_list, point_list)
# cutAdasVideos(video_list, point_list)
cutAdasVideosMultiprocess(video_list, point_list)
cut_rec_multiprocess.CutBin(timestamp_bin_file, timebin_list, point_list)
print(getTime()+"\033[1;32m [INFO]\033[0m Cuting video files completly, consuming {:.3f}s".format(time.time()-start_time))
def mergeTimeTxt(time_bin_path,output_path):
time_bin_file_list = cut_rec_multiprocess.GetMatchedFilePaths(time_bin_path,"*",[".bin"])
output_bin_file = os.path.join(output_path,'timestamp.txt')
if not os.path.exists(output_bin_file):
os.mknod(output_bin_file)
if time_bin_file_list == []:
print(getTime()+"\033[1;31m [ERROR]\033[0m not get any time.bin file")
return 0
ts_file_list = sorted(time_bin_file_list)
i = 0
def judgeFileSizeAndExist(judge_file, check_size=0, upsize=200000):
"as the function name descripted"
print("checkpoint4.1")
if os.path.exists(judge_file) and \
os.path.getsize(judge_file) / float(1024 * 1024) >= 0 and \
round(os.path.getsize(judge_file) / float(1024 * 1024), 1) < upsize:
print("checkpoint4.2")
return True
else:
print(getTime()+"\033[1;31m [ERROR]\033[0m file:", judge_file, " is\033[1;31m wrong\033[0m\n")
print("checkpoint4.3")
return False
def CutSimulatorScenario(data_dir, point_list):
'''
input:
rec_file:str
point_list:[{"time_point":121545, "front_duration":15,"behind_duration":5,"output_dir":/path/to/output}]
'''
config_ = loadTag('config/data_pipeline_config.json', '')
try:
if os.path.isfile(
os.path.join(
config_["senseauto_path"],
"senseauto-simulation/node/build/module/simulator/tools/scenario_log_tools/scenario_log_razor"
)):
razor = os.path.join(
config_["senseauto_path"],
"senseauto-simulation/node/build/module/simulator/tools/scenario_log_tools/scenario_log_razor"
)
elif os.path.isfile(
os.path.join(
config_["senseauto_path"],
"senseauto-simulation/node/module/simulator/tools/scenario_log_tools/scenario_log_razor"
)):
razor = os.path.join(
config_["senseauto_path"],
"senseauto-simulation/node/module/simulator/tools/scenario_log_tools/scenario_log_razor"
)
else:
print("checkpoint53")
print("cannot find the simulator_scenario_log_razor, exit")
for seg_point in point_list:
time_point = seg_point["time_point"]
front_duration = seg_point["front_duration"]
behind_duration = seg_point["behind_duration"]
output_dir = seg_point["output_dir"]
simulator_file = os.path.join(
data_dir, 'simulator_scenario/simulator_scenario_log.bin')
outout_dir_path = os.path.join(output_dir, 'simulator_scenario')
print("checkpoint54")
razor_cmd = "{} 1 {} {} {} {} {}".format(
razor, simulator_file, outout_dir_path,
str(int(time_point // 1000000)), str(front_duration + 15),
str(behind_duration + 20))
print("checkpoint55")
print(razor_cmd)
os.system(razor_cmd)
print("checkpoint56")
except Exception as e:
print("checkpoint57")
print(getTime() +
"\033[1;31m [ERROR]\033[0m cut simulator.bin failed ")
def CutVideos(video_path, point_list):
'''
input:
video_path:str
point_list:[{"start_index":121545, "end_index":1245454,"output_dir":/path/to/output}]
'''
if point_list == []:
return
# read video
fourcc = cv2.VideoWriter_fourcc(*"mp4v")
cap = cv2.VideoCapture(video_path)
if not cap.isOpened():
print("Cannot open video capture: {}".format(video_path))
width = cap.get(cv2.CAP_PROP_FRAME_WIDTH)
height = cap.get(cv2.CAP_PROP_FRAME_HEIGHT)
fps = cap.get(cv2.CAP_PROP_FPS)
print("code: {}, width: {}, height: {}, fps: {}".format(
fourcc, width, height, fps))
for seg_point in point_list:
output_dir = seg_point["output_dir"]
start_index = seg_point["start_index"]
end_index = seg_point["end_index"]
output_video_path = output_dir + "/" + os.path.basename(video_path)
writer = cv2.VideoWriter(output_video_path, fourcc, int(fps),
(int(width), int(height)))
if not writer.isOpened():
print("Cannot open video writer: {}".format(output_video_path))
counts = start_index
try:
cap.set(cv2.CAP_PROP_POS_FRAMES, start_index)
while counts <= end_index:
flag, frame = cap.read()
counts += 1
writer.write(frame)
except:
print(getTime() + "\033[1;31m [ERROR]\033[0m cut video failed ")
return 0
writer.release()
cap.release()
def cutTimetxt(merge_video,splited_path,segment_point):
start_time = (segment_point['time_point'] - segment_point['ofront_duration'] *1000000)*1000
end_time = (segment_point['time_point'] + segment_point['obehind_duration'] * 1000000) * 1000
time_bin_file_list = cut_rec_multiprocess.GetMatchedFilePaths(merge_video, "*", [".bin"])
output_bin_file = os.path.join(splited_path, 'splited_timestamp.txt')
if not os.path.exists(output_bin_file):
os.mknod(output_bin_file)
if time_bin_file_list == []:
print(getTime()+"\033[1;31m [ERROR]\033[0m not get any time.bin file")
return 0
ts_file_list = sorted(time_bin_file_list)
i = 0
with open(output_bin_file, 'w') as reuslt:
for ts_file in ts_file_list:
print(ts_file)
if os.path.getsize(ts_file) == 0:
continue
reader = TimeReader(ts_file)
timestamps = reader.get_time_list()
for time in timestamps:
if int(time) > int(start_time) and int(time) < int(end_time):
reuslt.write(str(i) + ', ' + str(time) + '\n')
i += 1
def main(data_dir, segment_list):
pattern = "port_*"
print("checkpoint46")
print(data_dir)
basler_video_file = GetMatchedFilePaths(data_dir, "port_0*", [".avi"],
True)
basler_txt_file = GetMatchedFilePaths(data_dir, "port_0*", [".txt"], True)
video_files = GetMatchedFilePaths(data_dir,
pattern,
formats=[".avi", ".h264", "mp4"],
recursive=False)
for video_file in video_files:
print(
" ---------- {}".format(video_file)
) # 2021_05_07_20_22_12_AutoCollect/port_0_camera_0_2021_5_7_19_36_1.avi
print("checkpoint47")
timestamp_files = GetMatchedFilePaths(
data_dir,
# 2021_05_07_20_22_12_AutoCollect/port_0_camera_0_2021_5_7_19_36_1.txt
pattern,
[".txt"],
recursive=False)
print("checkpoint48")
hmi_files = GetMatchedFilePaths(data_dir, "@*", [".hmi"], recursive=False)
for hmi_file in hmi_files:
print(" ---------- {}".format(hmi_file)) # did not find hmi file
screen_files = []
screen_txts = []
screen_cast_path = os.path.join(data_dir, 'screen_cast')
print("checkpoint49")
print(screen_cast_path)
if os.path.exists(screen_cast_path):
print("checkpoint50")
screen_files = GetMatchedFilePaths(screen_cast_path,
formats=[".mp4"],
recursive=False)
screen_txts = GetMatchedFilePaths(screen_cast_path,
formats=[".txt"],
recursive=False)
for timestamp_file in timestamp_files:
print(" ---------- {}".format(timestamp_file))
print("checkpoint51")
rec_files_ori = GetMatchedFilePaths(os.path.join(data_dir,
'sensors_record'),
formats=[".rec"],
recursive=False)
rec_files = copy.deepcopy(rec_files_ori)
for rec_file in rec_files_ori:
print(" ---------- {}".format(rec_file))
try:
if os.path.exists(rec_file) and os.path.getsize(rec_file) == 0:
os.remove(rec_file)
rec_files.remove(rec_file)
except:
print(getTime() +
"\033[1;31m [ERROR]\033[0m move 0-size rec failed ")
print("checkpoint52")
dpc_files = GetMatchedFilePaths(data_dir,
formats=[".bag"],
recursive=False)
for dpc_file in dpc_files:
print(" ---------- {}".format(dpc_file))
pool1 = multiprocessing.Pool(processes=12)
pool1.apply_async(CutRecs, args=(rec_files, segment_list))
pool1.apply_async(CutDpcs, args=(dpc_files, segment_list))
pool1.close()
pool1.join()
pool2 = multiprocessing.Pool(processes=12)
# CopyConfigCacheAndLogs(data_dir, segment_list)
pool2.apply_async(CopyConfigCacheAndLogs, args=(data_dir, segment_list))
pool2.close()
pool2.join()
print(getTime() +
"\033[1;32m [INFO]\033[0m Cutting Basler Video .........\n")
if len(basler_video_file) > 0 and len(basler_txt_file) > 0:
CutBasler(basler_video_file, basler_txt_file, segment_list)
print(getTime() +
"\033[1;32m [INFO]\033[0m Cutting Screencast Video .........\n")
pool = multiprocessing.Pool(processes=12)
if len(screen_files) > 0 and len(screen_txts) > 0:
pool.apply_async(CutScreenCast,
args=(screen_files, screen_txts, segment_list))
pool.close()
pool.join()
pool = multiprocessing.Pool(processes=12)
if len(video_files) > 0 and os.path.exists(video_files[0]):
print("checkpoint")
if len(hmi_files) > 0 and os.path.exists(hmi_files[0]):
print(getTime() +
"\033[1;32m [INFO]\033[0m Cutting HMI Video .........\n")
pool.apply_async(CutHmiFile, args=(hmi_files[0], segment_list))
pool.close()
pool.join()
print("checkpoint51.5")
print(data_dir, segment_list)
print(getTime() +
"\033[1;32m [INFO]\033[0m Cutting ADAS Video .........\n")
adas_pipeline.adasMainProcess(data_dir, segment_list)
print("checkpoint52.1")
print(data_dir)
print(segment_list)