def matchBackUpTag(self, dir_path):
try:
dir_name = os.path.basename(dir_path)
test_time_list = dir_name.split('_')
if test_time_list[-1] != "AutoCollect":
return None
test_time = [int(test_time_list[i]) for i in range(1, 5)]
for backup_tag in self.backup_tag_list:
backup_tag_name = os.path.basename(backup_tag)
tag_time_str = backup_tag_name.split('(')[1].split(')')[0]
tag_time = tag_time_str.split('-')
clock_list = [43200, 1440, 60, 1]
tag_gap = 0
for i, clock_time in enumerate(clock_list):
add_time = 1
tag_gap += (test_time[i] - int(tag_time[i])) * clock_time
if abs(tag_gap) < 4 and add_time > 0:
print(getTime() + "\033[1;32m [INFO]\033[0m", test_time,
tag_time)
tag_data = loadTag(self.input_data_path, backup_tag)
return tag_data
return None
except Exception as e:
print(getTime() + "\033[1;31m [ERROR]\033[0m backup tag not found")
def main(self, upload=False):
'''
main pipeline for collected data check & segment & upload
:param upload: whether upload
'''
start_time = time.time()
print(getTime() + "\033[1;32m [INFO]\033[0m Starting EVERYTHING")
self.mainSegment(self.file_list, upload)
print(
getTime() +
"\033[1;32m [INFO]\033[0m Finished everything , consuming {:.3f}s".
format(time.time() - start_time))
def getLocalizationResult(self, dir_path):
'''get the localization result'''
localization_result = defaultdict(lambda: {})
result_file_path = os.path.join(dir_path, 'logs/localization_eval')
localization_json = loadTag(result_file_path,
tag_file_name='evaluation_result.json')
if localization_json is not None:
try:
localization_result["Grade"] = localization_json["Grade"]
localization_result["Integrity"] = localization_json[
"Integrity"]
localization_result["Odometer(km)"] = localization_json[
"Odometer(km)"]
localization_result["Setting"] = localization_json["Setting"]
localization_result["Mileage(km)"] = localization_json.get(
"Mileage(km)")
localization_result["Mileage(km)"] = localization_json.get(
"Mileage(km)")
localization_result["Stability"] = localization_json.get(
"Stability")
localization_result["Tags_Num"] = localization_json.get(
"Tags_Num")
if localization_json.get(
"Odometer(km)") > 1.0 and localization_json.get(
"Odometer(km)") < 100.0:
self.tag_info["test_mileage"] = localization_json.get(
"Odometer(km)")
except Exception as e:
print(getTime() +
"\033[1;31m [ERROR]\033[0m get LMR eval info error ")
if self.tag_info["test_mileage"] < 1.0 or self.tag_info[
"test_mileage"] > 100.0:
self.tag_info["test_mileage"] = 18.0
return localization_result
def TransferPost(self, data_tag):
"post the data tag to senseFT"
curl = \
'https://fieldtest.sensetime.com/task/daemon/update/tag'
post_result = requests.post(curl,
headers=self.headerdata,
data=json.dumps(data_tag))
print(getTime() + "\033[1;32m [INFO]\033[0m ", post_result.text, '\n')
if post_result.text != u'{"result":"true"}':
try:
logger(2,
"Uploaded dir:" + data_tag["data_link"],
LOG_FILE="upload_list/post_failed_file.log")
saveTag('upload_list/', data_tag,
data_tag["file_name"] + '.json')
except Exception as e:
print(getTime() +
"\033[1;31m [ERROR]\033[0m save post-failed tag error ")
def falseDataArchive(self, dir_path, false_reason):
try:
#generate_data_report.main(dir_path, False, false_reason)
self.falseDataUpload(dir_path)
self.dirArchive(dir_path, tag_info=None, check_result=False)
return 1
except Exception as e:
logger(1, str(e), LOG_FILE="upload_list/error.log")
print(getTime() + "\033[1;31m [ERROR]\033[0m archive error ")
return 1
def dirArchive(self, dir_path, tag_info, check_result=True):
"archive the dri which has been uploaded"
input_data_path, dir_name = os.path.split(dir_path)
if not check_result:
try:
date_id = ''.join([
'/False_',
dir_name.split('_', -1)[1], '_',
dir_name.split('_', -1)[2]
])
except Exception as e:
date_id = '/False_ARH'
archive_path = ''.join([input_data_path, date_id, '_ARH'])
if not os.path.exists(archive_path):
os.makedirs(archive_path)
try:
shutil.move(input_data_path + '/' + dir_name, archive_path)
except Exception as e:
print("checkpoint1001")
print(getTime() +
"\033[1;31m [ERROR]\033[0m move dir to ARH failed ")
return archive_path
else:
test_date = tag_info["test_date"].split('_', 1)[1]
try:
date_id = ''.join(
['/', test_date, '_', tag_info["test_car_id"]])
except Exception as e:
date_id = 'True_ARH'
archive_path = ''.join([input_data_path, date_id, '_ARH'])
if not os.path.exists(archive_path):
os.makedirs(archive_path)
try:
shutil.move(''.join([input_data_path, '/', dir_name]),
archive_path)
except Exception as e:
print("checkpoint1002")
print(getTime() +
"\033[1;31m [ERROR]\033[0m move dir to ARH failed ")
return archive_path
def reouteCheckFi(self, input_path, vehicle_id):
try:
rec_parser = os.path.join(
self.config_["senseauto_path"],
"tools/rec_parser/scripts/offline_data_process.sh")
output_path = os.path.join(input_path, 'rec_parser')
config_path = os.path.join(self.config_["senseauto_path"],
"system/config/vehicle/", vehicle_id)
enu_config_path = os.path.join(
self.config_["senseauto_path"],
"tools/rec_parser/config/sh_enu_origin.txt")
rec_parse_cmd = "bash {} -i {} -o {} -c {} -e {}".format(
rec_parser, input_path, output_path, config_path,
enu_config_path)
gps_file = os.path.join(output_path, 'gps.txt')
os.system(rec_parse_cmd)
with open(gps_file, 'r') as r:
lines = r.readlines()
with open(gps_file, 'w') as fw:
for l in lines:
if "GPS-time" in l:
continue
items = l.split(',')[1:3]
a = items[1]
items[1] = items[0].replace("\n", "")
items[0] = a.replace("\n", "")
items = ''.join([
"unknown unknown unknown unknown unknown ",
str(items[0]), ', ',
str(items[1]) + ", unknown\n"
])
fw.write(items)
os.system(
"python3 ~/Codes/RouteMeFi-master/routemefi.py --input_path " +
gps_file +
" --db_path ~/Codes/sh_gps/database.txt --save_dir ~/Codes/sh_gps/ --query_merge"
)
record_tag = {}
record_tag["input_dir"] = input_path
record_tag["tagging_module"] = 0
self.case_tagging.tag_main(self.tag_info['global_tag'], input_path,
[record_tag])
overlap_json = loadTag(output_path, 'overlap_result.json')
return 'overlap_' + str(overlap_json["overlapped"])
except Exception as e:
print(getTime() + "\033[1;31m [ERROR]\033[0m get ins info error ")
def judgeFileSizeAndExist(self, dir_path, file_name, check_size=0.2):
"as the function name descripted"
judge_file = os.path.join(dir_path, file_name)
print("checkfilereleaseandsize is writing this :)")
print(file_name)
print(judge_file)
if os.path.exists(judge_file) and \
round(os.path.getsize(judge_file) / float(1024 * 1024), 1) >= check_size:
# print("no error, filesize:")
# print(os.path.getsize(judge_file))
return 1
else:
self.false_check_reasion.append(file_name)
print(getTime() + "\033[1;31m [ERROR]\033[0m file:", file_name,
" is\033[1;31m wrong\033[0m\n")
print("filesize:")
print(os.path.getsize(judge_file))
return 0
def getPredictionResult(self, dir_path):
"add the control evaluation result to tag"
ap_result = defaultdict(lambda: {})
result_file_path = os.path.join(dir_path,
'prediction_evaluation/result')
ap_json = loadTag(result_file_path, tag_file_name='result.json')
if ap_json is not None:
try:
ap_result["quality"] = ap_json["quality"]
except Exception as e:
logger(1, str(e), LOG_FILE="upload_list/error.log")
print(getTime() +
"\033[1;31m [ERROR]\033[0m get ap eval info error ")
ap_result["quality"] = {}
ap_result["quality"]['level'] = "bad"
return ap_result
def getControlResult(self, dir_path):
"add the control evaluation result to tag"
control_result = defaultdict(lambda: {})
result_file_path = os.path.join(dir_path, 'logs/control_eval')
control_json = loadTag(result_file_path,
tag_file_name='control_eval_results.json')
if control_json is not None:
try:
control_result = control_json["control_result"]
if 'stop_error' in control_result['control_precision']:
control_result['control_precision']['stop_error'][
'std'] = 0.901
except Exception as e:
logger(1, str(e), LOG_FILE="upload_list/error.log")
print(getTime() +
"\033[1;31m [ERROR]\033[0m get control eval info error ")
return control_result
def mainUpload(self, dir_path, upload):
'''
upload and then archive data
:param dir_path_without: data path end without /
'''
print("checkpoint36")
print(dir_path)
tag_info = loadTag(dir_path)
print(tag_info)
dir_name = os.path.split(dir_path)[1]
print(dir_name)
try:
self.data_upload(dir_path, tag_info, slice=False, upload=upload)
archive_path = self.dirArchive(dir_path, tag_info)
generate_data_report.main(archive_path + '/' + dir_name, True)
if os.path.exists(dir_path + "_slice/"):
self.data_upload(dir_path, tag_info, slice=True, upload=upload)
self.dirArchive(dir_path + "_slice", tag_info)
return 0
except Exception as e:
print(traceback.format_exc())
logger(1, str(e), LOG_FILE="upload_list/error.log")
print(getTime() + "\033[1;31m [ERROR]\033[0m upload error ")
return 1
def data_upload(self, dir_path, tag_info, slice=False, upload=True):
'data_upload with aws'
dir_name = os.path.split(dir_path)[1]
vehicle_id = tag_info["test_car_id"].replace("-", "")
data_month = tag_info["test_date"].rsplit("_", 1)[0]
try:
feature = False if tag_info["issue_id"] == ["repo_master"
] else True
except Exception as e:
feature = False
task_id = str(tag_info["task_id"]) + '/' if feature else ''
if slice:
upload_path = ''.join([dir_path, "_slice/ "])
dst_path = ''.join([
self.getDataCollectionDstLink(tag_info, data_month,
slice=True),
tag_info["test_date"], '/', vehicle_id, '/', task_id
])
self.sliceDataCheck(dir_path + '_slice/')
else:
upload_path = ''.join([dir_path, '/ '])
tag_path = ''.join([
self.getDataCollectionDstLink(tag_info,
data_month,
slice=False),
tag_info["test_date"], '/', vehicle_id, '/', task_id, dir_name
])
dst_path = ''.join([tag_path, '/'])
tag_info["data_link"] = tag_path
tag_info["data_type"] = "raw"
tag_info['aws_endpoint'] = self.end_point
if tag_info["test_type"] == 1:
aa, eval_result = self.getPPEvalResult(tag_path)
if aa:
tag_info["pp_evaluation"] = eval_result
if tag_info["test_type"] == 3 and self.config_["evaluation"]:
generate_evaluation_result.generateAdasEval(
dir_path, self.config_)
tag_info["adas_evaluation"] = self.getAdasResult(tag_path)
try:
if self.false_reason != []:
for false_check in self.false_reason:
tag_info["global_tag"].append("no_" + false_check)
if self.tprfile_resut != []:
for tprofile in self.tprfile_resut:
tag_info["global_tag"].append(tprofile)
except Exception as e:
print("write global tag error")
saveTag(dir_path, tag_info, file_name='data-tag.json')
arg2 = ''.join([
"s3 cp ", upload_path, dst_path, upload_recursive + self.end_point
])
# arg_slam =''.join(["s3 cp ", os.path.join(dir_path,'logs/map_update_areas.json '),
# "s3://sz21_data_collection/slam_file/"+data_month+'/'+tag_info["test_date"]+'/'+
# vehicle_id+'/'+task_id+dir_name+'/'+tag_info.get("route")+'/map_update_areas.json',
# " --profile ad_system_common --only-show-errors --endpoint-url=" + self.end_point])
print(getTime() + "\033[1;32m [INFO] Uploading ...\033[0m ", arg2)
if upload:
star_upload_time = time.time()
file_size = getFileSize(upload_path.replace(' ', ''))
upload_result = AWS_DRIVER.main(arg2.split())
# AWS_DRIVER.main(arg_slam.split())
cost_time = time.time() - star_upload_time
upload_speed = file_size / cost_time
logger(1, "Uploaded dir:" + upload_path)
if upload_result == 0:
print (getTime()+"\033[1;32m [INFO]", dir_name + \
"\033[0m", "\033[0m has\033[1;32m uploaded successfully! Speed:\033[0m "+ str(upload_speed) +" MB/s")
else:
print(getTime() + "\033[1;32m [INFO]", dir_name + "\033[0m",
"\033[0m \033[1;32m upload failed!\033[0m")
self.TransferPost(tag_info)
def dataSegment(self, dir_name, segment_point):
"data segment pipeline"
print("checkpoint40")
print(self.tag_module_list)
segment_list = []
case_tagging_list = []
prediction_tagging_list = []
for record_tag in segment_point: # record_tag is every set
#dir_path = os.path.join(self.input_data_path, dir_name)
dir_path = self.input_data_path
tag_name = record_tag.get("tag_en") # tag_name = cloudyday etc
if not tag_name in self.tag_module_list: # special manual tagging, take over, dangerous driving etc
if "data_type" in record_tag and record_tag[
"data_type"] == "eval":
level_name, log_type, tagging_module = ["EVAL", 0, 1]
else:
level_name, log_type, tagging_module = [
"3D-Perception", 0, 1
]
if self.tag_info['test_type'] == 3:
level_name, log_type, tagging_module = ["ADAS", 0, 1]
else:
level_name = self.tag_module_list[tag_name]["level_name"]
log_type = self.tag_module_list[tag_name]["log_type"]
tagging_module = self.tag_module_list[tag_name][
"tagging_module"]
print(getTime() + "\033[1;32m [INFO] \033[0m", level_name, "==>> ",
tag_name, "==== module_type:", log_type)
if "end" in record_tag:
front_time = 2
behind_time = (record_tag["end"] - record_tag["start"]) // 1000
else:
if not tag_name in self.tag_module_list:
front_time = 20
behind_time = 10
else:
front_time = self.tag_module_list[tag_name]["front_time"]
behind_time = self.tag_module_list[tag_name]["behind_time"]
if behind_time < 6:
behind_time = 10
if "end" in record_tag:
input_timestamp = (record_tag["start"] +
record_tag["end"]) / 2000
else:
input_timestamp = record_tag["start"] / 1000
time_point = record_tag["start"] * 1000
test_date = self.tag_info["test_date"]
test_time = str(record_tag["start_format"].replace(":", "_"))
segment_dir_name = ''.join([test_date, '_', test_time])
print("checkpoint41")
output_path = ''.join([
dir_path, '_slice/', level_name, '/', tag_name, '/',
segment_dir_name
])
print(output_path)
if self.tag_info["issue_id"][0] != "repo_master" and \
(record_tag['tag_en'] == "take_over" or record_tag['tag_en'] == "Emergency_brake"):
segment_list.append({
"output_dir": output_path,
"time_point": time_point,
"front_duration": front_time,
"behind_duration": behind_time,
"log_type": log_type
})
else:
segment_list.append({
"output_dir": output_path,
"time_point": time_point,
"front_duration": front_time,
"behind_duration": behind_time,
"log_type": log_type
})
if level_name != "EVAL":
case_tagging_list.append({
"input_dir":
''.join([output_path, '/']),
"module_name":
tag_name,
"input_timestamp":
input_timestamp,
"tagging_module":
tagging_module
})
if tag_name == "abnormal_prediction_trajectory":
prediction_tagging_list.append({
"input_dir":
''.join([output_path, '/']),
"module_name":
tag_name,
"input_timestamp":
input_timestamp
})
if not os.path.exists(output_path):
print("checkpoint42")
os.makedirs(output_path)
print("checkpoint43")
task_id = '' if self.tag_info["issue_id"][
0] == "repo_master" else str(self.tag_info["task_id"]) + '/'
vehicle_id = self.tag_info["test_car_id"].replace("-", "")
module_tag_data = copy.deepcopy(self.tag_info)
data_month = self.tag_info["test_date"].rsplit("_", 1)[0]
dst_path = ''.join([
self.getDataCollectionDstLink(self.tag_info,
data_month,
slice=True),
self.tag_info["test_date"], '/', vehicle_id, '/', task_id
])
module_tag_data["data_link"] = ''.join(
[dst_path, level_name, '/', tag_name, '/', segment_dir_name])
store_path = ''.join([
dir_path, '_slice/', level_name, '/', tag_name, '/',
segment_dir_name
])
module_tag_data["origin_record_tag"] = [record_tag]
if "data_type" in record_tag and record_tag["data_type"] == "eval":
module_tag_data["data_type"] = record_tag["data_type"]
module_tag_data["data_type"] = "segment"
module_tag_data["test_type"] = 9
self.post = True
else:
module_tag_data["data_type"] = "segment"
module_tag_data["file_name"] = segment_dir_name
module_tag_data["raw_data_link"] = ''.join([
self.getDataCollectionDstLink(self.tag_info,
data_month,
slice=False),
self.tag_info["test_date"], '/', vehicle_id, '/', task_id,
dir_name
])
module_tag_data['aws_endpoint'] = self.end_point
saveTag(store_path, module_tag_data, file_name='data-tag.json')
self.generateLogDownloadFile(log_type,
module_tag_data["raw_data_link"],
output_path)
print("checkpoint44")
print(dir_path)
print(segment_list)
cut_rec_multiprocess.main(dir_path, segment_list)
print("checkpoint45")
if self.tag_info["test_type"] == 1:
self.case_tagging.tagMain(self.tag_info["global_tag"], dir_path,
case_tagging_list)
def checkRec(self, dir_path, slice=False):
# looks through the following folders: cache, config, cv22, logs, screen_cast, sensor_logs
# sensors_record, timing logger, versions
print("checkpoint25")
print(dir_path)
'''
:param dir_path: data path end with /
:param slice: define the checking data is raw data or segment data
:return: True of False
'''
video_result = 0
check_result = 0
try:
check_tag = loadTag(dir_path, 'data-tag.json')
if check_tag is not None and check_tag["test_car_id"] == "CN-001":
return True, []
if check_tag is not None and (check_tag["test_car_id"] == "CN-006"
or check_tag["test_car_id"]
== "CN-007"):
check_result += 1
except Exception as e:
print(getTime() + "\033[1;31m [ERROR]\033[0m not CN-001 data ")
self.false_check_reasion = []
false_reason = []
if not os.path.exists(dir_path):
return False, false_reason
for file_name in self.check_file_name_list:
print(file_name)
check_size = self.check_file_name_list[file_name]
print("checkpoint21")
print(dir_path)
reult_0_1 = self.judgeFileSizeAndExist(dir_path,
file_name,
check_size=check_size)
print(file_name)
print(reult_0_1)
print("checkpoint22")
check_result += reult_0_1
print(check_result)
if reult_0_1 == 0:
false_reason.append(file_name)
pattern = "port_*"
video_files = self.getMatchedFilePaths(
dir_path,
pattern,
formats=[".avi", ".h264", "mp4"],
recursive=True)
for video_name in video_files:
print("checkpoint23")
video_reult_0_1 = self.judgeFileSizeAndExist(dir_path='',
file_name=video_name,
check_size=1)
print("checkpoint24")
print(video_reult_0_1)
video_result += video_reult_0_1
if video_reult_0_1 == 0:
false_reason.append(file_name)
for dir_name in ['cache', 'config', 'logs', 'timing_logger']:
log_dir_path = os.path.join(dir_path, dir_name)
if not os.path.exists(log_dir_path):
false_reason.append("cache")
if check_result >= 4 and video_result > 0:
print("checkpoint27")
print(getTime() + "\033[1;32m [INFO]\033[0m Dir:", dir_path,
"is \033[1;32m correct\033[0m")
return True, false_reason
else:
print("checkpoint26")
print(check_result)
print(video_result)
print(getTime() + "\033[1;31m [ERROR]\033[0m Dir:", dir_path,
"is\033[1;31m wrong\033[0m")
return False, false_reason
def mainSegment(self, file_list, upload=False):
'''
:param dir_name: the dir to be process
:param upload: whether upload , default as False
:return: o -> right ; 1-> error
'''
print("checkpoint1000")
print(self.input_data_path)
dir_path = self.input_data_path
# dir_path = os.path.join(self.input_data_path, dir_name)
# check the dir whether right
check_result, self.false_reason = self.checkRec(dir_path)
print("calling checkRec")
for dir_name in file_list:
print("checkpoint1006")
print(dir_name)
print(dir_path)
# get the data-tag.json of checked dir
tag_data = self.deterDirProperty(dir_path)
if not check_result:
if 'test_type' in tag_data.keys(
) and tag_data['test_type'] == 3:
pass
if "issue_id" not in tag_data.keys(
) or tag_data["issue_id"] == []:
tag_data["issue_id"] = ["feature"]
if tag_data is not None and tag_data["issue_id"] != [
"repo_master"
] and tag_data["task_id"] != 30000:
pass
else:
self.falseDataArchive(dir_path, self.false_reason)
return
if "backup" in tag_data:
tag_data = tag_data["backup"][0]["data_tag"]
tag_data["test_date"] = tag_data["test_date"].replace('-', '_')
if tag_data is None:
return 1
if not "global_tag" in tag_data:
tag_data["global_tag"] = []
## generate evluation result with different module
if tag_data["test_type"] == 1 and self.config_["evaluation"]:
pool1 = multiprocessing.Pool(processes=12)
pool1.apply_async(
generate_evaluation_result.generateLocalizationEval,
args=(
dir_path,
self.config_,
))
pool1.close()
pool1.join()
# try:
# generate_evaluation_result.generateTprofileEval(dir_path,self.config_)
# except Exception as e:
# print getTime()+"\033[1;31m [ERROR]\033[0m tprofiling error "
if "issue_id" not in tag_data.keys() or tag_data["issue_id"] == []:
tag_data["issue_id"] = ["feature"]
saveTag(dir_path, tag_data)
self.tag_info = tag_data
self.tag_info["localization_result"] = self.getLocalizationResult(
dir_path.replace(" ", ""))
self.tag_info["control_result"] = {}
self.tag_info["ap_result"] = {}
self.tag_info["file_name"] = dir_name
self.tag_info["disk_id"] = self.getDiskId(dir_path)
self.addEvalTag(dir_path)
self.tag_info["origin_record_tag"] = self.deDuplication(
self.tag_info["origin_record_tag"])
saveTag(dir_path, self.tag_info)
self.post = True
segment, upload = self.judgeIfSegmentAndUpload(
self.tag_info, check_result)
try:
if self.tag_info["test_type"] == 2 and self.tag_info[
"issue_id"] == ["AUTODRIVE-6814"]:
overlap = self.reouteCheckFi(dir_path,
self.tag_info["test_car_id"])
self.tag_info['global_tag'].append(overlap)
try:
self.tag_info = self.getObjectNumber(
dir_path, self.tag_info)
except:
pass
if None in self.tag_info['global_tag']:
self.tag_info['global_tag'].remove(None)
saveTag(dir_path, self.tag_info)
print("checkpoint29")
print(dir_path)
print(dir_name)
if segment:
print("checkpoint30")
self.segmentPreprationCollection(dir_name)
print("checkpoint31")
print(dir_path)
print(dir_name)
print("checkpoint32")
self.mainUpload(dir_path, upload)
print("checkpoint33")
except Exception as e:
print(traceback.format_exc())
logger(1, str(e), LOG_FILE="upload_list/error.log")
print(getTime() +
"\033[1;31m [ERROR]\033[0m segment or upload error ")
return 1
return 0
