def renameCaseTaggingTag(case_output_path):
"as the function name"
case_finder = loadTag(case_output_path, 'case_finder.json')
try:
if case_finder == [] or case_finder is None:
case_finder = {}
saveTag(case_output_path, case_finder, 'case_finder.json')
return
if "obstacle_id" in case_finder[0] and case_finder[0][
"obstacle_id"] == [""]:
case_finder[0]["obstacle_id"] = []
if "obstacle_id" in case_finder[0] and case_finder[0][
"obstacle_id"] == ["0"]:
case_finder[0]["obstacle_id"] = []
if "ego_tags" in case_finder[0]:
case_finder[0]["ego_tags"] = refineTag(case_finder[0]["ego_tags"])
if "obstacle_vehicle_tags" in case_finder[0]:
case_finder[0]["obstacle_vehicle_tags"] = refineTag(
case_finder[0]["obstacle_vehicle_tags"])
if "obstacle_vru_tags" in case_finder[0]:
case_finder[0]["obstacle_vru_tags"] = refineTag(
case_finder[0]["obstacle_vru_tags"])
saveTag(case_output_path, case_finder[0], 'case_finder.json')
except Exception as e:
print("222")
def main(dir_path):
tprofile_case = []
result = {}
tprofile_result = []
profile_label = {}
#try:
tprofile_case, result = moduleProfileParse(dir_path,
'module_tprofile_case.json',
tprofile_case, result)
tprofile_case, result = moduleProfileParse(dir_path,
'sensor_tprofile_case.json',
tprofile_case, result)
tprofile_case, result = moduleProfileParse(
dir_path, 'cpu_usage/cpu_tprofile_case.json', tprofile_case, result)
profile_label = labelAdd(dir_path, profile_label, 'tprofile_result.json')
profile_label = labelAdd(dir_path, profile_label,
'cpu_usage/tprofile_result.json')
saveTag(os.path.join(dir_path, 'screen_cast'), tprofile_case,
'tprofile_result.json')
# except Exception as e:
# print("tprofile_eval error")
for module in result.keys():
if result[module] > 2:
tprofile_result.append(module + '_tprofile_abnormal')
return profile_label, tprofile_result, tprofile_case
def renameCaseTaggingTag(self, dir_path):
case_output_path = os.path.join(dir_path, "screen_cast")
case_finder = loadTag(case_output_path, 'case_finder.json')
saved_ego_tags = []
if case_finder == [] or case_finder is None:
case_finder = {}
saveTag(case_output_path, case_finder, 'case_finder.json')
return
if "obstacle_id" in case_finder[0] and case_finder[0][
"obstacle_id"] == [""]:
case_finder[0]["obstacle_id"] = []
if "obstacle_id" in case_finder[0] and case_finder[0][
"obstacle_id"] == ["0"]:
case_finder[0]["obstacle_id"] = []
if "ego_tags" in case_finder[0]:
case_finder[0]["ego_tags"] = self.refineTag(
case_finder[0]["ego_tags"])
saved_ego_tags = case_finder[0]["ego_tags"]
if "obstacle_vehicle_tags" in case_finder[0]:
case_finder[0]["obstacle_vehicle_tags"] = self.refineTag(
case_finder[0]["obstacle_vehicle_tags"])
if "obstacle_vru_tags" in case_finder[0]:
case_finder[0]["obstacle_vru_tags"] = self.refineTag(
case_finder[0]["obstacle_vru_tags"])
saveTag(case_output_path, case_finder[0], 'case_finder.json')
return saved_ego_tags
def main(dir_path, bad_result, record_tag):
'''
:param dir_path:
:param label_list:
:return:
'''
#try:
data_tag = loadTag(record_tag["input_dir"])
abnormal_ap_label = data_tag.get("origin_record_tag")[0]
case_finder_json = loadTag(record_tag["input_dir"],
tag_file_name='screen_cast/case_finder.json')
if abnormal_ap_label == []:
return 1
bad_case_list = defaultdict(lambda: {})
for bad_case_time in bad_result:
bad_case_list[int(bad_case_time)] = bad_result[bad_case_time]["labels"]
if len(bad_case_list) == 0:
case_finder_json["ap_evaluation_result"] = []
case_finder_json["has_found"] = False
else:
case_finder_json["ap_evaluation_result"] = generateEvluationScore(
bad_case_list, abnormal_ap_label)
if len(case_finder_json["ap_evaluation_result"]) != 0:
case_finder_json["ap_evaluation_result"].append("HasFound")
print('\n-----', case_finder_json["ap_evaluation_result"], '\n')
saveTag(record_tag["input_dir"], case_finder_json,
'/screen_cast/case_finder.json')
def data_upload(self, dir_path_without, tag_info, slice=False):
'data_upload with aws'
dir_name = os.path.split(dir_path_without)[1]
vehicle_id = tag_info["test_car_id"].replace("-", "")
data_month = tag_info["test_date"].rsplit("_", 1)[0]
try:
if tag_info["topic_name"][0] == "repo_master":
feature = False
else:
feature = True
except Exception as e:
feature = False
if feature:
task_id = str(tag_info["task_id"]) + '/'
else:
task_id = ''
if slice:
upload_path = ''.join([dir_path_without, "_slice/ "])
dst_path = ''.join([
self.getDataCollectionDstLink(tag_info, data_month,
slice=True),
tag_info["test_date"], '/', vehicle_id, '/', task_id
])
self.sliceDataCheck(dir_path_without + '_slice/')
else:
upload_path = ''.join([dir_path_without, '/ '])
tag_path = ''.join([
self.getDataCollectionDstLink(tag_info,
data_month,
slice=False),
tag_info["test_date"], '/', vehicle_id, '/', task_id, dir_name
])
dst_path = ''.join([tag_path, '/'])
tag_info["data_link"] = tag_path
tag_info["data_type"] = "raw"
tag_info['aws_endpoint'] = end_point
saveTag(dir_path_without, tag_info, file_name='/data-tag.json')
arg2 = ''.join(
["s3 cp ", upload_path, dst_path, upload_recursive + end_point])
print(" ==== ", arg2)
upload_result = AWS_DRIVER.main(arg2.split())
logger(1, "Uploaded dir:" + upload_path)
self.TransferPost(tag_info)
if upload_result == 0:
print " ---- Dir:\033[1;32m", dir_name + "\033[0m" + dir_name, "\033[0m has\033[1;32m uploaded successfully!\033[0m---\n"
else:
print " ---- Dir:\033[1;32m", dir_name + "\033[0m" + dir_name, "\033[0m \033[1;32m upload failed!\033[0m---\n"
def addEgoTagToLabel(self, record_tag, ego_tags):
case_tag = loadTag(record_tag["input_dir"], 'data-tag.json')
if not "origin_record_tag" in case_tag.keys():
return
if not "labels" in case_tag["origin_record_tag"][0]:
case_tag["origin_record_tag"][0]["labels"] = []
if ego_tags is not None:
for tag in ego_tags:
case_tag["origin_record_tag"][0]["labels"].append(tag)
saveTag(record_tag["input_dir"], case_tag, 'data-tag.json')
def processObstacleId(self, dir_path, record_tag):
case_time = record_tag["start"] * 1000
result_tag = {"obstacle_id": [], "id_list": {}}
dpc_file_path = ''.join([dir_path, 'dmppcl.bag'])
tag_path = os.path.join(dir_path, 'screen_cast')
id_list = parse_obstacle_tag.parseProtoTag(dpc_file_path, case_time)
obstacle_id_list = self.filterObjectId(id_list)
result_tag["obstacle_id"] = obstacle_id_list
if id_list != []:
for id in id_list:
if id not in result_tag["id_list"].keys():
result_tag["id_list"][id] = 0
result_tag["id_list"][id] += 1
saveTag(tag_path, result_tag, 'obstacle.json')
def main(tprofile_thresh, dir_path):
TPROFILE_RESULT = []
try:
time_consuming_json = {}
time_consuming_json = moduleParsing(dir_path, time_consuming_json,
tprofile_thresh, TPROFILE_RESULT)
time_consuming_json = sensorParsing(dir_path, time_consuming_json,
tprofile_thresh, TPROFILE_RESULT)
saveTag(dir_path + 'screen_cast/', time_consuming_json,
'module_time_consuing.json')
except Exception as e:
print("tprofile_eval error")
print(TPROFILE_RESULT)
return TPROFILE_RESULT
def mainSegment(self, dir_name, upload=False):
'''
:param dir_name: the dir to be process
:param upload: whether upload , default as False
:return: o -> right ; 1-> error
'''
dir_path = ''.join([self.input_data_path, '/', dir_name, '/'])
dir_path_without = ''.join([self.input_data_path, '/', dir_name])
# check the dir whether right
# check_result,false_reason = self.checkRec(dir_path)
# if not check_result:
# tag_data = self.deterDirProperty(dir_path)
#
# if tag_data is not None and tag_data["topic_name"]!= ["repo_master"] and tag_data["task_id"] !=30000:
# pass
# else:
# self.falseDataArchive(dir_path_without,false_reason)
# return
# get the data-tag.json of checked dir
tag_data = self.deterDirProperty(dir_path)
if "backup" in tag_data.keys():
tag_data = tag_data["backup"][0]["data_tag"]
tag_data["test_date"] = tag_data["test_date"].replace('-', '_')
if tag_data is None:
return 1
print(tag_data["test_date"])
if tag_data["topic_name"] == []:
tag_data["topic_name"].append("feature")
saveTag(dir_path, tag_data)
self.tag_info = tag_data
saveTag(dir_path, self.tag_info)
self.generateCase(dir_path)
try:
self.post = False
self.segmentPreprationCollection(dir_name)
if upload:
self.mainUpload(dir_path_without)
except Exception as e:
logger(1, str(e), LOG_FILE="../upload_list/error.log")
logging.exception(e)
return 1
def TransferPost(self, data_tag):
"post the data tag to senseFT"
curl = \
'https://fieldtest.sensetime.com/task/daemon/update/tag'
post_result = requests.post(curl,
headers=self.headerdata,
data=json.dumps(data_tag))
print "\n \033[1;32m [INFO]\033[0m ", post_result.text
if post_result.text != u'{"result":"true"}':
try:
logger(2,
"Uploaded dir:" + data_tag["data_link"],
LOG_FILE="../upload_list/post_failed_file.log")
saveTag('../upload_list/', data_tag,
data_tag["file_name"] + '.json')
except Exception as e:
print("save upload failed tag failed")
def generateTag(self, dir_path):
generate_tag = defaultdict(lambda: {})
dir_name = dir_path.split("/")[-2]
generate_tag["assignee"] = "dingjinchao"
generate_tag["city"] = "shanghai"
generate_tag["data_type"] = "raw"
generate_tag["route"] = "o2l"
generate_tag["master"] = False
generate_tag["file_name"] = dir_name
generate_tag["frequency"] = 1
generate_tag["origin_record_tag"] = []
generate_tag["task_id"] = 30000
generate_tag["test_type"] = 1
generate_tag["test_date"] = dir_name[0:10]
generate_tag["test_duration"] = 2610
generate_tag["topic_name"] = ["feature"]
generate_tag["test_car_id"] = "CN-100"
saveTag(dir_path, generate_tag)
return generate_tag
def addWhetherInfo(self, global_tag, dir_path):
tag_path = os.path.join(dir_path, "screen_cast")
case_toss = loadTag(tag_path, "case_tag.json")
if case_toss is None or global_tag == []:
return
whether_tag = 0
for tag in global_tag:
if tag not in ["阳光直射", "阳光背射", "凌晨", "黄昏", "夜晚有路灯", "夜晚无路灯"]:
continue
if tag == "阳光直射" or tag == "阳光背射":
whether_tag = 1
elif tag == "黄昏":
whether_tag = 2
else:
whether_tag = 3
if "Global_label" not in case_toss.keys():
case_toss["Global_label"] = {}
case_toss["Global_label"]["day_time"] = whether_tag
saveTag(tag_path, case_toss, 'case_tag.json')
def tlProcess(self, dir_path, record_tag):
status = False
case_toss_file = os.path.join(dir_path, 'screen_cast/case_tag.json')
case_toss_tag = loadTag(case_toss_file, '')
if case_toss_tag is None or case_toss_tag == {}:
return record_tag, status
case_toss_tag["Attributes"] = {}
case_toss_tag["Object_check"] = {}
case_toss_tag["Attributes"]["object_problem"] = 0
case_toss_tag["Attributes"]["traffic_light_problem"] = 0
case_toss_tag["Object_check"]["object_pos"] = 0
if record_tag['labels'] != []:
for label in record_tag['labels']:
if label in self.auto_module_["supply_label_contrast"]:
labeled_module = self.auto_module_[
"supply_label_contrast"][label]
case_toss_tag["Attributes"][labeled_module] = 1
if labeled_module == "traffic_light_problem":
try:
case_toss_tag["Pr_check"][
"true_tl_label"] = self.auto_module_[
"label_to_number"][label]
except:
print("tl toss error")
saveTag(case_toss_file, case_toss_tag, '')
self.autoModuleDsitrib(dir_path)
if not os.path.isfile(
os.path.join(dir_path, 'screen_cast/auto_module.json')):
return record_tag, status
auto_module_tag = loadTag(
os.path.join(dir_path, 'screen_cast/auto_module.json'), '')
if auto_module_tag["Module"] != []:
status = True
if auto_module_tag["Module"] == [
"C-TL"
] and "红绿灯场景变化" in record_tag["labels"]:
auto_module_tag["Module"] = ["Road"]
for module in auto_module_tag["Module"]:
record_tag["modules"].append(module)
record_tag["modules"] = list(set(record_tag["modules"]))
return record_tag, status
def main(self, raw_dir_path, record_tag):
dir_path = record_tag["input_dir"]
tagging_module = record_tag["tagging_module"]
tag_save_path = ''.join([dir_path, 'screen_cast/'])
dpc_file_path = ''.join([dir_path, 'dmppcl.bag'])
logs_path = ''.join([raw_dir_path, '/logs/'])
bin_path = ''.join([dir_path, 'simulator_scenario/0/logger.bin'])
if tagging_module == 3:
self.case_finder(bin_path, tag_save_path, 3)
elif tagging_module == 2:
topic_names = ['/control/control_error']
tag_info = loadTag(dir_path)
record_timestamp = int(
tag_info["origin_record_tag"][0]["start"]) / 1000
print(record_timestamp)
bag_data = self.getDataFromDpcbag(dpc_file_path, topic_names,
record_timestamp)
brake_timestamp, ego_label = self.parseBrakeTimestampFromBag(
bag_data)
brake_timestamp_bj = self.unixToBjclock(brake_timestamp)
brake_timestamp_bj_1 = self.unixToBjclock(brake_timestamp - 1)
nsec = math.modf(brake_timestamp)[0] * 1000
print 'brake case timestamp: ', brake_timestamp, brake_timestamp_bj
decision_planning_log_file = self.getMatchedFilePaths(
logs_path, 'ros_decision_planning_node.*')
if decision_planning_log_file == []:
obstacle_id = [0]
id_list = []
else:
log_file = self.filterLogFile(decision_planning_log_file)
obstacle_id, id_list = self.parseDecisionPannningLog(
log_file, brake_timestamp_bj, brake_timestamp_bj_1,
int(nsec))
print 'obstacle id: ', obstacle_id
tag = {}
tag["obstacle_id"] = obstacle_id
tag["id_list"] = {}
if id_list != []:
for id in id_list:
if id not in tag["id_list"].keys():
tag["id_list"][id] = 0
tag["id_list"][id] += 1
saveTag(record_tag["input_dir"] + 'screen_cast/', tag,
'obstacle.json')
id_json_path = ''.join(
[record_tag["input_dir"], 'screen_cast/obstacle.json'])
self.case_finder(bin_path, tag_save_path, 2, id_json_path,
int(brake_timestamp), int(nsec), 5, 1)
else:
self.case_finder(bin_path, tag_save_path, 1, 0,
int(record_tag["input_timestamp"]), int(000), 5,
1)
eval_tag_path = os.path.join(dir_name, 'logs/control_eval/')
eval_tag = loadTag(eval_tag_path, 'control_eval_results.json')
if eval_tag is None:
return {}
eval_tag = eval_tag.get("Tag")
if len(eval_tag) < 1:
return None
for case_tag in eval_tag:
if not "labels" in case_tag.keys():
case_tag["labels"] = []
case_tag["labels"].append(case_tag["tag_en"])
case_tag["data_type"] = "eval"
return eval_tag
def TransferPost(data_tag):
"post the data tag to senseFT"
curl = \
'https://fieldtest.sensetime.com/production_line/upload'
post_result = requests.post(curl, data=json.dumps(data_tag))
print("\n \033[1;32m [INFO]\033[0m ", post_result.text)
# if post_result.text != u'{"result":"true"}':
# logger(2, "Uploaded dir:" + data_tag["data_link"], LOG_FILE="upload_list/post_failed_file.log")
if __name__ == "__main__":
dir_path = '/media/sensetime/FieldTest1/data/2020_05_14_CN-016/2020_05_14_13_47_55_AutoCollect/'
case_tag = getLocalEvalResult(dir_path)
print(case_tag)
saveTag(dir_path, case_tag, 'case-eval.json')
def predMain(self,dir_path, config_, record_tag):
map_object_num2type = {
0: "veh",
1: "veh",
2: "ped",
3: "cyc",
4: "oth"
}
status = False
if record_tag["modules"] != [] and record_tag["modules"] != ["FT"]:
print("modules is not empty")
return record_tag, status
screen_path = os.path.join(dir_path, 'screen_cast')
eval_result_path = os.path.join(dir_path, 'prediction_evaluation/result')
case_finer_tag = loadTag(screen_path, 'case_finder.json')
pp_obstacle_tag = loadTag(screen_path, 'obstacle.json')
if case_finer_tag is None or pp_obstacle_tag is None or "wrong_timestamp" not in case_finer_tag.keys():
return record_tag, status
generatePredictionEval(dir_path, config_,casetime=case_finer_tag["wrong_timestamp"],input_id="1111")
eval_bad_case_tag = loadTag(eval_result_path, 'bad_cases.json')
if eval_bad_case_tag is None:
return record_tag, status
pp_obstacle_id = pp_obstacle_tag["obstacle_id"]
value = self.processBadCase(eval_bad_case_tag)
saveTag(eval_result_path, value, 'value.json')
for id in value:
if not int(id) in pp_obstacle_id:
continue
type = map_object_num2type[value[id]["type"]]
threshold = self.pred_eval_thresh[type]
if "TurnWrong" in value or "LaneChangeWrong" in value[id]:
record_tag["modules"].append("Prediction")
record_tag["labels"].append("LaneChangeWrong")
status = True
print(record_tag,status)
return record_tag, status
times = 0
for label in threshold:
if value[id].get(label) is None:
continue
if value[id].get(label) > threshold[label]:
times += 1
if value[id].get("ConsistencyDisplacementError") is not None:
if type == "cyc":
if value[id].get("ConsistencyDisplacementError") > 2.4:
record_tag["modules"].append("Prediction")
status = True
if value[id].get("ConsistencyDisplacementError") > 0.8 and times > 0:
record_tag["modules"].append("Prediction")
status = True
elif type == "veh":
if value[id].get("ConsistencyDisplacementError") > 5.0:
record_tag["modules"].append("Prediction")
status = True
if value[id].get("ConsistencyDisplacementError") > 1.0 and times > 0:
record_tag["modules"].append("Prediction")
status = True
elif type == "ped":
if value[id].get("ConsistencyDisplacementError") > 1.5:
record_tag["modules"].append("Prediction")
status = True
if value[id].get("ConsistencyDisplacementError") > 0.5 and times > 0:
record_tag["modules"].append("Prediction")
status = True
if not status:
continue
for label in value[id].keys():
if label == "type":
continue
if not label in record_tag["labels"]:
record_tag["labels"].append(label)
if record_tag["modules"] == []:
module = self.JudgeInvalid(eval_result_path, pp_obstacle_id)
if module != "":
record_tag["modules"].append(module)
status = True
print(record_tag,status)
return record_tag,status
def padMain(dir_path, config_, record_tag):
if record_tag["modules"] != [] and record_tag["modules"] != ["FT"]:
return record_tag
screen_path = os.path.join(dir_path, 'screen_cast')
eval_result_path = os.path.join(dir_path, 'prediction_evaluation/result')
case_finer_tag = loadTag(screen_path, 'case_finder.json')
pp_obstacle_tag = loadTag(screen_path, 'obstacle.json')
if case_finer_tag is None or pp_obstacle_tag is None:
return record_tag
generatePredictionEval(dir_path,
config_,
casetime=case_finer_tag["end_timestamp"])
eval_bad_case_tag = loadTag(eval_result_path, 'bad_cases.json')
if eval_bad_case_tag is None:
return record_tag
pp_obstacle_id = []
eval_bad_case_id = {"id_list": []}
intime_bad_case_list = []
if "wrong_timestamp" in case_finer_tag.keys():
case_timestamp = case_finer_tag["wrong_timestamp"]
else:
case_timestamp = getCaseTime(dir_path)
case_timestamp = int(case_timestamp * 1000)
for obj_id in pp_obstacle_tag["id_list"].keys():
pp_obstacle_id.append(obj_id)
# id_list = getMatchedFilePaths(dir_path+'/screen_cast',"*",[".id"])
# if id_list == []:
# print("id is none")
# # return record_tag
# real_id = os.path.basename(id_list[0]).split('.')[0]
# print(real_id)
for bad_case_time in eval_bad_case_tag.keys():
if int(bad_case_time) > case_timestamp - 500 and int(
bad_case_time) < case_timestamp:
intime_bad_case_list.append(eval_bad_case_tag[bad_case_time])
print pp_obstacle_id
for intime_case in intime_bad_case_list:
bb = []
# for label in intime_case["labels"]:
# for aa in label.keys():
# bb.append(aa)
# if intime_case["id"] == [] or bb == ['ConsistencyDisplacementError']:
# continue
for id in intime_case["id"]:
# print(id, pp_obstacle_id)
eval_bad_case_id["id_list"].append(id)
if str(id) in pp_obstacle_id:
#print id
record_tag["modules"].append("Prediction")
# for label in intime_case["labels"]:
# for metrc in label.keys():
# if metrc in record_tag["labels"].keys():
# if record_tag["labels"][metrc] > label[metrc] or metrc =="LaneChangeWrong" or metrc =="TurnWrong":
# #print(id,metrc,label[metrc])
# pass
# else:
# record_tag["labels"][metrc] = label[metrc]
# #print(id,metrc,label[metrc])
# else:
# record_tag["labels"][metrc] = label[metrc]
#print(id, metrc, label[metrc])
record_tag["modules"] = list(set(record_tag["modules"]))
#record_tag["labels"] = list(set(record_tag["labels"]))
saveTag(screen_path, eval_bad_case_id, 'eval_bad_case_id.json')
if record_tag["modules"] == []:
module = JudgeInvalid(eval_result_path, pp_obstacle_id)
if module != "":
record_tag["modules"].append(module)
return record_tag
def dataSegment(self, dir_name, segment_point):
"data segment pipeline"
segment_list = []
case_finder_list = []
case_tagging_list = []
prediction_tagging_list = []
for finder_tag in segment_point:
dir_path = ''.join([self.input_data_path, '/', dir_name])
input_path = ''.join([dir_path, '/'])
record_tag = loadTag('record_tag.json', '')
if finder_tag.get("case_type") != "CaseType::junction":
continue
record_tag["tag_en"] = "traffic_light"
record_tag["tag"] = "红绿灯"
record_tag["start"] = int(finder_tag["start_timestamp"] * 1000)
record_tag["start_format"] = self.unixToBjclock(
finder_tag["case_timestamp"]).split(" ",
-1)[1].replace('_', ':')
record_tag["end"] = int(finder_tag["end_timestamp"] * 1000)
record_tag["labels"] = ["traffic_light_dataset"]
module_name = record_tag.get("tag_en")
level_name = "Finder"
log_type = 1
print("\033[1;32m ===>> \033[0m", level_name, "==>> ",
module_name, "==== module_type:", log_type, '\n')
if not "end" in record_tag.keys():
if level_name == "DPC":
front_time = 15
behind_time = 25
else:
front_time = 20
behind_time = 10
else:
front_time = 2
behind_time = (record_tag["end"] - record_tag["start"]) // 1000
if behind_time < 6:
behind_time = 10
time_point = record_tag["start"] * 1000
test_date = self.tag_info["test_date"]
test_time = str(record_tag["start_format"].replace(":", "_"))
segment_dir_name = ''.join([test_date, '_', test_time])
output_path = ''.join([
dir_path, '_slice/', level_name, '/', module_name, '/',
segment_dir_name
])
segment_list.append({
"output_dir": output_path,
"time_point": time_point,
"front_duration": front_time,
"behind_duration": behind_time,
"log_type": log_type
})
if not os.path.exists(output_path):
os.makedirs(output_path)
if self.tag_info["topic_name"][0] == "repo_master":
task_id = ''
else:
task_id = str(self.tag_info["task_id"]) + '/'
vehicle_id = self.tag_info["test_car_id"].replace("-", "")
module_tag_data = copy.deepcopy(self.tag_info)
data_month = self.tag_info["test_date"].rsplit("_", 1)[0]
dst_path = ''.join([
self.getDataCollectionDstLink(self.tag_info,
data_month,
slice=True),
self.tag_info["test_date"], '/', vehicle_id, '/', task_id
])
module_tag_data["data_link"] = ''.join([
dst_path, level_name, '/', module_name, '/', segment_dir_name
])
store_path = ''.join([
dir_path, '_slice/', level_name, '/', module_name, '/',
segment_dir_name
])
module_tag_data["origin_record_tag"] = [record_tag]
if "data_type" in record_tag.keys(
) and record_tag["data_type"] == "eval":
module_tag_data["data_type"] = record_tag["data_type"]
self.post = True
else:
module_tag_data["data_type"] = "segment"
module_tag_data["file_name"] = segment_dir_name
module_tag_data["raw_data_link"] = ''.join([
self.getDataCollectionDstLink(self.tag_info,
data_month,
slice=False),
self.tag_info["test_date"], '/', vehicle_id, '/', task_id,
dir_name
])
module_tag_data['aws_endpoint'] = end_point
saveTag(store_path, module_tag_data, file_name='/data-tag.json')
self.generateLogDownloadFile(log_type,
module_tag_data["raw_data_link"],
output_path)
cut_rec_multiprocess.main(input_path, segment_list)