def renameCaseTaggingTag(case_output_path): "as the function name" case_finder = loadTag(case_output_path, 'case_finder.json') try: if case_finder == [] or case_finder is None: case_finder = {} saveTag(case_output_path, case_finder, 'case_finder.json') return if "obstacle_id" in case_finder[0] and case_finder[0][ "obstacle_id"] == [""]: case_finder[0]["obstacle_id"] = [] if "obstacle_id" in case_finder[0] and case_finder[0][ "obstacle_id"] == ["0"]: case_finder[0]["obstacle_id"] = [] if "ego_tags" in case_finder[0]: case_finder[0]["ego_tags"] = refineTag(case_finder[0]["ego_tags"]) if "obstacle_vehicle_tags" in case_finder[0]: case_finder[0]["obstacle_vehicle_tags"] = refineTag( case_finder[0]["obstacle_vehicle_tags"]) if "obstacle_vru_tags" in case_finder[0]: case_finder[0]["obstacle_vru_tags"] = refineTag( case_finder[0]["obstacle_vru_tags"]) saveTag(case_output_path, case_finder[0], 'case_finder.json') except Exception as e: print("222")
def main(dir_path): tprofile_case = [] result = {} tprofile_result = [] profile_label = {} #try: tprofile_case, result = moduleProfileParse(dir_path, 'module_tprofile_case.json', tprofile_case, result) tprofile_case, result = moduleProfileParse(dir_path, 'sensor_tprofile_case.json', tprofile_case, result) tprofile_case, result = moduleProfileParse( dir_path, 'cpu_usage/cpu_tprofile_case.json', tprofile_case, result) profile_label = labelAdd(dir_path, profile_label, 'tprofile_result.json') profile_label = labelAdd(dir_path, profile_label, 'cpu_usage/tprofile_result.json') saveTag(os.path.join(dir_path, 'screen_cast'), tprofile_case, 'tprofile_result.json') # except Exception as e: # print("tprofile_eval error") for module in result.keys(): if result[module] > 2: tprofile_result.append(module + '_tprofile_abnormal') return profile_label, tprofile_result, tprofile_case
def renameCaseTaggingTag(self, dir_path): case_output_path = os.path.join(dir_path, "screen_cast") case_finder = loadTag(case_output_path, 'case_finder.json') saved_ego_tags = [] if case_finder == [] or case_finder is None: case_finder = {} saveTag(case_output_path, case_finder, 'case_finder.json') return if "obstacle_id" in case_finder[0] and case_finder[0][ "obstacle_id"] == [""]: case_finder[0]["obstacle_id"] = [] if "obstacle_id" in case_finder[0] and case_finder[0][ "obstacle_id"] == ["0"]: case_finder[0]["obstacle_id"] = [] if "ego_tags" in case_finder[0]: case_finder[0]["ego_tags"] = self.refineTag( case_finder[0]["ego_tags"]) saved_ego_tags = case_finder[0]["ego_tags"] if "obstacle_vehicle_tags" in case_finder[0]: case_finder[0]["obstacle_vehicle_tags"] = self.refineTag( case_finder[0]["obstacle_vehicle_tags"]) if "obstacle_vru_tags" in case_finder[0]: case_finder[0]["obstacle_vru_tags"] = self.refineTag( case_finder[0]["obstacle_vru_tags"]) saveTag(case_output_path, case_finder[0], 'case_finder.json') return saved_ego_tags
def main(dir_path, bad_result, record_tag): ''' :param dir_path: :param label_list: :return: ''' #try: data_tag = loadTag(record_tag["input_dir"]) abnormal_ap_label = data_tag.get("origin_record_tag")[0] case_finder_json = loadTag(record_tag["input_dir"], tag_file_name='screen_cast/case_finder.json') if abnormal_ap_label == []: return 1 bad_case_list = defaultdict(lambda: {}) for bad_case_time in bad_result: bad_case_list[int(bad_case_time)] = bad_result[bad_case_time]["labels"] if len(bad_case_list) == 0: case_finder_json["ap_evaluation_result"] = [] case_finder_json["has_found"] = False else: case_finder_json["ap_evaluation_result"] = generateEvluationScore( bad_case_list, abnormal_ap_label) if len(case_finder_json["ap_evaluation_result"]) != 0: case_finder_json["ap_evaluation_result"].append("HasFound") print('\n-----', case_finder_json["ap_evaluation_result"], '\n') saveTag(record_tag["input_dir"], case_finder_json, '/screen_cast/case_finder.json')
def data_upload(self, dir_path_without, tag_info, slice=False): 'data_upload with aws' dir_name = os.path.split(dir_path_without)[1] vehicle_id = tag_info["test_car_id"].replace("-", "") data_month = tag_info["test_date"].rsplit("_", 1)[0] try: if tag_info["topic_name"][0] == "repo_master": feature = False else: feature = True except Exception as e: feature = False if feature: task_id = str(tag_info["task_id"]) + '/' else: task_id = '' if slice: upload_path = ''.join([dir_path_without, "_slice/ "]) dst_path = ''.join([ self.getDataCollectionDstLink(tag_info, data_month, slice=True), tag_info["test_date"], '/', vehicle_id, '/', task_id ]) self.sliceDataCheck(dir_path_without + '_slice/') else: upload_path = ''.join([dir_path_without, '/ ']) tag_path = ''.join([ self.getDataCollectionDstLink(tag_info, data_month, slice=False), tag_info["test_date"], '/', vehicle_id, '/', task_id, dir_name ]) dst_path = ''.join([tag_path, '/']) tag_info["data_link"] = tag_path tag_info["data_type"] = "raw" tag_info['aws_endpoint'] = end_point saveTag(dir_path_without, tag_info, file_name='/data-tag.json') arg2 = ''.join( ["s3 cp ", upload_path, dst_path, upload_recursive + end_point]) print(" ==== ", arg2) upload_result = AWS_DRIVER.main(arg2.split()) logger(1, "Uploaded dir:" + upload_path) self.TransferPost(tag_info) if upload_result == 0: print " ---- Dir:\033[1;32m", dir_name + "\033[0m" + dir_name, "\033[0m has\033[1;32m uploaded successfully!\033[0m---\n" else: print " ---- Dir:\033[1;32m", dir_name + "\033[0m" + dir_name, "\033[0m \033[1;32m upload failed!\033[0m---\n"
def addEgoTagToLabel(self, record_tag, ego_tags): case_tag = loadTag(record_tag["input_dir"], 'data-tag.json') if not "origin_record_tag" in case_tag.keys(): return if not "labels" in case_tag["origin_record_tag"][0]: case_tag["origin_record_tag"][0]["labels"] = [] if ego_tags is not None: for tag in ego_tags: case_tag["origin_record_tag"][0]["labels"].append(tag) saveTag(record_tag["input_dir"], case_tag, 'data-tag.json')
def processObstacleId(self, dir_path, record_tag): case_time = record_tag["start"] * 1000 result_tag = {"obstacle_id": [], "id_list": {}} dpc_file_path = ''.join([dir_path, 'dmppcl.bag']) tag_path = os.path.join(dir_path, 'screen_cast') id_list = parse_obstacle_tag.parseProtoTag(dpc_file_path, case_time) obstacle_id_list = self.filterObjectId(id_list) result_tag["obstacle_id"] = obstacle_id_list if id_list != []: for id in id_list: if id not in result_tag["id_list"].keys(): result_tag["id_list"][id] = 0 result_tag["id_list"][id] += 1 saveTag(tag_path, result_tag, 'obstacle.json')
def main(tprofile_thresh, dir_path): TPROFILE_RESULT = [] try: time_consuming_json = {} time_consuming_json = moduleParsing(dir_path, time_consuming_json, tprofile_thresh, TPROFILE_RESULT) time_consuming_json = sensorParsing(dir_path, time_consuming_json, tprofile_thresh, TPROFILE_RESULT) saveTag(dir_path + 'screen_cast/', time_consuming_json, 'module_time_consuing.json') except Exception as e: print("tprofile_eval error") print(TPROFILE_RESULT) return TPROFILE_RESULT
def mainSegment(self, dir_name, upload=False): ''' :param dir_name: the dir to be process :param upload: whether upload , default as False :return: o -> right ; 1-> error ''' dir_path = ''.join([self.input_data_path, '/', dir_name, '/']) dir_path_without = ''.join([self.input_data_path, '/', dir_name]) # check the dir whether right # check_result,false_reason = self.checkRec(dir_path) # if not check_result: # tag_data = self.deterDirProperty(dir_path) # # if tag_data is not None and tag_data["topic_name"]!= ["repo_master"] and tag_data["task_id"] !=30000: # pass # else: # self.falseDataArchive(dir_path_without,false_reason) # return # get the data-tag.json of checked dir tag_data = self.deterDirProperty(dir_path) if "backup" in tag_data.keys(): tag_data = tag_data["backup"][0]["data_tag"] tag_data["test_date"] = tag_data["test_date"].replace('-', '_') if tag_data is None: return 1 print(tag_data["test_date"]) if tag_data["topic_name"] == []: tag_data["topic_name"].append("feature") saveTag(dir_path, tag_data) self.tag_info = tag_data saveTag(dir_path, self.tag_info) self.generateCase(dir_path) try: self.post = False self.segmentPreprationCollection(dir_name) if upload: self.mainUpload(dir_path_without) except Exception as e: logger(1, str(e), LOG_FILE="../upload_list/error.log") logging.exception(e) return 1
def TransferPost(self, data_tag): "post the data tag to senseFT" curl = \ 'https://fieldtest.sensetime.com/task/daemon/update/tag' post_result = requests.post(curl, headers=self.headerdata, data=json.dumps(data_tag)) print "\n \033[1;32m [INFO]\033[0m ", post_result.text if post_result.text != u'{"result":"true"}': try: logger(2, "Uploaded dir:" + data_tag["data_link"], LOG_FILE="../upload_list/post_failed_file.log") saveTag('../upload_list/', data_tag, data_tag["file_name"] + '.json') except Exception as e: print("save upload failed tag failed")
def generateTag(self, dir_path): generate_tag = defaultdict(lambda: {}) dir_name = dir_path.split("/")[-2] generate_tag["assignee"] = "dingjinchao" generate_tag["city"] = "shanghai" generate_tag["data_type"] = "raw" generate_tag["route"] = "o2l" generate_tag["master"] = False generate_tag["file_name"] = dir_name generate_tag["frequency"] = 1 generate_tag["origin_record_tag"] = [] generate_tag["task_id"] = 30000 generate_tag["test_type"] = 1 generate_tag["test_date"] = dir_name[0:10] generate_tag["test_duration"] = 2610 generate_tag["topic_name"] = ["feature"] generate_tag["test_car_id"] = "CN-100" saveTag(dir_path, generate_tag) return generate_tag
def addWhetherInfo(self, global_tag, dir_path): tag_path = os.path.join(dir_path, "screen_cast") case_toss = loadTag(tag_path, "case_tag.json") if case_toss is None or global_tag == []: return whether_tag = 0 for tag in global_tag: if tag not in ["阳光直射", "阳光背射", "凌晨", "黄昏", "夜晚有路灯", "夜晚无路灯"]: continue if tag == "阳光直射" or tag == "阳光背射": whether_tag = 1 elif tag == "黄昏": whether_tag = 2 else: whether_tag = 3 if "Global_label" not in case_toss.keys(): case_toss["Global_label"] = {} case_toss["Global_label"]["day_time"] = whether_tag saveTag(tag_path, case_toss, 'case_tag.json')
def tlProcess(self, dir_path, record_tag): status = False case_toss_file = os.path.join(dir_path, 'screen_cast/case_tag.json') case_toss_tag = loadTag(case_toss_file, '') if case_toss_tag is None or case_toss_tag == {}: return record_tag, status case_toss_tag["Attributes"] = {} case_toss_tag["Object_check"] = {} case_toss_tag["Attributes"]["object_problem"] = 0 case_toss_tag["Attributes"]["traffic_light_problem"] = 0 case_toss_tag["Object_check"]["object_pos"] = 0 if record_tag['labels'] != []: for label in record_tag['labels']: if label in self.auto_module_["supply_label_contrast"]: labeled_module = self.auto_module_[ "supply_label_contrast"][label] case_toss_tag["Attributes"][labeled_module] = 1 if labeled_module == "traffic_light_problem": try: case_toss_tag["Pr_check"][ "true_tl_label"] = self.auto_module_[ "label_to_number"][label] except: print("tl toss error") saveTag(case_toss_file, case_toss_tag, '') self.autoModuleDsitrib(dir_path) if not os.path.isfile( os.path.join(dir_path, 'screen_cast/auto_module.json')): return record_tag, status auto_module_tag = loadTag( os.path.join(dir_path, 'screen_cast/auto_module.json'), '') if auto_module_tag["Module"] != []: status = True if auto_module_tag["Module"] == [ "C-TL" ] and "红绿灯场景变化" in record_tag["labels"]: auto_module_tag["Module"] = ["Road"] for module in auto_module_tag["Module"]: record_tag["modules"].append(module) record_tag["modules"] = list(set(record_tag["modules"])) return record_tag, status
def main(self, raw_dir_path, record_tag): dir_path = record_tag["input_dir"] tagging_module = record_tag["tagging_module"] tag_save_path = ''.join([dir_path, 'screen_cast/']) dpc_file_path = ''.join([dir_path, 'dmppcl.bag']) logs_path = ''.join([raw_dir_path, '/logs/']) bin_path = ''.join([dir_path, 'simulator_scenario/0/logger.bin']) if tagging_module == 3: self.case_finder(bin_path, tag_save_path, 3) elif tagging_module == 2: topic_names = ['/control/control_error'] tag_info = loadTag(dir_path) record_timestamp = int( tag_info["origin_record_tag"][0]["start"]) / 1000 print(record_timestamp) bag_data = self.getDataFromDpcbag(dpc_file_path, topic_names, record_timestamp) brake_timestamp, ego_label = self.parseBrakeTimestampFromBag( bag_data) brake_timestamp_bj = self.unixToBjclock(brake_timestamp) brake_timestamp_bj_1 = self.unixToBjclock(brake_timestamp - 1) nsec = math.modf(brake_timestamp)[0] * 1000 print 'brake case timestamp: ', brake_timestamp, brake_timestamp_bj decision_planning_log_file = self.getMatchedFilePaths( logs_path, 'ros_decision_planning_node.*') if decision_planning_log_file == []: obstacle_id = [0] id_list = [] else: log_file = self.filterLogFile(decision_planning_log_file) obstacle_id, id_list = self.parseDecisionPannningLog( log_file, brake_timestamp_bj, brake_timestamp_bj_1, int(nsec)) print 'obstacle id: ', obstacle_id tag = {} tag["obstacle_id"] = obstacle_id tag["id_list"] = {} if id_list != []: for id in id_list: if id not in tag["id_list"].keys(): tag["id_list"][id] = 0 tag["id_list"][id] += 1 saveTag(record_tag["input_dir"] + 'screen_cast/', tag, 'obstacle.json') id_json_path = ''.join( [record_tag["input_dir"], 'screen_cast/obstacle.json']) self.case_finder(bin_path, tag_save_path, 2, id_json_path, int(brake_timestamp), int(nsec), 5, 1) else: self.case_finder(bin_path, tag_save_path, 1, 0, int(record_tag["input_timestamp"]), int(000), 5, 1)
eval_tag_path = os.path.join(dir_name, 'logs/control_eval/') eval_tag = loadTag(eval_tag_path, 'control_eval_results.json') if eval_tag is None: return {} eval_tag = eval_tag.get("Tag") if len(eval_tag) < 1: return None for case_tag in eval_tag: if not "labels" in case_tag.keys(): case_tag["labels"] = [] case_tag["labels"].append(case_tag["tag_en"]) case_tag["data_type"] = "eval" return eval_tag def TransferPost(data_tag): "post the data tag to senseFT" curl = \ 'https://fieldtest.sensetime.com/production_line/upload' post_result = requests.post(curl, data=json.dumps(data_tag)) print("\n \033[1;32m [INFO]\033[0m ", post_result.text) # if post_result.text != u'{"result":"true"}': # logger(2, "Uploaded dir:" + data_tag["data_link"], LOG_FILE="upload_list/post_failed_file.log") if __name__ == "__main__": dir_path = '/media/sensetime/FieldTest1/data/2020_05_14_CN-016/2020_05_14_13_47_55_AutoCollect/' case_tag = getLocalEvalResult(dir_path) print(case_tag) saveTag(dir_path, case_tag, 'case-eval.json')
def predMain(self,dir_path, config_, record_tag): map_object_num2type = { 0: "veh", 1: "veh", 2: "ped", 3: "cyc", 4: "oth" } status = False if record_tag["modules"] != [] and record_tag["modules"] != ["FT"]: print("modules is not empty") return record_tag, status screen_path = os.path.join(dir_path, 'screen_cast') eval_result_path = os.path.join(dir_path, 'prediction_evaluation/result') case_finer_tag = loadTag(screen_path, 'case_finder.json') pp_obstacle_tag = loadTag(screen_path, 'obstacle.json') if case_finer_tag is None or pp_obstacle_tag is None or "wrong_timestamp" not in case_finer_tag.keys(): return record_tag, status generatePredictionEval(dir_path, config_,casetime=case_finer_tag["wrong_timestamp"],input_id="1111") eval_bad_case_tag = loadTag(eval_result_path, 'bad_cases.json') if eval_bad_case_tag is None: return record_tag, status pp_obstacle_id = pp_obstacle_tag["obstacle_id"] value = self.processBadCase(eval_bad_case_tag) saveTag(eval_result_path, value, 'value.json') for id in value: if not int(id) in pp_obstacle_id: continue type = map_object_num2type[value[id]["type"]] threshold = self.pred_eval_thresh[type] if "TurnWrong" in value or "LaneChangeWrong" in value[id]: record_tag["modules"].append("Prediction") record_tag["labels"].append("LaneChangeWrong") status = True print(record_tag,status) return record_tag, status times = 0 for label in threshold: if value[id].get(label) is None: continue if value[id].get(label) > threshold[label]: times += 1 if value[id].get("ConsistencyDisplacementError") is not None: if type == "cyc": if value[id].get("ConsistencyDisplacementError") > 2.4: record_tag["modules"].append("Prediction") status = True if value[id].get("ConsistencyDisplacementError") > 0.8 and times > 0: record_tag["modules"].append("Prediction") status = True elif type == "veh": if value[id].get("ConsistencyDisplacementError") > 5.0: record_tag["modules"].append("Prediction") status = True if value[id].get("ConsistencyDisplacementError") > 1.0 and times > 0: record_tag["modules"].append("Prediction") status = True elif type == "ped": if value[id].get("ConsistencyDisplacementError") > 1.5: record_tag["modules"].append("Prediction") status = True if value[id].get("ConsistencyDisplacementError") > 0.5 and times > 0: record_tag["modules"].append("Prediction") status = True if not status: continue for label in value[id].keys(): if label == "type": continue if not label in record_tag["labels"]: record_tag["labels"].append(label) if record_tag["modules"] == []: module = self.JudgeInvalid(eval_result_path, pp_obstacle_id) if module != "": record_tag["modules"].append(module) status = True print(record_tag,status) return record_tag,status
def padMain(dir_path, config_, record_tag): if record_tag["modules"] != [] and record_tag["modules"] != ["FT"]: return record_tag screen_path = os.path.join(dir_path, 'screen_cast') eval_result_path = os.path.join(dir_path, 'prediction_evaluation/result') case_finer_tag = loadTag(screen_path, 'case_finder.json') pp_obstacle_tag = loadTag(screen_path, 'obstacle.json') if case_finer_tag is None or pp_obstacle_tag is None: return record_tag generatePredictionEval(dir_path, config_, casetime=case_finer_tag["end_timestamp"]) eval_bad_case_tag = loadTag(eval_result_path, 'bad_cases.json') if eval_bad_case_tag is None: return record_tag pp_obstacle_id = [] eval_bad_case_id = {"id_list": []} intime_bad_case_list = [] if "wrong_timestamp" in case_finer_tag.keys(): case_timestamp = case_finer_tag["wrong_timestamp"] else: case_timestamp = getCaseTime(dir_path) case_timestamp = int(case_timestamp * 1000) for obj_id in pp_obstacle_tag["id_list"].keys(): pp_obstacle_id.append(obj_id) # id_list = getMatchedFilePaths(dir_path+'/screen_cast',"*",[".id"]) # if id_list == []: # print("id is none") # # return record_tag # real_id = os.path.basename(id_list[0]).split('.')[0] # print(real_id) for bad_case_time in eval_bad_case_tag.keys(): if int(bad_case_time) > case_timestamp - 500 and int( bad_case_time) < case_timestamp: intime_bad_case_list.append(eval_bad_case_tag[bad_case_time]) print pp_obstacle_id for intime_case in intime_bad_case_list: bb = [] # for label in intime_case["labels"]: # for aa in label.keys(): # bb.append(aa) # if intime_case["id"] == [] or bb == ['ConsistencyDisplacementError']: # continue for id in intime_case["id"]: # print(id, pp_obstacle_id) eval_bad_case_id["id_list"].append(id) if str(id) in pp_obstacle_id: #print id record_tag["modules"].append("Prediction") # for label in intime_case["labels"]: # for metrc in label.keys(): # if metrc in record_tag["labels"].keys(): # if record_tag["labels"][metrc] > label[metrc] or metrc =="LaneChangeWrong" or metrc =="TurnWrong": # #print(id,metrc,label[metrc]) # pass # else: # record_tag["labels"][metrc] = label[metrc] # #print(id,metrc,label[metrc]) # else: # record_tag["labels"][metrc] = label[metrc] #print(id, metrc, label[metrc]) record_tag["modules"] = list(set(record_tag["modules"])) #record_tag["labels"] = list(set(record_tag["labels"])) saveTag(screen_path, eval_bad_case_id, 'eval_bad_case_id.json') if record_tag["modules"] == []: module = JudgeInvalid(eval_result_path, pp_obstacle_id) if module != "": record_tag["modules"].append(module) return record_tag
def dataSegment(self, dir_name, segment_point): "data segment pipeline" segment_list = [] case_finder_list = [] case_tagging_list = [] prediction_tagging_list = [] for finder_tag in segment_point: dir_path = ''.join([self.input_data_path, '/', dir_name]) input_path = ''.join([dir_path, '/']) record_tag = loadTag('record_tag.json', '') if finder_tag.get("case_type") != "CaseType::junction": continue record_tag["tag_en"] = "traffic_light" record_tag["tag"] = "红绿灯" record_tag["start"] = int(finder_tag["start_timestamp"] * 1000) record_tag["start_format"] = self.unixToBjclock( finder_tag["case_timestamp"]).split(" ", -1)[1].replace('_', ':') record_tag["end"] = int(finder_tag["end_timestamp"] * 1000) record_tag["labels"] = ["traffic_light_dataset"] module_name = record_tag.get("tag_en") level_name = "Finder" log_type = 1 print("\033[1;32m ===>> \033[0m", level_name, "==>> ", module_name, "==== module_type:", log_type, '\n') if not "end" in record_tag.keys(): if level_name == "DPC": front_time = 15 behind_time = 25 else: front_time = 20 behind_time = 10 else: front_time = 2 behind_time = (record_tag["end"] - record_tag["start"]) // 1000 if behind_time < 6: behind_time = 10 time_point = record_tag["start"] * 1000 test_date = self.tag_info["test_date"] test_time = str(record_tag["start_format"].replace(":", "_")) segment_dir_name = ''.join([test_date, '_', test_time]) output_path = ''.join([ dir_path, '_slice/', level_name, '/', module_name, '/', segment_dir_name ]) segment_list.append({ "output_dir": output_path, "time_point": time_point, "front_duration": front_time, "behind_duration": behind_time, "log_type": log_type }) if not os.path.exists(output_path): os.makedirs(output_path) if self.tag_info["topic_name"][0] == "repo_master": task_id = '' else: task_id = str(self.tag_info["task_id"]) + '/' vehicle_id = self.tag_info["test_car_id"].replace("-", "") module_tag_data = copy.deepcopy(self.tag_info) data_month = self.tag_info["test_date"].rsplit("_", 1)[0] dst_path = ''.join([ self.getDataCollectionDstLink(self.tag_info, data_month, slice=True), self.tag_info["test_date"], '/', vehicle_id, '/', task_id ]) module_tag_data["data_link"] = ''.join([ dst_path, level_name, '/', module_name, '/', segment_dir_name ]) store_path = ''.join([ dir_path, '_slice/', level_name, '/', module_name, '/', segment_dir_name ]) module_tag_data["origin_record_tag"] = [record_tag] if "data_type" in record_tag.keys( ) and record_tag["data_type"] == "eval": module_tag_data["data_type"] = record_tag["data_type"] self.post = True else: module_tag_data["data_type"] = "segment" module_tag_data["file_name"] = segment_dir_name module_tag_data["raw_data_link"] = ''.join([ self.getDataCollectionDstLink(self.tag_info, data_month, slice=False), self.tag_info["test_date"], '/', vehicle_id, '/', task_id, dir_name ]) module_tag_data['aws_endpoint'] = end_point saveTag(store_path, module_tag_data, file_name='/data-tag.json') self.generateLogDownloadFile(log_type, module_tag_data["raw_data_link"], output_path) cut_rec_multiprocess.main(input_path, segment_list)