def sliceDataCheck(self, dir_path): "segmented data check and post" data_tag_paths = self.getMatchedFilePaths(dir_path, pattern="data-ta*", formats=[".json"], recursive=True) for slice_data_tag_path in data_tag_paths: slice_data_path = os.path.split(slice_data_tag_path)[0] check_result, false_reason = self.checkRec(slice_data_path, slice=True) slice_data_tag = loadTag(slice_data_tag_path, '') record_tag = slice_data_tag["origin_record_tag"][0] slice_data_tag["origin_record_tag"] = \ [self.case_toss.mainToss(self.local_eval_tag,os.path.split(slice_data_tag_path)[0], record_tag,slice_data_tag)] saveTag(slice_data_tag_path, slice_data_tag, '') if self.tag_info['master'] == False: pass else: if not check_result: saveTag('upload_list/false_segment', slice_data_tag, slice_data_tag["file_name"] + '.json') continue if self.tag_info['test_type'] == 3 and not os.path.exists( os.path.split(slice_data_tag_path)[0] + '/cv22'): continue self.TransferPost(slice_data_tag)
def generateTag(self, dir_path): generated_tag = loadTag("config/default_data-tag.json", '') dir_name = os.path.basename(dir_path) generated_tag["file_name"] = dir_name if len(dir_name) > 10: generated_tag["test_date"] = dir_name[0:10] saveTag(dir_path, generated_tag) return generated_tag
def main(): statics_result ={"test_duration":0, "test_mileage":0, 'route':{ }, "record_tag":{ }} vnumber = { "fewer_objects":0, "normal_objects":0, "many_objects":0 } dir_path = "/media/sensetime/FieldTest2/aaa/" tag_list = getAllDataDir(dir_path) print(len(tag_list)) i = 0 result_list = [] for tag_path in tag_list: # tag_data = loadTag(tag_path,'') # if tag_data is None: # continue tag_path= os.path.join(dir_path,tag_path) bin_path = os.path.join(tag_path,'simulator_scenario/simulator_scenario_log.bin') try: case_finder(bin_path, tag_path, 0) except: continue object_number = getObjectNumber(tag_path) if object_number !=0 and object_number != []: result_list+=object_number # if "backup" in tag_data.keys(): # tag_data = tag_data["backup"][0]["data_tag"] # tag_data["test_date"] = tag_data["test_date"].replace('-', '_') # # if "test_duration" in tag_data.keys(): # statics_result["test_duration"]+=tag_data["test_duration"] # if "test_mileage" in tag_data.keys() and tag_data["test_mileage"]>0 and tag_data["test_mileage"]<200: # statics_result["test_mileage"]+=tag_data["test_mileage"] # if "origin_record_tag" in tag_data.keys() and tag_data["origin_record_tag"] != []: # for record_tag in tag_data["origin_record_tag"]: # if not record_tag["tag_en"] in statics_result["record_tag"].keys(): # statics_result["record_tag"][record_tag["tag_en"]] = 0 # statics_result["record_tag"][record_tag["tag_en"]] +=1 # if 'route' in tag_data.keys(): # if tag_data['route'] not in statics_result["route"]: # statics_result["route"][tag_data['route']] =0 # statics_result["route"][tag_data['route']] += 1 # i+=1 # print(i) result_list = sorted(result_list,reverse = True) print(result_list) statics_result["object_number"] = result_list #statics_result["object_size"] = sum(result_list)/len(result_list) saveTag(dir_path,statics_result,'result.json')
def main(): statics_result = {} vnumber = {"fewer_objects": 0, "normal_objects": 0, "many_objects": 0} dir_path = "/media/sensetime/FieldTest1/data/04_10_CN-013_ARH/" tag_list = getMatchedFilePaths(dir_path, 'cpu_resu*', '.json', True) print(len(tag_list)) i = 0 result_list = [] for tag_path in tag_list: i += 1 tag_data = loadTag(tag_path, '') if tag_data is None: continue if "backup" in tag_data.keys(): tag_data = tag_data["backup"][0]["data_tag"] tag_data["test_date"] = tag_data["test_date"].replace('-', '_') print(tag_data) for core in tag_data: if core not in statics_result: statics_result[core] = 0 statics_result[core] += tag_data[core] # for core in tag_data["sensor_eval"]: # if core not in statics_result: # statics_result[core]={} # statics_result[core]["stddev_frame_rate"] = 0 # statics_result[core]["avg_frame_rate"] = 0 # statics_result[core]["stddev_frame_rate"] += tag_data["sensor_eval"][core]["stddev_frame_rate"] # statics_result[core]["avg_frame_rate"] += tag_data["sensor_eval"][core]["avg_frame_rate"] # for core in tag_data["module_eval"]: # if core not in statics_result: # statics_result[core] = {} # statics_result[core]["stddev_frame_rate"] = 0 # statics_result[core]["avg_frame_rate"] = 0 # statics_result[core]["stddev_frame_rate"] += tag_data["module_eval"][core]["stddev_frame_rate"] # statics_result[core]["avg_frame_rate"] += tag_data["module_eval"][core]["avg_frame_rate"] print(i) for core in statics_result: # statics_result[core]["stddev_frame_rate"] = statics_result[core]["stddev_frame_rate"]/i # statics_result[core]["avg_frame_rate"] = statics_result[core]["avg_frame_rate"] / i statics_result[core] = statics_result[core] / i saveTag(dir_path, statics_result, 'result.json') saveTag(dir_path, vnumber, 'statics_result_12.json')
def TransferPost(data_tag): curl = \ 'https://fieldtest.sensetime.com/task/daemon/update/tag' post_result = requests.post(curl, headers=headerdata, data=json.dumps(data_tag)) print "\n \033[1;32m [INFO]\033[0m ", post_result.text print(data_tag["file_name"]) if post_result.text != u'{"result":"true"}': try: saveTag('../upload_list/', data_tag, data_tag["file_name"] + '.json') except Exception as e: print("save upload failed tag failed") return False else: return True
def TransferPost(self, data_tag): "post the data tag to senseFT" curl = \ 'https://fieldtest.sensetime.com/task/daemon/update/tag' post_result = requests.post(curl, headers=self.headerdata, data=json.dumps(data_tag)) print(getTime() + "\033[1;32m [INFO]\033[0m ", post_result.text, '\n') if post_result.text != u'{"result":"true"}': try: logger(2, "Uploaded dir:" + data_tag["data_link"], LOG_FILE="upload_list/post_failed_file.log") saveTag('upload_list/', data_tag, data_tag["file_name"] + '.json') except Exception as e: print(getTime() + "\033[1;31m [ERROR]\033[0m save post-failed tag error ")
def getFalseList(file_path, check_result=False): ''' :param input_file_list: :param check: :return: ''' # if not judge_file_data(file_path): # return data_report_tag = loadTag('../data_report/', formatted_today + '.json') date = str(formatted_today) total_file_size = 0 print('===', date) if not date in data_report_tag.keys(): data_report_tag[date] = defaultdict(lambda: {}) file_name = file_path.split('/', -1)[-1] file_size = getFileSize(file_path) if not 'false' in data_report_tag[date]: data_report_tag[date]['false'] = defaultdict(lambda: {}) if not file_name in data_report_tag[date]['false']: data_report_tag[date]['false'][file_name] = defaultdict(lambda: {}) data_report_tag[date]['false'][file_name]["upload_date"] = date data_report_tag[date]['false'][file_name]["data_type"] = "raw" data_report_tag[date]['false'][file_name]["data_name"] = file_name data_report_tag[date]['false'][file_name]["check_result"] = check_result data_report_tag[date]['false'][file_name]["file_size(MB)"] = file_size data_report_tag[date]['false'][file_name]["data_link"] = ''.join(["s3://sh40_fieldtest_dataset/",data_month,'/false_data/', file_name]) total_file_size += file_size file_size = str(check_result) + "_file_size(MB)" if not file_size in data_report_tag[date].keys(): data_report_tag[date][file_size] = 0 data_report_tag[date][file_size] += total_file_size print("\033[1;32m [INFO]\033[0m! generate data report successfully\n") saveTag('../data_report/', data_report_tag, formatted_today + '.json')
def getObjectNumber(dir_path): object_count = loadTag(dir_path, 'object_count.json') number = [] try: for count in object_count["Object_number"]: for a in count: number.append(count[a]) #object_size = number / len(object_count["Object_number"]) except: return 0 # if object_size < 800: # tag_info['global_tag'].append("fewer_objects") # elif object_size > 800 and object_size < 1800: # tag_info['global_tag'].append("normal_objects") # elif object_size > 1800: # tag_info['global_tag'].append("many_objects") return number saveTag(dir_path,vnumber,'statics_result_12.json')
def main(): statics_result = { "test_duration": 0, "test_mileage": 0, 'route': {}, "record_tag": {} } vnumber = {"fewer_objects": 0, "normal_objects": 0, "many_objects": 0} dir_path = "/media/sensetime/FieldTest2/data/123/" tag_list = getMatchedFilePaths(dir_path, 'data*', '.json', True) print(len(tag_list)) i = 0 result_list = [] for tag_path in tag_list: tag_data = loadTag(tag_path, '') if tag_data is None: continue # for count in tag_data["Object_number"]: # for a in count: # if count[a] < 800: # vnumber['fewer_objects'] += 1 # elif count[a] > 800 and count[a] < 1800: # vnumber['normal_objects'] += 1 # elif count[a] > 1800: # vnumber['many_objects'] += 1 # if "ego_tags" in tag_data.keys() and "CloseToJunction" in tag_data["ego_tags"]: # result_list.append(os.path.basename(os.path.split(tag_path.split('screen_cast')[0])[0])) # print(result_list) if "backup" in tag_data.keys(): tag_data = tag_data["backup"][0]["data_tag"] tag_data["test_date"] = tag_data["test_date"].replace('-', '_') # try: # if tag_data["test_date"].split('_',-1)[0] +'_'+tag_data["test_date"].split('_',-1)[1]!= "2020_09": # continue # except Exception as e: # continue # if tag_data["test_car_id"] != "CN-012": # continue # if tag_data["test_type"] != 2: # continue #print(tag_path) if "test_duration" in tag_data.keys(): statics_result["test_duration"] += tag_data["test_duration"] if "test_mileage" in tag_data.keys( ) and tag_data["test_mileage"] > 0 and tag_data["test_mileage"] < 200: statics_result["test_mileage"] += tag_data["test_mileage"] if "origin_record_tag" in tag_data.keys( ) and tag_data["origin_record_tag"] != []: for record_tag in tag_data["origin_record_tag"]: if not record_tag["tag_en"] in statics_result[ "record_tag"].keys(): statics_result["record_tag"][record_tag["tag_en"]] = 0 statics_result["record_tag"][record_tag["tag_en"]] += 1 if 'route' in tag_data.keys(): if tag_data['route'] not in statics_result["route"]: statics_result["route"][tag_data['route']] = 0 statics_result["route"][tag_data['route']] += 1 i += 1 print(i) saveTag(dir_path, statics_result, 'result.json') # if "global_tag" in tag_data.keys(): # for tag in tag_data["global_tag"]: # if tag in ["fewer_objects","normal_objects","many_objects"]: # vnumber[tag] +=1 # with open('/media/sensetime/FieldTest1/data/result.txt', 'w') as fw: # for path in result_list: # fw.write(path + '\n') saveTag(dir_path, vnumber, 'statics_result_12.json')
def data_upload(self, dir_path, tag_info, slice=False, upload=True): 'data_upload with aws' dir_name = os.path.split(dir_path)[1] vehicle_id = tag_info["test_car_id"].replace("-", "") data_month = tag_info["test_date"].rsplit("_", 1)[0] try: feature = False if tag_info["issue_id"] == ["repo_master" ] else True except Exception as e: feature = False task_id = str(tag_info["task_id"]) + '/' if feature else '' if slice: upload_path = ''.join([dir_path, "_slice/ "]) dst_path = ''.join([ self.getDataCollectionDstLink(tag_info, data_month, slice=True), tag_info["test_date"], '/', vehicle_id, '/', task_id ]) self.sliceDataCheck(dir_path + '_slice/') else: upload_path = ''.join([dir_path, '/ ']) tag_path = ''.join([ self.getDataCollectionDstLink(tag_info, data_month, slice=False), tag_info["test_date"], '/', vehicle_id, '/', task_id, dir_name ]) dst_path = ''.join([tag_path, '/']) tag_info["data_link"] = tag_path tag_info["data_type"] = "raw" tag_info['aws_endpoint'] = self.end_point if tag_info["test_type"] == 1: aa, eval_result = self.getPPEvalResult(tag_path) if aa: tag_info["pp_evaluation"] = eval_result if tag_info["test_type"] == 3 and self.config_["evaluation"]: generate_evaluation_result.generateAdasEval( dir_path, self.config_) tag_info["adas_evaluation"] = self.getAdasResult(tag_path) try: if self.false_reason != []: for false_check in self.false_reason: tag_info["global_tag"].append("no_" + false_check) if self.tprfile_resut != []: for tprofile in self.tprfile_resut: tag_info["global_tag"].append(tprofile) except Exception as e: print("write global tag error") saveTag(dir_path, tag_info, file_name='data-tag.json') arg2 = ''.join([ "s3 cp ", upload_path, dst_path, upload_recursive + self.end_point ]) # arg_slam =''.join(["s3 cp ", os.path.join(dir_path,'logs/map_update_areas.json '), # "s3://sz21_data_collection/slam_file/"+data_month+'/'+tag_info["test_date"]+'/'+ # vehicle_id+'/'+task_id+dir_name+'/'+tag_info.get("route")+'/map_update_areas.json', # " --profile ad_system_common --only-show-errors --endpoint-url=" + self.end_point]) print(getTime() + "\033[1;32m [INFO] Uploading ...\033[0m ", arg2) if upload: star_upload_time = time.time() file_size = getFileSize(upload_path.replace(' ', '')) upload_result = AWS_DRIVER.main(arg2.split()) # AWS_DRIVER.main(arg_slam.split()) cost_time = time.time() - star_upload_time upload_speed = file_size / cost_time logger(1, "Uploaded dir:" + upload_path) if upload_result == 0: print (getTime()+"\033[1;32m [INFO]", dir_name + \ "\033[0m", "\033[0m has\033[1;32m uploaded successfully! Speed:\033[0m "+ str(upload_speed) +" MB/s") else: print(getTime() + "\033[1;32m [INFO]", dir_name + "\033[0m", "\033[0m \033[1;32m upload failed!\033[0m") self.TransferPost(tag_info)
def dataSegment(self, dir_name, segment_point): "data segment pipeline" print("checkpoint40") print(self.tag_module_list) segment_list = [] case_tagging_list = [] prediction_tagging_list = [] for record_tag in segment_point: # record_tag is every set #dir_path = os.path.join(self.input_data_path, dir_name) dir_path = self.input_data_path tag_name = record_tag.get("tag_en") # tag_name = cloudyday etc if not tag_name in self.tag_module_list: # special manual tagging, take over, dangerous driving etc if "data_type" in record_tag and record_tag[ "data_type"] == "eval": level_name, log_type, tagging_module = ["EVAL", 0, 1] else: level_name, log_type, tagging_module = [ "3D-Perception", 0, 1 ] if self.tag_info['test_type'] == 3: level_name, log_type, tagging_module = ["ADAS", 0, 1] else: level_name = self.tag_module_list[tag_name]["level_name"] log_type = self.tag_module_list[tag_name]["log_type"] tagging_module = self.tag_module_list[tag_name][ "tagging_module"] print(getTime() + "\033[1;32m [INFO] \033[0m", level_name, "==>> ", tag_name, "==== module_type:", log_type) if "end" in record_tag: front_time = 2 behind_time = (record_tag["end"] - record_tag["start"]) // 1000 else: if not tag_name in self.tag_module_list: front_time = 20 behind_time = 10 else: front_time = self.tag_module_list[tag_name]["front_time"] behind_time = self.tag_module_list[tag_name]["behind_time"] if behind_time < 6: behind_time = 10 if "end" in record_tag: input_timestamp = (record_tag["start"] + record_tag["end"]) / 2000 else: input_timestamp = record_tag["start"] / 1000 time_point = record_tag["start"] * 1000 test_date = self.tag_info["test_date"] test_time = str(record_tag["start_format"].replace(":", "_")) segment_dir_name = ''.join([test_date, '_', test_time]) print("checkpoint41") output_path = ''.join([ dir_path, '_slice/', level_name, '/', tag_name, '/', segment_dir_name ]) print(output_path) if self.tag_info["issue_id"][0] != "repo_master" and \ (record_tag['tag_en'] == "take_over" or record_tag['tag_en'] == "Emergency_brake"): segment_list.append({ "output_dir": output_path, "time_point": time_point, "front_duration": front_time, "behind_duration": behind_time, "log_type": log_type }) else: segment_list.append({ "output_dir": output_path, "time_point": time_point, "front_duration": front_time, "behind_duration": behind_time, "log_type": log_type }) if level_name != "EVAL": case_tagging_list.append({ "input_dir": ''.join([output_path, '/']), "module_name": tag_name, "input_timestamp": input_timestamp, "tagging_module": tagging_module }) if tag_name == "abnormal_prediction_trajectory": prediction_tagging_list.append({ "input_dir": ''.join([output_path, '/']), "module_name": tag_name, "input_timestamp": input_timestamp }) if not os.path.exists(output_path): print("checkpoint42") os.makedirs(output_path) print("checkpoint43") task_id = '' if self.tag_info["issue_id"][ 0] == "repo_master" else str(self.tag_info["task_id"]) + '/' vehicle_id = self.tag_info["test_car_id"].replace("-", "") module_tag_data = copy.deepcopy(self.tag_info) data_month = self.tag_info["test_date"].rsplit("_", 1)[0] dst_path = ''.join([ self.getDataCollectionDstLink(self.tag_info, data_month, slice=True), self.tag_info["test_date"], '/', vehicle_id, '/', task_id ]) module_tag_data["data_link"] = ''.join( [dst_path, level_name, '/', tag_name, '/', segment_dir_name]) store_path = ''.join([ dir_path, '_slice/', level_name, '/', tag_name, '/', segment_dir_name ]) module_tag_data["origin_record_tag"] = [record_tag] if "data_type" in record_tag and record_tag["data_type"] == "eval": module_tag_data["data_type"] = record_tag["data_type"] module_tag_data["data_type"] = "segment" module_tag_data["test_type"] = 9 self.post = True else: module_tag_data["data_type"] = "segment" module_tag_data["file_name"] = segment_dir_name module_tag_data["raw_data_link"] = ''.join([ self.getDataCollectionDstLink(self.tag_info, data_month, slice=False), self.tag_info["test_date"], '/', vehicle_id, '/', task_id, dir_name ]) module_tag_data['aws_endpoint'] = self.end_point saveTag(store_path, module_tag_data, file_name='data-tag.json') self.generateLogDownloadFile(log_type, module_tag_data["raw_data_link"], output_path) print("checkpoint44") print(dir_path) print(segment_list) cut_rec_multiprocess.main(dir_path, segment_list) print("checkpoint45") if self.tag_info["test_type"] == 1: self.case_tagging.tagMain(self.tag_info["global_tag"], dir_path, case_tagging_list)
def mainSegment(self, file_list, upload=False): ''' :param dir_name: the dir to be process :param upload: whether upload , default as False :return: o -> right ; 1-> error ''' print("checkpoint1000") print(self.input_data_path) dir_path = self.input_data_path # dir_path = os.path.join(self.input_data_path, dir_name) # check the dir whether right check_result, self.false_reason = self.checkRec(dir_path) print("calling checkRec") for dir_name in file_list: print("checkpoint1006") print(dir_name) print(dir_path) # get the data-tag.json of checked dir tag_data = self.deterDirProperty(dir_path) if not check_result: if 'test_type' in tag_data.keys( ) and tag_data['test_type'] == 3: pass if "issue_id" not in tag_data.keys( ) or tag_data["issue_id"] == []: tag_data["issue_id"] = ["feature"] if tag_data is not None and tag_data["issue_id"] != [ "repo_master" ] and tag_data["task_id"] != 30000: pass else: self.falseDataArchive(dir_path, self.false_reason) return if "backup" in tag_data: tag_data = tag_data["backup"][0]["data_tag"] tag_data["test_date"] = tag_data["test_date"].replace('-', '_') if tag_data is None: return 1 if not "global_tag" in tag_data: tag_data["global_tag"] = [] ## generate evluation result with different module if tag_data["test_type"] == 1 and self.config_["evaluation"]: pool1 = multiprocessing.Pool(processes=12) pool1.apply_async( generate_evaluation_result.generateLocalizationEval, args=( dir_path, self.config_, )) pool1.close() pool1.join() # try: # generate_evaluation_result.generateTprofileEval(dir_path,self.config_) # except Exception as e: # print getTime()+"\033[1;31m [ERROR]\033[0m tprofiling error " if "issue_id" not in tag_data.keys() or tag_data["issue_id"] == []: tag_data["issue_id"] = ["feature"] saveTag(dir_path, tag_data) self.tag_info = tag_data self.tag_info["localization_result"] = self.getLocalizationResult( dir_path.replace(" ", "")) self.tag_info["control_result"] = {} self.tag_info["ap_result"] = {} self.tag_info["file_name"] = dir_name self.tag_info["disk_id"] = self.getDiskId(dir_path) self.addEvalTag(dir_path) self.tag_info["origin_record_tag"] = self.deDuplication( self.tag_info["origin_record_tag"]) saveTag(dir_path, self.tag_info) self.post = True segment, upload = self.judgeIfSegmentAndUpload( self.tag_info, check_result) try: if self.tag_info["test_type"] == 2 and self.tag_info[ "issue_id"] == ["AUTODRIVE-6814"]: overlap = self.reouteCheckFi(dir_path, self.tag_info["test_car_id"]) self.tag_info['global_tag'].append(overlap) try: self.tag_info = self.getObjectNumber( dir_path, self.tag_info) except: pass if None in self.tag_info['global_tag']: self.tag_info['global_tag'].remove(None) saveTag(dir_path, self.tag_info) print("checkpoint29") print(dir_path) print(dir_name) if segment: print("checkpoint30") self.segmentPreprationCollection(dir_name) print("checkpoint31") print(dir_path) print(dir_name) print("checkpoint32") self.mainUpload(dir_path, upload) print("checkpoint33") except Exception as e: print(traceback.format_exc()) logger(1, str(e), LOG_FILE="upload_list/error.log") print(getTime() + "\033[1;31m [ERROR]\033[0m segment or upload error ") return 1 return 0
def main(): statics_result ={"test_duration":0, "test_mileage":0, 'route':{ }, "record_tag":{ }} vnumber = { "fewer_objects":0, "normal_objects":0, "many_objects":0 } dir_path = "/media/sensetime/FieldTest1/data/04_02_CN-009_ARH/" tag_list = getMatchedFilePaths(dir_path,'module_*','.json',True) print(len(tag_list)) result_list = {} i = 0 for tag_path in tag_list: tag_data = loadTag(tag_path,'') if tag_data is None: continue for cpu_core in tag_data["module_eval"]: if cpu_core not in result_list: result_list[cpu_core]= {} result_list[cpu_core]["stddev_frame_rate"] =0 result_list[cpu_core]["avg_frame_rate"] = 0 if tag_data["module_eval"][cpu_core]["stddev_frame_rate"] is not None and \ tag_data["module_eval"][cpu_core]["avg_frame_rate"] is not None: result_list[cpu_core]["stddev_frame_rate"] = \ result_list[cpu_core]["stddev_frame_rate"]+tag_data["module_eval"][cpu_core]["stddev_frame_rate"] result_list[cpu_core]["avg_frame_rate"] = \ result_list[cpu_core]["avg_frame_rate"] + tag_data["module_eval"][cpu_core]["avg_frame_rate"] # if "test_duration" in tag_data.keys(): # statics_result["test_duration"]+=tag_data["test_duration"] # if "test_mileage" in tag_data.keys() and tag_data["test_mileage"]>0 and tag_data["test_mileage"]<200: # statics_result["test_mileage"]+=tag_data["test_mileage"] # if "origin_record_tag" in tag_data.keys() and tag_data["origin_record_tag"] != []: # for record_tag in tag_data["origin_record_tag"]: # if not record_tag["tag_en"] in statics_result["record_tag"].keys(): # statics_result["record_tag"][record_tag["tag_en"]] = 0 # statics_result["record_tag"][record_tag["tag_en"]] +=1 # if 'route' in tag_data.keys(): # if tag_data['route'] not in statics_result["route"]: # statics_result["route"][tag_data['route']] =0 # statics_result["route"][tag_data['route']] += 1 i+=1 print(i) for core in result_list: result_list[core]["avg_frame_rate"] = result_list[core]["avg_frame_rate"]/5 result_list[core]["stddev_frame_rate"] = result_list[core]["stddev_frame_rate"] / 5 saveTag(dir_path,result_list,'result.json') # if "global_tag" in tag_data.keys(): # for tag in tag_data["global_tag"]: # if tag in ["fewer_objects","normal_objects","many_objects"]: # vnumber[tag] +=1 # with open('/media/sensetime/FieldTest1/data/result.txt', 'w') as fw: # for path in result_list: # fw.write(path + '\n') saveTag(dir_path,vnumber,'statics_result_12.json')
def main(): input_path = '/media/sensetime/FieldTest/data/local_eval/true_label' file_list = getAllDataDir(input_path) for file_name in file_list: eval_file_path = os.path.join(input_path, file_name, 'localization_eval') tag_file_path = os.path.join(input_path, file_name) eval_info = loadTag(eval_file_path, 'evaluation_result.json') tag_info = loadTag(tag_file_path, '12.json') for tag in tag_info["tags"]: print(file_name) save_dict = {} start = int(str(int(tag["start"] * 1000))[0:11]) - 25 if "end" in tag: end = int(str(int(tag["end"] * 1000))[0:11]) + 25 else: end = start + 50 for eval in eval_info["Tags"]: eval_start = int(str(int(eval["start"] * 1000))[0:11]) eval_end = int(str(int(eval["end"] * 1000))[0:11]) if not judgeIfOverLap(start, end, eval_start, eval_end): continue if eval["tag_en"] == "longitudinal_jump": if not eval["tag_en"] in save_dict: save_dict[eval["tag_en"]] = eval[ "max_longitudinal_jump(m/s)"] else: if save_dict[eval["tag_en"]] < eval[ "max_longitudinal_jump(m/s)"]: save_dict[eval["tag_en"]] = eval[ "max_longitudinal_jump(m/s)"] elif eval["tag_en"] == "lateral_jump": if not eval["tag_en"] in save_dict: save_dict[ eval["tag_en"]] = eval["max_lateral_jump(m/s)"] else: if save_dict[eval["tag_en"]] < eval[ "max_lateral_jump(m/s)"]: save_dict[ eval["tag_en"]] = eval["max_lateral_jump(m/s)"] elif eval["tag_en"] == "heading_jump": if not eval["tag_en"] in save_dict: save_dict[ eval["tag_en"]] = eval["max_heading_jump(deg/s)"] else: if save_dict[eval["tag_en"]] < eval[ "max_heading_jump(deg/s)"]: save_dict[eval["tag_en"]] = eval[ "max_heading_jump(deg/s)"] elif eval["tag_en"] == "large_longitudinal_error": if not eval["tag_en"] in save_dict: save_dict[ eval["tag_en"]] = eval["max_longitudinal_error(m)"] else: if save_dict[eval["tag_en"]] < eval[ "max_longitudinal_error(m)"]: save_dict[eval["tag_en"]] = eval[ "max_longitudinal_error(m)"] elif eval["tag_en"] == "large_lateral_error": if not eval["tag_en"] in save_dict: save_dict[ eval["tag_en"]] = eval["max_lateral_error(m)"] else: if save_dict[ eval["tag_en"]] < eval["max_lateral_error(m)"]: save_dict[ eval["tag_en"]] = eval["max_lateral_error(m)"] elif eval["tag_en"] == "large_heading_error": error_num = "max_heading_error(m)" for key in eval.keys(): if key.split('(')[0] == "max_heading_error": error_num = key if not eval["tag_en"] in save_dict: save_dict[eval["tag_en"]] = eval[error_num] else: if save_dict[eval["tag_en"]] < eval[error_num]: save_dict[eval["tag_en"]] = eval[error_num] saveTag(tag_file_path, save_dict, tag["start_format"].replace(':', "_") + '.json') plot_dict = { "longitudinal_jump": [], "lateral_jump": [], "heading_jump": [], "large_longitudinal_error": [], "large_lateral_error": [], "large_heading_error": [], "large_time_gap": [] } tag_file_path = os.path.join(input_path, file_name, 'localization_eval') eval_tag = loadTag(tag_file_path, 'evaluation_result.json') if eval_tag is None: continue for tag in eval_tag["Tags"]: print(file_name) print(tag) st = int(str(int(tag["start"] * 1000))[0:11]) - 1599000000 ed = int(str(int(tag["end"] * 1000))[0:11]) - 1599000000 plot_dict[tag["tag_en"]].append([st, ed]) plt = plotMap(plot_dict) plt.savefig(tag_file_path + "/tag_file.png", dpi=120)
def getTrueList(file_path, check_result=True): ''' :param input_file_list: :param check: :return: ''' # if not judge_file_data(file_path): # return if not os.path.exists('../data_report'): os.makedirs('../data_report') data_report_tag = loadTag('../data_report/', formatted_today + '.json') date = str(formatted_today) total_file_size = 0 total_file_time_length = 0 if not date in data_report_tag.keys(): data_report_tag[date] = defaultdict(lambda: {}) file_name = file_path.split('/', -1)[-1] tag_data = loadTag(file_path) if tag_data == {}: return file_size = getFileSize(file_path) if 'test_duration' in tag_data.keys(): file_time_length = tag_data.get('test_duration') else: file_time_length = 1800 if not 'true' in data_report_tag[date]: data_report_tag[date]['true'] = defaultdict(lambda: {}) if not file_name in data_report_tag[date]['true']: data_report_tag[date]['true'][file_name] = defaultdict(lambda: {}) data_report_tag[date]['true'][file_name]["upload_date"] = date data_report_tag[date]['true'][file_name]["data_type"] = 'raw' data_report_tag[date]['true'][file_name]["data_name"] = file_name data_report_tag[date]['true'][file_name]["test_car_id"] = tag_data["test_car_id"] data_report_tag[date]['true'][file_name]["check_result"] = check_result data_report_tag[date]['true'][file_name]["file_size(MB)"] = file_size data_report_tag[date]['true'][file_name]["data_link"] = tag_data["data_link"] total_file_size += file_size total_file_time_length += file_time_length / 60 print(file_path + '_slice') if os.path.exists(file_path + '_slice'): slice_file_size = getFileSize(file_path + '_slice') slice_file_name = file_name + '_slice' if not slice_file_name in data_report_tag[date]['true']: data_report_tag[date]['true'][slice_file_name] = defaultdict(lambda: {}) data_report_tag[date]['true'][slice_file_name]["upload_date"] = date data_report_tag[date]['true'][slice_file_name]["data_type"] = 'segment' data_report_tag[date]['true'][slice_file_name]["data_name"] = slice_file_name data_report_tag[date]['true'][file_name]["test_car_id"] = tag_data["test_car_id"] data_report_tag[date]['true'][slice_file_name]["check_result"] = check_result data_report_tag[date]['true'][slice_file_name]["file_size(MB)"] = slice_file_size data_report_tag[date]['true'][slice_file_name]["data_link"] = tag_data["data_link"].replace('raw', 'segment') + '_slice' total_file_size += slice_file_size file_size_str = str(check_result) + "_file_size(MB)" if not file_size_str in data_report_tag[date].keys(): data_report_tag[date][file_size_str] = 0 data_report_tag[date][file_size_str] += total_file_size time_length_str = "Test_length(Min)" if not time_length_str in data_report_tag[date].keys(): data_report_tag[date][time_length_str] = 0 data_report_tag[date][time_length_str] += total_file_time_length print("\033[1;32m [INFO]\033[0m! generate data report successfully\n") saveTag('../data_report/', data_report_tag, formatted_today + '.json')