def process_downsampler_compare(yuv_list): for one_yuv in yuv_list: width, height, framerate = CodecUtil.get_resolution_from_name(one_yuv) jsvm_out = jsvm_downsampler(one_yuv, width, height) downsampler1_out, downsampler2_out = test_downsampler(one_yuv, width, height) # psnr ing bitrate, psnr_y1, psnr_u1, psnr_v1 = CodecUtil.calculate_psnr(width, height, jsvm_out, downsampler1_out) bitrate, psnr_y2, psnr_u2, psnr_v2 = CodecUtil.calculate_psnr(width, height, jsvm_out, downsampler2_out)
def process_downsampler_compare(yuv_list): for one_yuv in yuv_list: width, height, framerate = CodecUtil.get_resolution_from_name(one_yuv) jsvm_out = jsvm_downsampler(one_yuv, width, height) downsampler1_out, downsampler2_out = test_downsampler( one_yuv, width, height) # psnr ing bitrate, psnr_y1, psnr_u1, psnr_v1 = CodecUtil.calculate_psnr( width, height, jsvm_out, downsampler1_out) bitrate, psnr_y2, psnr_u2, psnr_v2 = CodecUtil.calculate_psnr( width, height, jsvm_out, downsampler2_out)
def sendRequest(strUrl, strRequestMethod="GET", objData=None): bufBodyResult = cStringIO.StringIO() bufHeaderResullt = cStringIO.StringIO() curlRequest = pycurl.Curl() curlRequest.setopt(curlRequest.URL, strUrl) curlRequest.setopt(curlRequest.SSL_VERIFYPEER, False) # curlRequest.setopt(curlRequest.FAILONERROR, True) curlRequest.setopt(curlRequest.USERPWD, "%s:%s" % (ONEPLUS_JIRA_USER, ONEPLUS_JIRA_PWD)) curlRequest.setopt(curlRequest.HTTPHEADER, ['Content-Type: application/json']) curlRequest.setopt(curlRequest.WRITEFUNCTION, bufBodyResult.write) curlRequest.setopt(curlRequest.HEADERFUNCTION, bufHeaderResullt.write) if strRequestMethod != "GET": curlRequest.setopt(curlRequest.CUSTOMREQUEST, strRequestMethod) if objData is not None: if isinstance(objData, str): curlRequest.setopt(curlRequest.POSTFIELDS, objData) elif isinstance(objData, file): curlRequest.setopt(curlRequest.HTTPHEADER, ['X-Atlassian-Token: nocheck']) curlRequest.setopt(curlRequest.HTTPPOST, [ ('file', (pycurl.FORM_BUFFER, CodecUtil.returnUtfStr(os.path.basename(objData.name)), pycurl.FORM_BUFFERPTR, objData.read())), ]) objData.close() try: curlRequest.perform() except Exception, e: logger.exception("Exception Caught") return (None, e)
def resultAnalysis(self, strTestTempPath): """ Analysis files under ~/abts/mnt/test_result/ """ listReporAttachment = [] #return Attachment list add by libin859 listReportImage = [] dictIssueList = {} dictTestResult = {"Pass": None, "Fail": None, "Skip": None, "Abort": None} for key in dictTestResult: dictTestResult[key] = {"TotalNum": 0, "TestCase": []} strTestResultFilePath = os.path.join( strTestTempPath, "testresult.json") if os.path.isfile(strTestResultFilePath) is False: logger.warning("Canot Find Test Result File Under '%s'" % strTestResultFilePath) return (dictTestResult, dictIssueList, listReportImage, listReporAttachment) try: # fileResultXml = open(strTestResultFilePath, 'r') # strResultXml = fileResultXml.read() # ordereddictResult = XmlToDict.parse(strResultXml) # ordereddictResult = ordereddictResult.get("TestResult", {}) fileResultJson = open(strTestResultFilePath, 'r') strResultJson = fileResultJson.read() dictResult = json.loads(strResultJson, encoding="utf-8") logger.info("Make Sure No Unicode") dictResult = CodecUtil.transferUnicodeToStr(dictResult) except Exception, e: logger.exception("Exception Caught: %s" %str(e))
def generate_yuv(bsname, width, height, skip_list): rec_yuv = bsname+'_dec.yuv' CodecUtil.decode(bsname, rec_yuv) copy_yuv = bsname+'_dec_copy.yuv' print(skip_list) actual_skip_list = [] for i in skip_list: if i == 0: sys.stdout.write("Incorrect Skip Idx = 0!\n") return None copy_one_frame(rec_yuv, copy_yuv, width, height, i-1) actual_skip_list.append(i-1) os.rename(copy_yuv, rec_yuv) # end of processing yuv print(actual_skip_list) return rec_yuv
def encode_one_yuv(exe_path, one_yuv, usage_type=0, qp=24): name = ((one_yuv.split(os.sep))[-1]) width, height, frame_rate = __init__.get_resolution_from_name(name) if frame_rate == 0: frame_rate = 30 # encoding current_path = os.getcwd() os.chdir(exe_path) print("current path is %s\n" %exe_path) if not SKIP_ENCODING: if TEST_JM: bs_name, log_name = CodecUtil.jm_encoder(one_yuv, usage_type, width, height, qp, ' -p IDRPeriod=1 -p LevelIDC=51 ') else: bs_name, log_name, result_line = CodecUtil.openh264_encoder_qp(one_yuv, usage_type, width, height, qp, ' -iper 0 ') else: bs_name = one_yuv.split(os.sep)[-1] + '_br' + str(qp) + '.264' log_name = one_yuv.split(os.sep)[-1] + '_br' + str(qp) + '.log' #deal with log file fps = CodecUtil.openh264_encoder_log_fps(log_name) if not PERFORMANCE_ONLY: # decoding rec_yuv = bs_name+'_dec.yuv' if TEST_JM: CodecUtil.jm_decode(bs_name, rec_yuv) else: CodecUtil.decode(bs_name, rec_yuv) # psnr ing frame_num, bitrate, psnr_y, psnr_u, psnr_v = CodecUtil.PSNRStaticd(width, height, one_yuv, rec_yuv, rec_yuv+'.log', bs_name, frame_rate) ssim, msssim = 0,0 if VQMT_ON: psnr2, ssim, msssim = CodecUtil.Vqmt(width, height, one_yuv, rec_yuv, frame_num) file_size = os.path.getsize(bs_name) current_test_point = OneTestPoint(width, height, frame_rate, frame_num, qp, file_size, 0, fps, bitrate, psnr_y, psnr_u, psnr_v, ssim, msssim) os.remove(rec_yuv) else: current_test_point = OneTestPoint(width, height, frame_rate, 0, qp, 0, 0, fps, 0, 0, 0, 0, 0, 0) if DEBUG==0: os.remove(bs_name) os.remove(log_name) os.chdir(current_path) return current_test_point
def encode_one_yuv(exe_path, one_yuv, usage_type=0, qp=24): name = ((one_yuv.split(os.sep))[-1]) width, height, frame_rate = CodecUtil.get_resolution_from_name(name) if frame_rate == 0: frame_rate = 30 # encoding current_path = os.getcwd() os.chdir(exe_path) print("current path is %s\n" % exe_path) bs_name, log_name = CodecUtil.call_encoder_qp(one_yuv, usage_type, width, height, qp, ' -threadIdc 4 ') #deal with log file fps = CodecUtil.encoder_log_file(log_name) if fps == -1: return OneTestPoint(qp, 0, 0, 0, 0, 0) if not PERFORMANCE_ONLY: # decoding rec_yuv = bs_name + '_dec.yuv' CodecUtil.decode(bs_name, rec_yuv) # psnr ing bitrate, psnr_y, psnr_u, psnr_v = CodecUtil.calculate_psnr( width, height, one_yuv, rec_yuv, rec_yuv + '.log', bs_name, frame_rate) current_test_point = OneTestPoint(qp, fps, bitrate, psnr_y, psnr_u, psnr_v) os.remove(rec_yuv) else: current_test_point = OneTestPoint(qp, fps, 0, 0, 0, 0) os.chdir(current_path) return current_test_point
def process_compare_enc(yuv_list, downscale): TestPoint_dict = {} out_path = __init__.OUT_DATA_PATH BitRateTable = CodecUtil.cBitRateTable() for one_yuv in yuv_list: width, height, frame_rate = __init__.get_resolution_from_name(one_yuv) width_out = width / downscale height_out = height / downscale out_yuv_resolution = "%d" % width_out + "x" + "%d" % height_out jsvm_out = ( out_path + os.sep + os.path.basename(one_yuv)[0:-4] + "_to_" + out_yuv_resolution + "_downConvert.yuv" ) downsampler1_out = ( out_path + os.sep + os.path.basename(one_yuv)[0:-4] + "_to_" + out_yuv_resolution + "_downsampler1.yuv" ) downsampler2_out = ( out_path + os.sep + os.path.basename(one_yuv)[0:-4] + "_to_" + out_yuv_resolution + "_downsampler2.yuv" ) # encoder three yuv files qp = 36 usage_type = 0 result_path = os.path.abspath(__init__.OUT_DATA_PATH) jsvm_out = os.path.abspath(jsvm_out) downsampler1_out = os.path.abspath(downsampler1_out) downsampler2_out = os.path.abspath(downsampler2_out) current_path = os.getcwd() os.chdir(H264CODEC_PATH) for source in (jsvm_out, downsampler1_out, downsampler2_out): if RC_ON: bs_name, log_name, result_line = CodecUtil.openh264_encoder_rc( source, usage_type, width_out, height_out, frame_rate, BitRateTable.get_one_camera_point(width_out, height_out), BitRateTable.get_one_camera_point(width_out, height_out) * 2, 0, 0, ) else: bs_name, log_name, result_line = CodecUtil.openh264_encoder_qp( source, usage_type, width_out, height_out, qp ) bs_name = __init__.move_to_result_path(bs_name, result_path) log_name = __init__.move_to_result_path(log_name, result_path) rec_yuv = bs_name[0:-4] + "_dec.yuv" CodecUtil.decode(bs_name, rec_yuv) # encoder information frames, encode_time, fps = CodecUtil.openh264_encoder_log_all(log_name) # psnr ing frame_num, bitrate, psnr_y, psnr_u, psnr_v = CodecUtil.PSNRStaticd( width_out, height_out, source, rec_yuv, rec_yuv + ".log", bs_name, frame_rate ) psnr2, ssim = 0, 0 if VQMT_ON: psnr2, ssim = CodecUtil.Vqmt(width_out, height_out, source, rec_yuv, frame_num) file_size = os.path.getsize(bs_name) current_test_point = OneTestPoint( width_out, height_out, frame_rate, frames, qp, file_size, encode_time, fps, bitrate, psnr_y, psnr_u, psnr_v, psnr2, ssim, ) if not TestPoint_dict.has_key(source): TestPoint_dict[source] = {} if not TestPoint_dict[source].has_key(qp): TestPoint_dict[source][qp] = {} TestPoint_dict[source][qp] = current_test_point os.remove(rec_yuv) for source in (downsampler1_out, downsampler2_out): bs_name, log_name, result_line = CodecUtil.openh264_encoder_qp( source, usage_type, width_out, height_out, qp ) bs_name = __init__.move_to_result_path(bs_name, result_path) rec_yuv = bs_name[0:-4] + "_dec.yuv" CodecUtil.decode(bs_name, rec_yuv) frame_num, bitrate, psnr_y, psnr_u, psnr_v = CodecUtil.PSNRStaticd(width_out, height_out, jsvm_out, rec_yuv) psnr2, ssim = 0, 0 if VQMT_ON: psnr2, ssim = CodecUtil.Vqmt(width_out, height_out, source, rec_yuv, frame_num) current_test_point = OneTestPoint( width_out, height_out, frame_rate, frame_num, qp, 0, 0, 0, bitrate, psnr_y, psnr_u, psnr_v, psnr2, ssim ) current_name = str(os.path.basename(source) + "_comparejsvm") if not TestPoint_dict.has_key(current_name): TestPoint_dict[current_name] = {} if not TestPoint_dict[current_name].has_key(qp): TestPoint_dict[current_name][qp] = {} TestPoint_dict[current_name][qp] = current_test_point os.remove(rec_yuv) os.chdir(current_path) write_testpoint_to_csv(str("encCompare" + "_%d" % downscale), __init__.OUT_DATA_PATH, TestPoint_dict)
def batch_encoder_test(enc_path, usage_type, bit_rate_list, common_fps, multi_layer_flag=False): current_path = os.getcwd() os.chdir(enc_path) for f in glob.glob(enc_path + os.sep + '*.log'): os.remove(f) for f in glob.glob(enc_path + os.sep + '*.264'): os.remove(f) fout = open('Results.csv', 'w') fout.write('bs_name, rc_mode, frame_skip, target_br, average_br, average_br_ratio, max_qp, psnr_y, ' 'max_burst_ratio, avg_burst_ratio, max_exceed_times, max_exceed_times_ratio, max_br_burst_ratio, period_exceed_ratio, skip_ratio, skip_successive\n') if usage_type == 0: seq_path = CAMERA_SEQ_PATH elif usage_type == 1: seq_path = SCREEN_SEQ_PATH for frame_skip_iter in frame_skip_list: for rc_mode_iter in RC_MODE_LIST: for f in glob.glob(seq_path + os.sep + '*.yuv'): width, height, framerate = CodecUtil.get_resolution_from_name(f) max_item = 0 cur_bit_rate_list = [] for item in bit_rate_list: if cur_bit_rate_list == [] or ( (width*height/256) >= item and item > max_item ): cur_bit_rate_list = bit_rate_list[item] max_item = item for bit_rate_item in cur_bit_rate_list: if multi_layer_flag is False: target_br = int(bit_rate_item*framerate/common_fps) max_br = int(target_br * float(MAX_BR_RATIO)) # process each file bs_name, log_name = CodecUtil.call_encoder(f, usage_type, width, height, framerate, target_br, max_br, rc_mode_iter, frame_skip_iter) #encoded current_log = cCurrentLog() current_log.read_logs(log_name) max_exceed_times, max_exceed_times_ratio, max_br_burst_ratio, max_burst_ratio, avg_burst_ratio, period_exceed_ratio = current_log.check_one_setting(target_br, max_br) skip_ratio, skip_successive = current_log.get_skip_status() skip_list = current_log.get_skip_list() if frame_skip_iter == 0 and skip_list!=[]: sys.stdout.write("Error! Frameskip(%d) not allowed but there is skipped: %s\n" %(frame_skip_iter, str(skip_list))) return elif skip_list != [] and skip_list[0] == 0: sys.stdout.write("Incorrect Skip Idx = 0!\n") return average_bit_rate = current_log.get_single_layer_average_bit_rate() if average_bit_rate <= 0: continue rec_yuv = generate_yuv(bs_name, width, height, skip_list) bitrate, psnr_y, psnr_u, psnr_v = calculate_psnr(width, height, f, rec_yuv, 'Results_PSNR', bs_name, framerate) fout.write('%s, %d, %d, %d, %d, %f, %d, %f, %f, %f, %d, %f, %d, %f, %f, %d\n' %(bs_name, rc_mode_iter, frame_skip_iter, target_br, average_bit_rate, average_bit_rate*100/(target_br*1000), current_log.get_max_qp(), psnr_y, max_burst_ratio, avg_burst_ratio, max_exceed_times, max_exceed_times_ratio, max_br_burst_ratio, period_exceed_ratio, skip_ratio, skip_successive)) os.remove(rec_yuv) else: target_br = [ int(item*framerate/common_fps) for item in bit_rate_item ] max_br = [ int(item * float(MAX_BR_RATIO)) for item in target_br ] bs_name, log_name = CodecUtil.call_multilayer_encoder(f, usage_type, width, height, framerate, target_br, max_br, rc_mode_iter, frame_skip_iter) #encoded current_log = cCurrentLog() current_log.read_logs(log_name) average_bit_rate = current_log.get_average_bit_rate() for did in range(4): if average_bit_rate[did] <= 0: continue max_exceed_times, max_exceed_times_ratio, max_br_burst_ratio, max_burst_ratio, average_burst_ratio, period_exceed_flag \ = current_log.check_one_setting_multi_layer(did, target_br[did], max_br[did]) skip_ratio, skip_successive = current_log.get_skip_status() fout.write('%s_layer%d, %s, %d, %d, %d, %f, %f, %f, %f, %d, %f, %f, %f, %d\n' %(bs_name, did, rc_mode_iter, frame_skip_iter, target_br[did], average_bit_rate[did], average_bit_rate[did]*100/(target_br[did]*1000), 0, max_burst_ratio, average_burst_ratio, max_exceed_times, max_exceed_times_ratio, period_exceed_flag, skip_ratio, skip_successive)) fout.close() os.chdir(current_path)
def resultAnalysis(self, strTestTempPath): """ Analysis files under ~/abts/mnt/test_result/ """ listReportImage = [] dictIssueList = {} dictTestResult = { "Pass": None, "Fail": None, "Skip": None, "Abort": None } for key in dictTestResult: dictTestResult[key] = {"TotalNum": 0, "TestCase": []} strTestResultFilePath = os.path.join(strTestTempPath, "testresult.json") if os.path.isfile(strTestResultFilePath) is False: logger.warning("Canot Find Test Result File Under '%s'" % strTestResultFilePath) return (dictTestResult, dictIssueList, listReportImage) try: # fileResultXml = open(strTestResultFilePath, 'r') # strResultXml = fileResultXml.read() # ordereddictResult = XmlToDict.parse(strResultXml) # ordereddictResult = ordereddictResult.get("TestResult", {}) fileResultJson = open(strTestResultFilePath, 'r') strResultJson = fileResultJson.read() dictResult = json.loads(strResultJson, encoding="utf-8") logger.info("Make Sure No Unicode") dictResult = CodecUtil.transferUnicodeToStr(dictResult) except Exception: logger.exception("Exception Caught") else: fileResultJson.close() listTestDetailsInfo = dictResult.get("TestDetails", []) listReportImage = dictResult.get("ReportImage", []) logger.info("Analysis On Test Test Details") for dictCurrentTestDetailInfo in listTestDetailsInfo: strCurrentTestResult = dictCurrentTestDetailInfo.get( "Result", "Unknown").capitalize() if strCurrentTestResult not in dictTestResult: logger.warning( "Test Result Not Valid, Use 'Abort' Instead") strCurrentTestResult = "Abort" strCurrentTestName = dictCurrentTestDetailInfo.get( "TestName", "Unknown") strCurrentTestErrorMsg = dictCurrentTestDetailInfo.get( "ErrorMsg", "") dictTestResult[strCurrentTestResult]["TestCase"].append( strCurrentTestName) dictTestResult[strCurrentTestResult]["TotalNum"] += 1 if strCurrentTestResult != "Pass": logger.info("Test Case '%s' %s, Analysis" % (strCurrentTestName, strCurrentTestResult)) dictCurrentIssueJiraTicketInfo = dictCurrentTestDetailInfo.get( "JiraTicket", None) listCurrentIssueJiraAttachment = dictCurrentTestDetailInfo.get( "JiraAttachment", []) strCurrentJiraTicketUrl = None if dictCurrentIssueJiraTicketInfo is not None: logger.info("Submit A Jira Ticket") (boolJiraCreatedResult, dictJiraTicketInfo) = JiraRest.creatIssue( self.strProjectKey, "Bug", dictCurrentIssueJiraTicketInfo) if boolJiraCreatedResult is False: strCurrentJiraTicketUrl = "Failed To Create" logger.error("Failed to create Jira ticket") else: strJiraKey = dictJiraTicketInfo.get("key", "") logger.info("Jira Ticket Created, Key: '%s'" % strJiraKey) strCurrentJiraTicketUrl = "https://pjira.oneplus.cn/jira/browse/%s" % strJiraKey for strJiraAttachmentPath in listCurrentIssueJiraAttachment: logger.info( "Upload Attachment '%s' To Jira Ticket" % strJiraAttachmentPath) JiraRest.addAttachment(strJiraKey, strJiraAttachmentPath) logger.info("Add Info To Issue List") dictIssueList[strCurrentTestName] = {} dictIssueList[strCurrentTestName][ "ErrorMsg"] = strCurrentTestErrorMsg dictIssueList[strCurrentTestName][ "IssueType"] = strCurrentTestResult dictIssueList[strCurrentTestName][ "JiraTicket"] = strCurrentJiraTicketUrl logger.info("Analysis On Test Report Image") for dictReportImageInfo in listReportImage: strCurrentImagePath = dictReportImageInfo.get("Path", None) if strCurrentImagePath is None or os.path.isfile( strCurrentImagePath) is False: logger.warning("Report Image '%s' Not Valid, Remove" % str(dictReportImageInfo)) listReportImage.remove(dictReportImageInfo) return (dictTestResult, dictIssueList, listReportImage)
keyResult] += intCurrentResultNum dictTestDetails.setdefault( strTestScriptName, {})[keyResult] = dictCurrentTestResult[keyResult].get( "TestCase", [])[:] if len(dictCurrentIssueList) > 0: dictIssueList[ strTestScriptName] = dictCurrentIssueList.copy() listReportImage.extend(listCurrentReportImage) finally: CurrentTestScript = None logger.info("Make Sure No Unicode") dictIssueList = CodecUtil.transferUnicodeToStr(dictIssueList) listReportImage = CodecUtil.transferUnicodeToStr(listReportImage) dictTestDetails = CodecUtil.transferUnicodeToStr(dictTestDetails) return (boolResult, strLastError, dictTestSummarize, dictIssueList, listReportImage, dictTestDetails) if __name__ == '__main__': import sys hdlr = logging.StreamHandler(sys.stdout) hdlr.setFormatter( logging.Formatter( '%(asctime)s - %(name)s [%(levelname)s]: %(message)s')) logger.addHandler(hdlr) aaa = TestRunner("/home/rafe/abts/testqueuepool/1464943670101/",
if args.camera_seq_path is not None: CAMERA_SEQ_PATH = args.camera_seq_path if args.screen_seq_path is not None: SCREEN_SEQ_PATH = args.screen_seq_path if args.rc_mode: RC_MODE_LIST = [args.rc_mode, ] if args.max_range and args.max_range>1: MAX_BR_RATIO = args.max_range if args.time_window_camera and args.time_window_camera>1000: TIME_WINDOW = args.time_window_camera BitRateTable = CodecUtil.cBitRateTable(args.multi_layer_encoding is None) if args.multi_layer_encoding is None: camera_bit_rate_list = { 3600: [900, 1200, 1400, 1600], #1280x720 1800: [400, 600, 800, 1000], #960x540 900: [350, 450, 550, 650], #640x480~640x512 500: [300, 330, 360, 390], 225: [160,],#[160, 200, 250, 360], #320x180~320x192~320x240 80: [80, 100, 140, 160], #160x90~160x128 } batch_encoder_test(args.enc, 0, BitRateTable.get_camera_br_list(), 30) #batch_encoder_test(args.enc, 1, camera_bit_rate_list, 10, args.rc_mode) else: batch_encoder_test(args.enc, 0, BitRateTable.get_camera_br_list(), 30, True)
#!/usr/bin/env python import os, glob import argparse import CodecUtil import config if __name__ == '__main__': #set you search path in config.py argParser = argparse.ArgumentParser() argParser.add_argument("folder", nargs='?', help="log file") args = argParser.parse_args() if not os.path.isdir(args.folder): exit() for file in glob.glob(args.folder + os.sep + '*.264'): CodecUtil.jm_decode(file)