def genSplit(testDataDir): splitConfig = cfg.nnDataDirNameTest + os.path.sep + cfg.splitConfig writeSplitConfig(splitConfig) datamv = 'NONE' splitCmd = 'python %s' % (os.path.join(cfg.path_pyTools_scripts, 'dataGen/GenSynData.py')) splitCmd = '%s %s %s %s %s' % (splitCmd, splitConfig, testDataDir, testDataDir, datamv) exe_cmd(splitCmd, cfg.debug) display.self_print('Output features are generated to %s' % (testDataDir), 'highlight')
def createFileLst(dataDirs, dataExts, dataDim, dataListDirs, trnList, valList): """ create data lists output *.scp will be in dataListDirs """ dataDirs = dataDirs.split(',') dataExts = dataExts.split(',') dataDims = [int(x) for x in dataDim.split('_')] assert len(dataDirs) == len( dataExts), 'Error: sub_1_prepare_list.py dataDirs and dataExts wrong' # get the cross-set of file lists dataList = lstdirNoExt(dataDirs[0], dataExts[0]) for dataDir, dataExt in zip(dataDirs[1:], dataExts[1:]): listTmp = lstdirNoExt(dataDir, dataExt) dataList = crossSet(dataList, listTmp) # check if file exists if len(dataList) < 1: display.self_print("Error: fail to found data. Please check:", 'error') display.self_print( "path_acous_feats, ext_acous_feats, path_waveform in config.py;", 'error') display.self_print("Please also check the names of input data files.", 'error') raise Exception("Error: fail to generate file list.") # check if data exists pre_defined_trn_list = readwrite.read_txt_list(trnList) pre_defined_val_list = readwrite.read_txt_list(valList) diff_trn_list = diff_list(pre_defined_trn_list, dataList) diff_val_list = diff_list(pre_defined_val_list, dataList) if len(diff_trn_list): display.self_print("Error: training data missing. Please check:", 'error') print(diff_trn_list) raise Exception("Error: fail to prepare file list.") if len(diff_val_list): display.self_print("Error: validation data missing. Please check:", 'error') print(diff_val_list) raise Exception("Error: fail to prepare file list.") # before start, take a simple test on the configuration of feature dimension frameNum = None for inputDir, featDim, featName in zip(dataDirs[0:-1], dataDims[0:-1], dataExts[0:-1]): inputFile = os.path.join(inputDir, dataList[0]) + '.' + featName.lstrip('.') if os.path.isfile(inputFile): tmpframeNum = readwrite.read_raw_mat(inputFile, featDim).shape[0] if frameNum is None or frameNum < tmpframeNum: frameNum = tmpframeNum for inputDir, featDim, featName in zip(dataDirs[0:-1], dataDims[0:-1], dataExts[0:-1]): inputFile = os.path.join(inputDir, dataList[0]) + '.' + featName.lstrip('.') if os.path.isfile(inputFile): tmpframeNum = readwrite.read_raw_mat(inputFile, featDim).shape[0] if np.abs(frameNum - tmpframeNum) * 1.0 / frameNum > 0.1: if featDim == readwrite.read_raw_mat(inputFile, 1).shape[0]: pass else: display.self_print("Large mismatch of frame numbers %s" % (inputFile)) display.self_print( "Please check whether inputDim are correct", 'error') display.self_print("Or check input features are corrupted", 'error') raise Exception("Error: mismatch of frame numbers") display.self_print('Generating data lists in to %s' % (dataListDirs), 'highlight') if True: trainSet = pre_defined_trn_list valSet = pre_defined_val_list if len(valSet) > len(trainSet): display.self_print( "Warning: validation set is larger than training set", 'warning') display.self_print("It's better to change train_utts in config.py", 'warning') trainFileOut = dataListDirs + os.path.sep + 'train.lst' trainFilePtr = open(trainFileOut, 'w') for fileName in trainSet: trainFilePtr.write('%s\n' % (fileName)) trainFilePtr.close() if len(valSet): valFileOut = dataListDirs + os.path.sep + 'val.lst' valFilePtr = open(valFileOut, 'w') for fileName in valSet: valFilePtr.write('%s\n' % (fileName)) valFilePtr.close() display.self_print( '\ttrain/val sizes: %d, %d' % (len(trainSet), len(valSet)), 'warning') # done return
def exe_cmd(cmd, debug=False): display.self_print(cmd + '\n', 'highlight') if not debug: os.system(cmd)
pass tmp_data_nc_config = cfg.tmp_test_data_nc_config cmd = 'sh %s' % ( cfg.path_scripts) + os.path.sep + 'sub_05_package_datanc.sh' cmd = cmd + ' %s %s' % (tmp_sub_nc_dir, tmp_idx_dir) cmd = cmd + ' testset %s %s %s' % (tmp_test_lst, tmp_data_nc_config, cfg.path_pyTools_scripts) exe_cmd(cmd, cfg.debug) if os.path.isfile(tmp_test_nc_scp): tmp_test_data_nc_list = readwrite.read_txt_list(tmp_test_nc_scp) if len(tmp_test_data_nc_list) < 1: display.self_print( 'Error: not found test data.nc in %s' % (tmp_sub_nc_dir), 'error') quit() tmp_test_data_nc_args = ','.join(tmp_test_data_nc_list) else: display.self_print('Error: not found %s' % (tmp_test_nc_scp), 'error') quit() # No need to get F0 mean and std if False: # get F0 mean and std dimCnt = 0 f0Dim = -1 meanstd_data = readwrite.read_raw_mat(tmp_mv_data, 1) for acousExt, acousDim in zip(cfg.ext_acous_feats,
tmp_trn_nc_dir = tmp_nc_dir + os.path.sep + cfg.tmp_nc_dir_train tmp_trn_nc_scp = tmp_trn_nc_dir + os.path.sep + 'data.scp' tmp_val_nc_dir = tmp_nc_dir + os.path.sep + cfg.tmp_nc_dir_val tmp_val_nc_scp = tmp_val_nc_dir + os.path.sep + 'data.scp' tmp_mv_data = os.getcwd() + os.path.sep + cfg.tmp_name_mean_file tmp_mdn_config = os.getcwd() + os.path.sep + cfg.tmp_data_dir tmp_mdn_config = tmp_mdn_config + os.path.sep + cfg.tmp_mdn_config_name # Get the string of training data.nc files if os.path.isfile(tmp_trn_nc_scp): tmp_trn_data_nc_list = readwrite.read_txt_list(tmp_trn_nc_scp) if len(tmp_trn_data_nc_list) < 1: display.self_print( 'Error: not found train data.nc in %s' % (tmp_trn_nc_dir), 'error') quit() tmp_trn_data_nc_args = ','.join(tmp_trn_data_nc_list) else: display.self_print('Error: not found %s' % (tmp_trn_nc_scp), 'error') quit() if os.path.isfile(tmp_val_nc_scp): tmp_val_data_nc_list = readwrite.read_txt_list(tmp_val_nc_scp) if len(tmp_val_data_nc_list) < 1: display.self_print('Warning: val data.nc is not used', 'warning') tmp_val_data_nc_args = '' tmp_val_data_nc_args = ','.join(tmp_val_data_nc_list) else: display.self_print(
def prepareData(dataDir): # create directories try: os.mkdir(dataDir) except OSError: pass dataListPath = dataDir + os.path.sep + 'lists' try: os.mkdir(dataListPath) except OSError: pass dataRawDir = dataDir + os.path.sep + cfg.idxDirName try: os.mkdir(dataRawDir) except OSError: pass dataLinkDir = dataDir + os.path.sep + cfg.linkDirname try: os.mkdir(dataLinkDir) except OSError: pass testdataDirs = [] # create file list for inputDirSet, outputDirSet, dataPart in zip(cfg.test_inputDirs, cfg.test_outputDirs, cfg.test_dataDivision): display.self_print('Processing ' + dataPart + ' data', 'highlight') # get the cross-set of file list listInput = lstdirNoExt(inputDirSet[0]) if cfg.outputUttNum > 0: fileList = listInput[0:cfg.outputUttNum] else: fileList = listInput if inputDirSet: for inputDir in inputDirSet: listInput2 = lstdirNoExt(inputDir) fileList = crossSet(fileList, listInput2) if outputDirSet: listOutput = lstdirNoExt(outputDirSet[0]) if outputDirSet: for outputDir in outputDirSet: listOutput2 = lstdirNoExt(outputDir) fileList = crossSet(fileList, listOutput2) # writing file lst fileListFilePath = dataListPath + os.path.sep + dataPart + '.lst' filePtr = open(fileListFilePath, 'w') for fileName in fileList: filePtr.write('%s\n' % (fileName)) filePtr.close() # create file directories dataSaveDir = dataDir + os.path.sep + dataPart testdataDirs.append(dataSaveDir) try: os.mkdir(dataSaveDir) except OSError: pass inputScpList = [] outputScpList = [] # create the fileName + fileExt lists for inputDir, featDim, featName in zip(inputDirSet, cfg.inputDim, cfg.inputExt): tmpFileScp = dataSaveDir + os.path.sep + featName + '.scp' inputScpList.append(tmpFileScp) filePtr = open(tmpFileScp, 'w') for fileName in fileList: filePtr.write('%s%s%s.%s\n' % (inputDir, os.path.sep, fileName, featName)) os.system("ln -f -s %s%s%s.%s %s%s%s.%s" % (inputDir, os.path.sep, fileName, featName, dataLinkDir, os.path.sep, fileName, featName)) filePtr.close() if outputDirSet: for outputDir, featDim, featName in zip(outputDirSet, cfg.outputDim, cfg.outputExt): tmpFileScp = dataSaveDir + os.path.sep + featName + '.scp' outputScpList.append(tmpFileScp) filePtr = open(tmpFileScp, 'w') for fileName in fileList: filePtr.write('%s%s%s.%s\n' % (outputDir, os.path.sep, fileName, featName)) os.system("ln -f -s %s%s%s.%s %s%s%s.%s" % (outputDir, os.path.sep, fileName, featName, dataLinkDir, os.path.sep, fileName, featName)) filePtr.close() filePtr = open(dataSaveDir + os.path.sep + cfg.idxFileName + '.scp', 'w') for fileName in fileList: filePtr.write('%s%s%s.%s\n' % (dataRawDir, os.path.sep, fileName, cfg.idxFileName)) filePtr.close() # create the lab index lists cmd = 'python %s/dataPrepare/getLabIdx5ms.py' % ( cfg.path_pyTools_scripts) cmd = '%s %s %s %s %s %s %s' % (cmd, inputDirSet[0], cfg.inputExt[0], cfg.inputDim[0], dataRawDir, cfg.idxFileName, fileListFilePath) display.self_print('Creating idx', 'highlight') exe_cmd(cmd, cfg.debug) # write data_config.cfg writeDataConfig(dataSaveDir + os.path.sep + 'data_config.py', cfg.idxFileName + '.scp', cfg.fileNumInEachNCPack) # pack data packDataCmd = 'sh %s/sub_05_package_datanc.sh %s %s' % ( cfg.path_scripts, dataSaveDir, cfg.path_pyTools_scripts) display.self_print('Packing data', 'highlight') exe_cmd(packDataCmd, cfg.debug) return testdataDirs
def wavCreate(testDataDir): mlpgFlag = [] pythonScript = os.path.join(cfg.path_pyTools_scripts, 'synWav.py') if cfg.wavformGenerator == 'WORLD': for featName in cfg.wavGenWorldRequire: try: mlpgFlagIdx = cfg.outputExt.index(featName) mlpgFlag.append(cfg.mlpgFlag[mlpgFlagIdx]) except ValueError: print("outputExt in synconfig has no %s " % (featName)) raise Exception("WORLD vocoder requires %s " % (featName)) elif cfg.wavformGenerator == 'STRAIGHT': for featName in cfg.wavGenWorldRequire: try: mlpgFlagIdx = cfg.outputExt.index(featName) mlpgFlag.append(cfg.mlpgFlag[mlpgFlagIdx]) except ValueError: print("outputExt in synconfig has no %s " % (featName)) raise Exception("STRAIGHT vocoder requires %s " % (featName)) else: raise Exception("Unknown wavFormGenerator = %s" % (cfg.wavFormGenerator)) if sum(mlpgFlag) > 0: datamv = cfg.mlpgVar assert os.path.isdir(datamv), "Cannot find mlpgVar = %s for MLPG" else: datamv = 'NONE' if hasattr(cfg, 'vu_threshold'): vuThres = cfg.vu_threshold display.self_print("Use vu_threshold %f" % (vuThres), 'highlight') else: vuThres = 0.5 exe_cmd("ls %s/*.htk > %s/gen.scp" % (testDataDir, testDataDir), cfg.debug) wavCmd = 'python %s %s %f %s %d %s %d %s %d %s %s %s %s %f' \ % (pythonScript, testDataDir, cfg.wavPostFilter, testDataDir, mlpgFlag[0], testDataDir, mlpgFlag[1], testDataDir, mlpgFlag[2], "%s/gen.scp" % (testDataDir), cfg.wavformGenerator, datamv, cfg.path_pyTools_scripts, vuThres) exe_cmd(wavCmd, cfg.debug) if cfg.lf0UV: uvScript = os.path.join(cfg.path_pyTools_scripts, 'dataGen/useUVonLf0.py') if cfg.lf0UVExternalDir is None: uvCmd = 'python %s %s %s %s %s %s' % (uvScript, testDataDir, vuThres, cfg.lf0ext, cfg.vuvext, testDataDir) else: uvCmd = 'python %s %s %s %s %s %s' % ( uvScript, testDataDir, cfg.lf0UVExternalThre, cfg.lf0ext, cfg.lf0UVExternalExt, cfg.lf0UVExternalDir) exe_cmd(uvCmd, cfg.debug) if cfg.lf02f0: f0Script = os.path.join(cfg.path_pyTools, 'speechTools/f0convert.py') f0Cmd = 'python %s %s %s %s' % (f0Script, testDataDir, cfg.lf0ext, cfg.f0ext) exe_cmd(f0Cmd, cfg.debug)
f0Script = os.path.join(cfg.path_pyTools, 'speechTools/f0convert.py') f0Cmd = 'python %s %s %s %s' % (f0Script, testDataDir, cfg.lf0ext, cfg.f0ext) exe_cmd(f0Cmd, cfg.debug) if __name__ == "__main__": # prepare test data if cfg.step03: display.self_print_with_date('Step3. Generating from networks', 'h') if cfg.step03Prepare_DATA is True: testDataDirs = prepareData(cfg.nnDataDirNameTest) else: testDataDirs = [] display.self_print('Skip packaing data', 'highlight') # for each test data dir for testDataDir in testDataDirs: display.self_print(testDataDir, 'highlight') if cfg.step03NNGen is True: # generate output data (HTK format) from CURRENNT genSynCfg(testDataDir) # extract the output features from the HTK genreated by CURRENNT genSplit(cfg.outputDir) else: display.self_print('Skip generating output from network', 'highlight') if cfg.step03WaveFormGen is True: # generate waveform
def prepareData(): """ prepreData: 1. create the file list 2. create the symbolic link to the feature data 3. create the index file (used by CURRENNT) 4. create package data of index file (data.nc) 5. calculate the mean and std for a specific data set """ # create directories dataDir = cfg.nnDataDirName try: os.mkdir(dataDir) except OSError: pass dataListPath = dataDir + os.path.sep + 'lists' try: os.mkdir(dataListPath) except OSError: pass dataRawDir = dataDir + os.path.sep + cfg.idxDirName try: os.mkdir(dataRawDir) except OSError: pass # decide whether create the symbolic link to each file if len(cfg.inputDirs) == 1 and len(cfg.outputDirs) == 1: # no validation set flagFileUseSymbolLink = False elif listSameContent(cfg.inputDirs) and listSameContent( cfg.outputDirs) and listSameContent( cfg.inputDirs[0]) and listSameContent(cfg.outputDirs[0]): # all data have been in the same directory flagFileUseSymbolLink = False else: flagFileUseSymbolLink = True #dataLinkDir = dataDir + os.path.sep + cfg.linkDirname dataLinkDirInput = dataDir + os.path.sep + cfg.linkDirname_input dataLinkDirOutput = dataDir + os.path.sep + cfg.linkDirname_output # prepare for data link if flagFileUseSymbolLink: try: os.mkdir(dataLinkDirInput) os.mkdir(dataLinkDirOutput) except OSError: pass else: if os.path.islink(dataLinkDirInput): os.system("rm %s" % (dataLinkDirInput)) if os.path.islink(dataLinkDirOutput): os.system("rm %s" % (dataLinkDirOutput)) os.system("ln -s %s %s" % (cfg.inputDirs[0][0], dataLinkDirInput)) os.system("ln -s %s %s" % (cfg.outputDirs[0][0], dataLinkDirOutput)) # create file list for dataList, inputDirSet, outputDirSet, dataPart in zip( cfg.dataLists, cfg.inputDirs, cfg.outputDirs, cfg.dataDivision): display.self_print('Processing ' + dataPart + ' data', 'highlight') if dataList is None: # get the cross-set of file list listInput = readwrite.list_file_name_in_dir(inputDirSet[0]) listOutput = readwrite.list_file_name_in_dir(outputDirSet[0]) fileList = listInput if inputDirSet: for inputDir in inputDirSet: listInput2 = readwrite.list_file_name_in_dir(inputDir) fileList, diffSet = crossSet(fileList, listInput2) tmpName = os.path.join( dataListPath, dataPart + os.path.basename(inputDir) + '.dif.lst') readwrite.write_txt_list(diffSet, tmpName) if outputDirSet: for outputDir in outputDirSet: listOutput2 = readwrite.list_file_name_in_dir(outputDir) fileList, diffSet = crossSet(fileList, listOutput2) tmpName = os.path.join( dataListPath, dataPart + os.path.basename(outputDir) + '.dif.lst') readwrite.write_txt_list(diffSet, tmpName) # writing the list of file name random.shuffle(fileList) fileListFilePath = dataListPath + os.path.sep + dataPart + '.lst' readwrite.write_txt_list(fileList, fileListFilePath) else: fileListFilePath = dataListPath + os.path.sep + dataPart + '.lst' os.system("cp %s %s" % (dataList, fileListFilePath)) fileList = readwrite.read_txt_list(fileListFilePath) # before start, take a simple test on the configuration of feature dimension frameNum = None for inputDir, featDim, featName in zip(inputDirSet, cfg.inputDim, cfg.inputExt): inputFile = os.path.join(inputDir, fileList[0]) + '.' + featName if os.path.isfile(inputFile): tmpframeNum = readwrite.read_raw_mat(inputFile, featDim).shape[0] if frameNum is None: frameNum = tmpframeNum elif np.abs(frameNum - tmpframeNum) * 1.0 / frameNum > 0.1: display.self_print("Large mismatch of frame numbers %s" % (fileList[0])) display.self_print( "Please check whether inputDim are correct", 'error') display.self_print("Or check input features are corrupted", 'error') raise Exception("Error: mismatch of frame numbers") for outputDir, featDim, featName in zip(outputDirSet, cfg.outputDim, cfg.outputExt): outputFile = os.path.join(outputDir, fileList[0]) + '.' + featName if os.path.isfile(outputFile): tmpframeNum = readwrite.read_raw_mat(outputFile, featDim).shape[0] if np.abs(frameNum - tmpframeNum) * 1.0 / frameNum > 0.1: display.self_print("Large mismatch of frame numbers %s" % (fileList[0])) display.self_print( "Please check whether inputDim are correct", 'error') display.self_print("Or check input features are corrupted", 'error') raise Exception("Error: mismatch of frame numbers") # create file directories dataSaveDir = dataDir + os.path.sep + dataPart try: os.mkdir(dataSaveDir) except OSError: pass inputScpList = [] outputScpList = [] # create the fileName + fileExt lists # create symbolic link for inputDir, featDim, featName in zip(inputDirSet, cfg.inputDim, cfg.inputExt): tmpFileScp = dataSaveDir + os.path.sep + featName + '.scp' inputScpList.append(tmpFileScp) filePtr = open(tmpFileScp, 'w') for fileName in fileList: # write full path to the feature filePtr.write('%s%s%s.%s\n' % (inputDir, os.path.sep, fileName, featName)) if cfg.step01Prepare_LINK is True and flagFileUseSymbolLink: os.system("ln -f -s %s%s%s.%s %s%s%s.%s" % \ (inputDir, os.path.sep, fileName, featName, dataLinkDirInput, os.path.sep, fileName, featName)) filePtr.close() for outputDir, featDim, featName in zip(outputDirSet, cfg.outputDim, cfg.outputExt): tmpFileScp = dataSaveDir + os.path.sep + featName + '.scp' outputScpList.append(tmpFileScp) filePtr = open(tmpFileScp, 'w') for fileName in fileList: filePtr.write('%s%s%s.%s\n' % (outputDir, os.path.sep, fileName, featName)) if cfg.step01Prepare_LINK is True and flagFileUseSymbolLink: os.system("ln -f -s %s%s%s.%s %s%s%s.%s" % \ (outputDir, os.path.sep, fileName, featName, dataLinkDirOutput, os.path.sep, fileName, featName)) filePtr.close() # create index file list filePtr = open(dataSaveDir + os.path.sep + cfg.idxFileName + '.scp', 'w') for fileName in fileList: filePtr.write('%s%s%s.%s\n' % (dataRawDir, os.path.sep, fileName, cfg.idxFileName)) filePtr.close() # create index files if cfg.step01Prepare_IDX is True or cfg.step01Prepare_PACK is True: # create the lab index lists cmd = 'python %s/dataPrepare/getLabIdx5ms.py' % ( cfg.path_pyTools_scripts) cmd = '%s %s %s %s %s %s %s' % ( cmd, inputDirSet[0], cfg.inputExt[0], cfg.inputDim[0], dataRawDir, cfg.idxFileName, fileListFilePath) display.self_print('Creating time index files', 'highlight') exe_cmd(cmd, cfg.debug) else: display.self_print('skip creating time index', 'highlight') # package the data if cfg.step01Prepare_IDX is True or cfg.step01Prepare_PACK is True: # write data_config.cfg writeDataConfig(dataSaveDir + os.path.sep + 'data_config.py', cfg.idxFileName + '.scp', cfg.fileNumInEachNCPack) # pack data packDataCmd = 'sh %s/sub_05_package_datanc.sh %s %s' % ( cfg.path_scripts, dataSaveDir, cfg.path_pyTools_scripts) display.self_print('Packing data', 'highlight') exe_cmd(packDataCmd, cfg.debug) else: display.self_print('skip packing data', 'highlight') # create file list for inputDirSet, outputDirSet, dataPart in zip(cfg.inputDirs, cfg.outputDirs, cfg.dataDivision): dataSaveDir = dataDir + os.path.sep + dataPart inputScpList = [] outputScpList = [] for inputDir, featDim, featName in zip(inputDirSet, cfg.inputDim, cfg.inputExt): inputScpList.append(dataSaveDir + os.path.sep + featName + '.scp') for outputDir, featDim, featName in zip(outputDirSet, cfg.outputDim, cfg.outputExt): outputScpList.append(dataSaveDir + os.path.sep + featName + '.scp') # calculate mean and std if dataPart == cfg.computMeanStdOn and cfg.step01Prepare_MV is True: display.self_print('Calculating mean and std', 'highlight') meanStdTool.meanStdNormMask( inputScpList, cfg.inputDim, cfg.inputNormMask, dataSaveDir + os.path.sep + cfg.nnDataInputMV) display.self_print( "\nSave input mean-std as %s" % (os.path.join(dataSaveDir, cfg.nnDataInputMV)), 'highlight') meanStdTool.meanStdNormMask( outputScpList, cfg.outputDim, cfg.outputNormMask, dataSaveDir + os.path.sep + cfg.nnDataOutputMV) display.self_print( "\nSave output mean-std as %s" % (os.path.join(dataSaveDir, cfg.nnDataOutputMV)), 'highlight') else: display.self_print('skip calculating mean and std', 'highlight')
def trainCURRENNT(model_dir): os.chdir(model_dir) display.self_print('******** train config ******\n', 'highlight') os.system("cat ./%s" % (cfg.nnModelCfgname)) display.self_print('****************************\n', 'highlight') runCmd = '%s --options_file %s --verbose 1' % (cfg.path_currennt, cfg.nnModelCfgname) runCmd = runCmd + ' >%s 2>%s' % (cfg.tmp_network_trn_log, cfg.tmp_network_trn_err) display.self_print("GPU job submitted. Please wait until terminated.", 'ok') display.self_print("Please open another terminal to check nvidia-smi", 'ok') display.self_print( "Also check %s" % (os.path.join(model_dir, cfg.tmp_network_trn_log)), 'ok') display.self_print( "Also check %s" % (os.path.join(model_dir, cfg.tmp_network_trn_err)), 'ok') exe_cmd(runCmd, cfg.debug) display.self_print_with_date( 'Processed terminated. Please check %s %s' % (os.path.join(model_dir, cfg.tmp_network_trn_log), os.path.join(model_dir, cfg.tmp_network_trn_err)), 'ok')
def exe_cmd(cmd, debug=False): display.self_print("Execute command:", 'ok') display.self_print(cmd + '\n', 'highlight') if not debug: try: subprocess.check_call(cmd, shell=True) display.self_print("Command is successfully executed:\n%s\n\n" % (cmd), 'ok') except subprocess.CalledProcessError as e: display.self_print("Failed to run:" + cmd, 'error') display.self_print("Please check the printed error message", 'error') display.self_print("Process terminated with %s" % (e.returncode), 'error') sys.exit(1)
display.self_print_with_date('step1.1 copying data lists', 'm') tmp_acous_path = ','.join(cfg.path_acous_feats) tmp_feat_ext = ','.join(cfg.ext_acous_feats) tmp_feat_dim = '_'.join([str(x) for x in cfg.dim_acous_feats]) cmd = 'python %s' % ( cfg.path_scripts) + os.path.sep + 'sub_01_check_list.py' cmd = cmd + ' %s,%s' % (tmp_acous_path, cfg.path_waveform) cmd = cmd + ' %s,.wav' % (tmp_feat_ext) cmd = cmd + ' %s_1' % (tmp_feat_dim) cmd = cmd + ' %s' % (tmp_data_scp_dir) cmd = cmd + ' %s' % (cfg.trn_list) cmd = cmd + ' %s' % (cfg.val_list) exe_cmd(cmd, cfg.debug) else: display.self_print( 'cannot find %s %s' % (cfg.trn_list, cfg.val_list), 'error') quit() else: display.self_print_with_date('step1.1 generating data lists', 'm') tmp_acous_path = ','.join(cfg.path_acous_feats) tmp_feat_ext = ','.join(cfg.ext_acous_feats) tmp_feat_dim = '_'.join([str(x) for x in cfg.dim_acous_feats]) cmd = 'python %s' % ( cfg.path_scripts) + os.path.sep + 'sub_01_prepare_list.py' cmd = cmd + ' %s,%s' % (tmp_acous_path, cfg.path_waveform) cmd = cmd + ' %s,.wav' % (tmp_feat_ext) cmd = cmd + ' %s_1' % (tmp_feat_dim) cmd = cmd + ' %s' % (tmp_data_scp_dir) cmd = cmd + ' %f' % (cfg.train_utts) exe_cmd(cmd, cfg.debug)
def PrepareScp(InScpFile, OutScpFile, inDim, outDim, allScp, datadir, txtScp, txtDim): """ Count the number of frames of each input and output file. """ assert len(InScpFile)==len(inDim), \ "Unequal length of input scp and in dim" assert len(OutScpFile)==0 or len(OutScpFile)==len(outDim), \ "Unequal length of output scp and in dim" numSeque = 0 # number of sequence numFrame = 0 # number of total frame inPatDim = 0 # dimension of input ouPatDim = 0 # dimension of output maxSeqLe = 0 # maximum length of name sequence fileLabBuffer = ['',] fileInBuffer = ['',] fileOutBuffer = ['',] seqLenBuffer = [0,] # Pre-process the input file # check the duration of each input file # keep the shortest duration of input files for one entry display.self_print("\nNote: Different feature files of one utterance may be different in length.", 'warning') display.self_print("Trim value shows how many data are discarded in order to match the shortest file.", 'warning') display.self_print("Large Trim value may indicate data length mismatch.", 'warning') display.self_print("Please check configuration carefully if it happens!\n", 'warning') print("Processing the input file") for scpFile, dim, scpIndex in zip(InScpFile, inDim, range(len(InScpFile))): fPtr = open(scpFile,'r') fileCtr = 0 while 1: line = fPtr.readline() if len(line.strip())==0: break if scpIndex==0: numSeque = numSeque + 1 fileline = line.strip() # only check for relative path #if not os.path.isfile(fileline): # fileline = datadir + os.path.sep + fileline assert os.path.isfile(fileline), "Can't find file"+fileline if scpIndex==0: # loading lab file fileLabBuffer.append(fileline) if len(fileline) > maxSeqLe: maxSeqLe = len(fileline) fileInBuffer.append(fileline) # check the time step of file if scpIndex>0 and fileline==fileLabBuffer[fileCtr+1]: pass # if this file is the same as the input lab file # pass it (for the case when several columns of the # input lab will be extracted based on inputMask) else: try: tempFrame = funcs.Bytes(fileline, dim)/np.dtype(dataType).itemsize except TypeError: tempFrame = funcs.Bytes(fileline.encode(), dim)/np.dtype(dataType).itemsize if scpIndex==0: numFrame = numFrame + tempFrame seqLenBuffer.append(tempFrame) else: if np.abs(seqLenBuffer[fileCtr+1] - tempFrame) / float(seqLenBuffer[fileCtr+1]) > 0.1: if seqLenBuffer[fileCtr+1] > tempFrame: addiFrame = seqLenBuffer[fileCtr+1]-tempFrame display.self_print("%s has %d data, less than %d " % (fileline, tempFrame, seqLenBuffer[fileCtr+1]), 'warning') display.self_print("Other files will be trimmed %d data to fit %s" % (addiFrame,fileline), 'warning') elif seqLenBuffer[fileCtr+1] < tempFrame: addiFrame = -1*seqLenBuffer[fileCtr+1]+tempFrame display.self_print("Trim %d from %s" % (addiFrame,fileline), 'warning') if seqLenBuffer[fileCtr+1]>tempFrame: seqLenBuffer[fileCtr+1]=tempFrame fileCtr = fileCtr + 1 print("Input %d %d\r" % (scpIndex, fileCtr), end=' ') sys.stdout.flush() #sys.stdout.write("\rInput:"+str(fileCtr)) print("") if fPtr.tell() == os.fstat(fPtr.fileno()).st_size: flagTer = True # all files have been processed fPtr.close() assert len(fileInBuffer)-1==(len(inDim)*numSeque), "Unequal file input numbers" # Pre-process the output file # check the duration of output file print("Processing the output file") if len(OutScpFile)==0: # void output files for dim in outDim: for x in range(numSeque): fileOutBuffer.append('#') else: # multiple output files for scpFile, dim, scpIndex in zip(OutScpFile, outDim, range(len(OutScpFile))): fPtr = open(scpFile,'r') fileCtr = 0 while 1: line = fPtr.readline() fileline = line.strip() if len(line.strip())==0: break # only check for relative path #if not os.path.isfile(fileline): # fileline = datadir + os.path.sep + fileline assert os.path.isfile(fileline), "Can't find file"+fileline #print line fileOutBuffer.append(fileline) try: tempFrame = funcs.Bytes(fileline, dim)/np.dtype(dataType).itemsize except TypeError: tempFrame = funcs.Bytes(fileline.encode(), dim)/np.dtype(dataType).itemsize if np.abs(seqLenBuffer[fileCtr+1] - tempFrame) / float(seqLenBuffer[fileCtr+1]) > 0.1: if seqLenBuffer[fileCtr+1] > tempFrame: addiFrame = seqLenBuffer[fileCtr+1]-tempFrame display.self_print("%s has %d data, less than %d " % (fileline, tempFrame, seqLenBuffer[fileCtr+1]), 'warning') display.self_print("Other files will be trimmed %d data to fit %s" % (addiFrame,fileline), 'warning') elif seqLenBuffer[fileCtr+1] < tempFrame: addiFrame = -1*seqLenBuffer[fileCtr+1]+tempFrame display.self_print("Trim %d from %s" % (addiFrame,fileline), 'warning') if seqLenBuffer[fileCtr+1]>tempFrame: seqLenBuffer[fileCtr+1]=tempFrame fileCtr = fileCtr + 1 print("Output %d %d\r" % (scpIndex, fileCtr), end=' ') sys.stdout.flush() print("") fPtr.close() assert len(fileOutBuffer)-1==(len(outDim)*numSeque), "Unequal file output numbers" # if text scp exists, check if len(txtScp) > 0: textBuffer = [] with open(txtScp, 'r') as filePtr: for i, fileline in enumerate(filePtr): filename = fileline.rstrip('\n') name1 = os.path.splitext(os.path.basename(filename))[0] name2 = os.path.splitext(os.path.basename(fileLabBuffer[i+1]))[0] assert name1==name2, "textScpFile unmatch %s, %s, %d-th line" % (name1, name2, i) textBuffer.append(filename) assert len(textBuffer)==(len(fileLabBuffer)-1), "textScpFile, unmatched length" else: textBuffer = [] # Write the scp for packaging data scpFileCtr = 1 fileCtr = 0 fPtr = open(allScp+str(scpFileCtr), mode='w') nameBuf = ['', ] numFrameBuf = [0, ] numUttBuf = [0, ] frameBuf = 0 assert numSeque, "Found no utterance to pack" for i in range(numSeque): outputline = "%s %d %d %d" % \ ( os.path.splitext(os.path.basename(fileLabBuffer[i+1]))[0], len(inDim), len(outDim), seqLenBuffer[i+1] ) frameBuf = frameBuf + seqLenBuffer[i+1] for j in range(len(inDim)): index = (j)*numSeque+i+1 outputline = outputline + " %d %s" % (inDim[j], fileInBuffer[index]) for j in range(len(outDim)): index = (j)*numSeque+i+1 outputline = outputline + " %d %s" % (outDim[j], fileOutBuffer[index]) fileCtr = fileCtr + 1 fPtr.write(outputline) if len(textBuffer) > 0: assert os.path.isfile(textBuffer[i]), "Can't find %s" % (textBuffer[i]) try: itemNum = funcs.Bytes(textBuffer[i], txtDim)/np.dtype(dataType).itemsize except TypeError: itemNum = funcs.Bytes(textBuffer[i].encode(), txtDim)/np.dtype(dataType).itemsize temp = " %d %d %s" % (itemNum, txtDim, textBuffer[i]) fPtr.write(temp) fPtr.write("\n") flagLock = False if fileCtr >= flushThreshold: numUttBuf.append(fileCtr) numFrameBuf.append(frameBuf) fileCtr = 0 frameBuf = 0 fPtr.close() nameBuf.append(allScp+str(scpFileCtr)) scpFileCtr = scpFileCtr + 1 fPtr = open(allScp+str(scpFileCtr), mode='w') flagLock = True if flagLock==False: nameBuf.append(allScp+str(scpFileCtr)) numUttBuf.append(fileCtr) numFrameBuf.append(frameBuf) fPtr.close() # Return assert sum(numUttBuf)==numSeque, "Unequal utterance number" return numSeque, numFrame, maxSeqLe, numFrameBuf[1:], numUttBuf[1:], nameBuf[1:]
try: acous_dim = int(sys.argv[4]) except ValueError: raise Exception("Error: input acoustic dim incorrect %s, sub_08" % (sys.argv[4])) try: waveform_quantization_bits = int(sys.argv[5]) except ValueError: raise Exception( "Error: waveform quantization config incorrect %s, sub_08" % (sys.argv[5])) if not os.path.isfile(network_path): display.self_print('Error: not found %s' % (network_path), 'error') quit() network_path_tmp = network_path + '.tmp' with open(network_path, 'r') as file_ptr: network_data = json.load(file_ptr) # change if necessary if 'layers' in network_data: for layer_idx in range(len(network_data['layers'])): # change time resolution for condition layer if 'resolution' in network_data['layers'][layer_idx]: if network_data['layers'][layer_idx]['resolution'] > 1: network_data['layers'][layer_idx][ 'resolution'] = resolution
def createFileLst(dataDirs, dataExts, dataDim, dataListDirs, trainortest, trainSetRatio=0.8, random_seed=12345): """ create data lists output *.scp will be in dataListDirs """ dataDirs = dataDirs.split(',') dataExts = dataExts.split(',') dataDims = [int(x) for x in dataDim.split('_')] assert len(dataDirs) == len( dataExts), 'Error: sub_1_prepare_list.py dataDirs and dataExts wrong' # get the cross-set of file lists dataList = lstdirNoExt(dataDirs[0], dataExts[0]) for dataDir, dataExt in zip(dataDirs[1:], dataExts[1:]): listTmp = lstdirNoExt(dataDir, dataExt) dataList = crossSet(dataList, listTmp) # check if file exists if len(dataList) < 1: display.self_print("Error: fail to generate file list. Please check:", 'error') display.self_print( "path_acous_feats, ext_acous_feats, path_waveform in config.py;", 'error') display.self_print("Please also check the names of input data files.", 'error') raise Exception("Error: fail to generate file list.") # randomize the data file list # sort at first, dataList.sort() random.seed(random_seed) random_shuffle(dataList) # before start, take a simple test on the configuration of feature dimension frameNum = None for inputDir, featDim, featName in zip(dataDirs[0:-1], dataDims[0:-1], dataExts[0:-1]): inputFile = os.path.join(inputDir, dataList[0]) + '.' + featName.lstrip('.') if os.path.isfile(inputFile): tmpframeNum = readwrite.read_raw_mat(inputFile, featDim).shape[0] if frameNum is None or frameNum < tmpframeNum: frameNum = tmpframeNum for inputDir, featDim, featName in zip(dataDirs[0:-1], dataDims[0:-1], dataExts[0:-1]): inputFile = os.path.join(inputDir, dataList[0]) + '.' + featName.lstrip('.') if os.path.isfile(inputFile): tmpframeNum = readwrite.read_raw_mat(inputFile, featDim).shape[0] if np.abs(frameNum - tmpframeNum) * 1.0 / frameNum > 0.1: if featDim == readwrite.read_raw_mat(inputFile, 1).shape[0]: pass else: display.self_print("Large mismatch of frame numbers %s" % (inputFile)) display.self_print( "Please check whether inputDim are correct", 'error') display.self_print("Or check input features are corrupted", 'error') raise Exception("Error: mismatch of frame numbers") display.self_print('Generating data lists in to %s' % (dataListDirs), 'highlight') if trainortest == 'test': # generating for test set display.self_print('\ttest size: %d' % (len(dataList)), 'highlight') testFileOut = dataListDirs + os.path.sep + 'test.lst' testFilePtr = open(testFileOut, 'w') for fileName in dataList: testFilePtr.write('%s\n' % (fileName)) testFilePtr.close() else: # determine the train and validatition set if necessary if trainSetRatio is not None and trainSetRatio > 0.0: if trainSetRatio < 1.0: # a ratio trainSetDivide = int(np.round(len(dataList) * trainSetRatio)) elif trainSetRatio < len(dataList): # absolute value of the number of training utterances trainSetDivide = int(trainSetRatio) else: # a default ratio 0.8 display.self_print( 'Warning: train_utts = 0.8 is used to divide train/val', 'warning') trainSetDivide = int(np.round(len(dataList) * 0.8)) trainSet = dataList[0:trainSetDivide] valSet = dataList[trainSetDivide:] if len(valSet) > len(trainSet): display.self_print( "Warning: validation set is larger than training set", 'warning') display.self_print( "It's better to change train_utts in config.py", 'warning') trainFileOut = dataListDirs + os.path.sep + 'train.lst' trainFilePtr = open(trainFileOut, 'w') for fileName in trainSet: trainFilePtr.write('%s\n' % (fileName)) trainFilePtr.close() if len(valSet): valFileOut = dataListDirs + os.path.sep + 'val.lst' valFilePtr = open(valFileOut, 'w') for fileName in valSet: valFilePtr.write('%s\n' % (fileName)) valFilePtr.close() display.self_print( '\ttrain/val sizes: %d, %d' % (len(trainSet), len(valSet)), 'warning') else: display.self_print('\ttrain/val sizes: %d, 0' % (len(dataList)), 'warning') trainFileOut = dataListDirs + os.path.sep + 'train.lst' trainFilePtr = open(trainFileOut, 'w') for fileName in dataList: trainFilePtr.write('%s\n' % (fileName)) trainFilePtr.close() # done return