Example #1
0
def score_folder_SD(fileList, fileDict, diarcollar, write_msg, out_path):
    sctk_path = util.get_fs02sctk_path()
    py_val_path = sctk_path + 'scutils/dscore/validate_rttm.py'
    py_score_path = sctk_path + 'scutils/dscore/score.py'
    log_write_path = util.get_logs_path() + util.get_bname(out_path) + '.log'
    derDict = {}
    log_list = []
    unsuc_files = []
    for fn in fileList:
        ref_rttm = fileDict['ref'][fn]
        hyp_rttm = fileDict['hyp'][fn]
        ref_uem = ref_rttm.replace('/RTTM/', '/UEM/').replace('.rttm', '.uem')
        der, curr_log = score_file_SAD(py_val_path, py_score_path, fn,
                                       ref_rttm, hyp_rttm, ref_uem, diarcollar)
        if der != 'NaN':
            derDict[fn] = der
        else:
            unsuc_files.append(fn)
        log_list += curr_log
    with open(log_write_path, 'w') as file:
        file.write('\n'.join(log_list))
        print('\n\nLog File for SD Task - DER evaluation',
              'written to path:\n\t', log_write_path, '\n\n')

    write_msg.append('\n\n')
    write_msg.append('Number of Files to be Evaluated:' + str(len(fileList)) +
                     '\n\n')
    write_msg.append('Number of Files Successfully Evaluated:' +
                     str(len(derDict)) + '\n\n')
    if len(unsuc_files) > 0:
        write_msg.append('Files that could not be evaluated:\n\t' +
                         ' '.join(unsuc_files))
    return derDict, write_msg
def score_file_SAD(gt_fp, hyp_fp, sadcollar):

    temp_out_fp = util.get_temp_path() + util.getfName(gt_fp) + '.out'
    util.remove_file(temp_out_fp)
    scprl_fath = util.get_fs02sctk_path() + 'scutils/scoreFile_SAD.pl'
    term_cmd = [
        'perl', scprl_fath, '-r', gt_fp, '-h', hyp_fp, '-s', '2', '-e', '3',
        '-g', '4', '-t', '5', '-f', '6', '-u', '7', '-o', temp_out_fp
    ]

    termOut = util.get_term_output(term_cmd)

    termOut = termOut.strip().replace('\n\n', '\n').split('\n')
    termOut = [x.strip() for x in termOut][2:]
    util.remove_file(temp_out_fp)

    collarDict = {'0.0': 3, '0.25': 7, '0.5': 11, '1.0': 15, '2.0': 19}
    dcf_desired = termOut[int(collarDict[sadcollar])].split()[2].strip()
    collarDict_fp = {'0.0': 2, '0.25': 6, '0.5': 10, '1.0': 14, '2.0': 18}
    pfp_desired = termOut[int(collarDict_fp[sadcollar])].split()[2].strip()
    collarDict_fn = {'0.0': 1, '0.25': 5, '0.5': 9, '1.0': 13, '2.0': 17}
    pfn_desire = termOut[int(collarDict_fn[sadcollar])].split()[2].strip()
    if not util.is_number(dcf_desired):
        pfp_desired = 'NaN'
    if not util.is_number(pfp_desired):
        pfn_desire = 'NaN'
    if not util.is_number(pfn_desire):
        pfn_desire = 'NaN'
    return dcf_desired, pfp_desired, pfn_desire
def parse_arguments():

    sctk_path = util.get_fs02sctk_path()
    def_out_path = util.get_results_path(
    ) + 'SID_TopN_Result_' + util.getDateTimeStrStamp() + '.txt'


    desc='Wrapper File to generate Top-N Accuracy Scores for FS02 Challenge SID Task.' +\
        'For more information regarding scoring input and hypothesis files, '+\
        'refer below arguments description.'

    ref_mp = 'egs/ref_gt/SID/FS01_SID_uttID2spkID_Dev.txt'
    hyp_mp = 'egs/sys_results/SID/FS01_SID_uttID2spkID_Dev.txt'

    ref_def = sctk_path + ref_mp
    hyp_def = sctk_path + hyp_mp


    ref_str = 'Reference (ground truth) File Path. '+\
        'This file must be the SID ground truth file. '+\
        'Please refer ./'+ref_mp+' file for example.'
    hyp_str = 'Hypothesis (system output) File Path. '+\
        'This directory must include only SAD system output files. '+\
        'Please refer ./'+hyp_mp+' file for example and file format.'
    out_str = 'Output (overall system score) File Path. '+\
        'Default: Result file will stored in '+util.get_results_path()+' directory. '+\
        'Additional log files will be stored in '+util.get_logs_path()
    clr_str = 'Desired Top-N Accuracy for SID evaluation. '+\
        'Default: Top-5 Accuracy.'

    parser = argparse.ArgumentParser(description=desc)
    parser.add_argument('-ref',
                        '--ref',
                        type=str,
                        default=ref_def,
                        help=ref_str)
    parser.add_argument('-hyp',
                        '--hyp',
                        type=str,
                        default=hyp_def,
                        help=hyp_str)
    parser.add_argument('-out',
                        '--out',
                        type=str,
                        default=def_out_path,
                        help=out_str)
    parser.add_argument('-topN', '--topN', type=int, default=5, help=clr_str)

    args = parser.parse_args()
    ref_path = util.processInpPath(args.ref, inpType='file', checkExists=True)
    hyp_path = util.processInpPath(args.hyp, inpType='file', checkExists=True)
    out_path = util.processInpPath(args.out, inpType='file')
    max_TopN = validate_hyp_file(hyp_path)
    topN_num = proc_topN_inp(args.topN, max_TopN)

    return ref_path, hyp_path, out_path, topN_num
Example #4
0
def parse_arguments():

    sctk_path = util.get_fs02sctk_path()
    def_out_path = util.get_results_path(
    ) + 'SD_DER_Result_' + util.getDateTimeStrStamp() + '.txt'
    coll_inps_str = 'Allowed Inputs: 0, 0.25, 0.5, 1, 2'


    desc='Wrapper File to generate DER Scores for FS02 Challenge SD (track1 and track2) Tasks.' +\
        'Scoring mechanism for both tracks will be the same. For more '+\
        'information regarding scoring input and hypothesis files, refer '+\
        'below arguments description. Open-Source Software Credits: '+\
        'This script uses dscore toolkit developed by Neville Ryant for '+\
        'generating DER scores. for more info, refer: (https://github.com/nryant/dscore)'

    ref_mp = 'egs/ref_gt/SD/'
    hyp_mp = 'egs/sys_results/SD/'

    ref_def = sctk_path + ref_mp
    hyp_def = sctk_path + hyp_mp


    ref_str = 'Reference (ground truth) Directory Path. '+\
        'This directory must include only SD ground truth RTTM '+\
        'and UEM folders. Please refer ./'+ref_mp+' directory for examples.'
    hyp_str = 'Hypothesis (system output) Directory Path. '+\
        'This directory must include only diarization system output RTTM files. '+\
        'Please refer ./'+hyp_mp+' directory for examples and file format.'
    out_str = 'Output (per file and overall system score) File Path. '+\
        'Default: Result file will stored in '+util.get_results_path()+' directory. '+\
        'Additional log files if generated will be stored in '+util.get_logs_path()
    clr_str = 'Desired forgiveness Collar for SD evaluation. '+coll_inps_str+\
        ' Default collar length: 0.25 secs.'

    parser = argparse.ArgumentParser(description=desc)
    parser.add_argument('-ref',
                        '--ref',
                        type=str,
                        default=ref_def,
                        help=ref_str)
    parser.add_argument('-hyp',
                        '--hyp',
                        type=str,
                        default=hyp_def,
                        help=hyp_str)
    parser.add_argument('-out',
                        '--out',
                        type=str,
                        default=def_out_path,
                        help=out_str)
    parser.add_argument('-diarcollar',
                        '--diarcollar',
                        type=float,
                        default=0.25,
                        help=clr_str)
    args = parser.parse_args()

    ref_path = proc_sd_ref_files(util.processInpPath(args.ref))
    hyp_path = util.processInpPath(args.hyp)
    out_path = util.processInpPath(args.out, inpType='file')
    diarcollar = proc_sd_collar(args.diarcollar)

    return ref_path, hyp_path, out_path, diarcollar
Example #5
0
def parse_arguments():
    sctk_path = util.get_fs02sctk_path()
    def_out_path = util.get_results_path()+'ASR_WER_Result_'+util.getDateTimeStrStamp()+'.txt'
   
    
    
    desc='Wrapper File to generate WER Scores for FS02 Challenge ASR (track1 and track2) Task.' +\
        'Scoring mechanism for both tracks is the same, but the system output '+\
        'hypothesis file/folder expected from the user will be different. '+\
        '(folder with json files for track-1, and a plain text file for track-2)'+\
        'For more information regarding scoring input and hypothesis files, '+\
        'refer below arguments description. Open-Source Software Credits: '+\
        'This script uses compute-wer tool from the Kaldi Speech Recognition '+\
        'Toolkit. for more info, refer: (http://kaldi-asr.org/doc/tools.html)'
    
    ref_mp = 'egs/ref_gt/ASR/ASR_track'
    hyp_mp = 'egs/sys_results/ASR/ASR_track'
    
    ref_def = sctk_path+ref_mp
    hyp_def = sctk_path+hyp_mp
            
    ref_str = 'Reference (ground truth) Path. '+\
        '(Directory Path for Track-1, and File Path for Track-2) '+\
        'Directory Path for Track-1 must include only ASR ground truth files. '+\
        'For ASR_track1: directory containing json format ground truth files required. '+\
        'Please refer ./'+ref_mp+'1/ directory for examples. '+\
        'For ASR_track2: kaldi "text" file. Refer: { https://kaldi-asr.org/doc/data_prep.html#data_prep_data }. '+\
        '   file contents of File Path for Track-2 must include only FS02_ASR_track2 '+\
        'file-names followed by associated transcripts (like in Kaldi "text" format)'+\
        'Please refer ./'+ref_mp+'2/ directory for examples.'
    hyp_str = 'Hypothesis (system output) Directory/File Path. '+\
        '(Directory Path for Track-1, and File Path for Track-2) '+\
        'Directory Path for Track-1 must include only FS02-ASR system output files. '+\
        'For ASR_track1: directory containing json format system output files required. '+\
        'Please refer ./'+ref_mp+'1/ directory for examples and file format. '+\
        'For ASR_track2: kaldi "text" file. Refer: { https://kaldi-asr.org/doc/data_prep.html#data_prep_data }. '+\
        'file contents of File Path for Track-2 must include only FS02_ASR_track2 '+\
        'file-names followed by associated transcripts (like in Kaldi "text" format)'+\
        'Please refer ./'+ref_mp+'2/ directory for examples and file format.'
    out_str = 'Output (overall system score) File Path. '+\
        'Default: Result file will stored in '+util.get_results_path()+' directory. '+\
        'Additional log files if generated will be stored in '+util.get_logs_path()
    trk_str = 'Track number of the ASR Task to be evaluated. '+\
        'Input Options: (as string) "1" or "2"'
    kld_str = 'base path to the locally installed kaldi directory. '+\
        'e.g. /home/crss/kaldi. This argument is required for'
    
    
    parser = argparse.ArgumentParser(description=desc)
    parser.add_argument('-track', '--track', type=str, default='1', help=trk_str)
    parser.add_argument('-kaldi','--kaldi', type=str, required=True, help=kld_str)
    parser.add_argument('-ref', '--ref', type=str, default=ref_def, help=ref_str)
    parser.add_argument('-hyp', '--hyp', type=str, default=hyp_def, help=hyp_str)
    parser.add_argument('-out', '--out', type=str, default=def_out_path, help=out_str)
     
    args = parser.parse_args()
    
    track_num = proc_track_num(args.track)
    kaldi_path = util.processInpPath(args.kaldi)    
    if args.ref == ref_def or args.hyp == hyp_def:
        ad = str(track_num)+'/'
        t2_name = 'FS01_ASR_track2_transcriptions_Dev'
        print('ref or hyp paths are either not provided, or match default paths.')
        print('Running Script on default example for Track-',str(track_num))
        if track_num == 2:
            args.ref = ref_def+ad+t2_name
            args.hyp = hyp_def+ad+t2_name
        else:
            args.ref = ref_def+ad
            args.hyp = hyp_def+ad
    if track_num == 2:
        ref_path = util.processInpPath(args.ref, inpType='file', checkExists=True)
        hyp_path = util.processInpPath(args.hyp, inpType='file', checkExists=True)
    else:
        ref_path = util.processInpPath(args.ref)
        hyp_path = util.processInpPath(args.hyp)
    out_path = util.processInpPath(args.out, inpType='file')
    
    return ref_path, hyp_path, out_path, track_num, kaldi_path
def parse_arguments():

    sctk_path = util.get_fs02sctk_path()
    def_out_path = util.get_results_path(
    ) + 'SAD_DCF_Result_' + util.getDateTimeStrStamp() + '.txt'
    coll_inps_str = 'Allowed Inputs: 0, 0.25, 0.5, 1, 2'


    desc='Wrapper File to generate DCF Scores for FS02 Challenge SAD Task.' +\
        'For more information regarding scoring input and hypothesis files, '+\
        'refer below arguments description. '+\
        'Open-Source Software Credits: This script uses scoreFile_SAD.pl '+\
        'developed by NIST. for more info, refer: (https://www.nist.gov/'+\
        'itl/iad/mig/nist-open-speech-activity-detection-evaluation)'

    ref_mp = 'egs/ref_gt/SAD/'
    hyp_mp = 'egs/sys_results/SAD/'

    ref_def = sctk_path + ref_mp
    hyp_def = sctk_path + hyp_mp


    ref_str = 'Reference (ground truth) Directory Path. '+\
        'This directory must include only SAD ground truth files. '+\
        'Please refer ./'+ref_mp+' directory for examples.'
    hyp_str = 'Hypothesis (system output) Directory Path. '+\
        'This directory must include only SAD system output files. '+\
        'Please refer ./'+hyp_mp+' directory for examples and file format.'
    out_str = 'Output (per file and overall system score) File Path. '+\
        'Default: Result file will stored in '+util.get_results_path()+' directory. '+\
        'Additional log files if generated will be stored in '+util.get_logs_path()
    clr_str = 'Desired forgiveness Collar for SAD evaluation. '+coll_inps_str+\
        ' Default collar length: 0.5 secs.'

    parser = argparse.ArgumentParser(description=desc)
    parser.add_argument('-ref',
                        '--ref',
                        type=str,
                        default=ref_def,
                        help=ref_str)
    parser.add_argument('-hyp',
                        '--hyp',
                        type=str,
                        default=hyp_def,
                        help=hyp_str)
    parser.add_argument('-out',
                        '--out',
                        type=str,
                        default=def_out_path,
                        help=out_str)
    parser.add_argument('-sadcollar',
                        '--sadcollar',
                        type=float,
                        default=0.5,
                        help=clr_str)
    args = parser.parse_args()

    ref_path = util.processInpPath(args.ref)
    hyp_path = util.processInpPath(args.hyp)
    out_path = util.processInpPath(args.out, inpType='file')
    sadcollar = proc_sad_collar(args.sadcollar)

    return ref_path, hyp_path, out_path, sadcollar