#path to the mappings folder we want to write the mapping files to #this can be temp and then removed. #mapping_dir = os.path.dirname(os.path.realpath(__file__)) +\ # "/../data/raw_data/swbd_swda2MS_mapping_temp" #make that dir if it doesn't exist #if not os.path.isdir(mapping_dir): # os.mkdir(mapping_dir) ranges = sorted([line.strip("\n") for line in open(range_file)]) print len(ranges), "files to process" dialogue_speakers = [] #for disf_file in disfluency_files: IDs, mappings, utts, pos_tags, labels = \ load_data_from_disfluency_corpus_file(disf_file) # print labels dialogue_speakers.extend(sort_into_dialogue_speakers(IDs, mappings, utts, pos_tags, labels, convert_to_dnn_tags=False)) print len(dialogue_speakers), "dialogue speakers" #The main loop- has every word in both formats, needs to map from MS file # timings to the SWDA ones #Given the original SWDA transcripts are IN the MSaligned files, it's safer #to map to them, #then use the delete/insert/sub operations in the files to get the pointer. #So the mapping is MSnew -> MSSWDAold -> SWDAold, #where the last mapping is done through min. edit distance string alignment tests = ["2549B"] print "Creating word timing aligned corpus..." for dialogue_triple in sorted(dialogue_speakers, key=lambda x: x[0]):
args = parser.parse_args() word_dict = load_word_rep(args.word_rep_file) pos_dict = load_word_rep(args.pos_rep_file) label_dict = load_tags(args.label_rep_file) print 'tags', args.label_rep_file use_timing_data = False if "timings" in args.corpus_file: dialogues = load_data_from_corpus_file(args.corpus_file) use_timing_data = True else: print "no timings" IDs, timings, seq, pos_seq, targets = \ load_data_from_disfluency_corpus_file( args.corpus_file, convert_to_dnn_format=True) raw_dialogues = sort_into_dialogue_speakers(IDs, timings, seq, pos_seq, targets, add_uttseg="uttseg" in args.label_rep_file, add_dialogue_acts="dact" in args.label_rep_file) dialogues = [] for conv_no, indices, lex_data, pos_data, labels in raw_dialogues: frames = indices dialogues.append( (conv_no, (frames, lex_data, pos_data, indices, labels)))
] heldout_file = open(asr_dir + "SWDisfHeldout_increco.text", "w") test_file = open(asr_dir + "SWDisfTest_increco.text", "w") leftout_ranges = [line.strip("\n") for line in open(asr_dir + "leftout_asr")] #split the big disfluency marked -up files into individual file tuples #it is possible to do the matching on the utterance level as they should have consistent mark-up between the two disf_dir = "../data/disfluency_detection/switchboard" disfluency_files = [ disf_dir + "/swbd_heldout_partial_data.csv", disf_dir + "/swbd_test_partial_data.csv" ] dialogue_speakers = [] for key, disf_file in zip(["heldout", "test"], disfluency_files): IDs, mappings, utts, pos_tags, labels = load_data_from_disfluency_corpus_file( disf_file) dialogue_speakers.extend( sort_into_dialogue_speakers(IDs, mappings, utts, pos_tags, labels)) word_pos_data = {} #map from the file name to the data for data in dialogue_speakers: dialogue, a, b, c, d = data word_pos_data[dialogue] = (a, b, c, d) for key in sorted(word_pos_data.keys()): print key print word_pos_data[key][1][:100] break #quit() average_wer = [] leftout = []