def load_deepfix_ac_data_for_generator(filter_unk=False): vocab = create_deepfix_common_error_vocabulary( begin_tokens=['<BEGIN>', '<INNER_BEGIN>'], end_tokens=['<END>', '<INNER_END>'], unk_token='<UNK>', addition_tokens=['<PAD>']) df = read_deepfix_ac_data() df = convert_deepfix_to_c_code(df) tokenize_fn = tokenize_by_clex_fn() parse_test_param = [vocab, tokenize_fn, True] df_data = parse_test_tokens(df, 'deepfix', *parse_test_param) df_data = list(df_data) if filter_unk: unk_id = vocab.word_to_id(vocab.unk) df_data[0] = df_data[0][df_data[0].map(lambda x: unk_id not in x)] df = df.loc[df_data[0].index.values] deepfix_dict = { 'ac_token_id_list': df_data[0], 'includes': df['includes'], 'distance': df['errorcount'], 'ac_token_name_list': df_data[1] } return deepfix_dict
def read_deepfix_dataset(): tokenize_fn = tokenize_by_clex_fn() vocabulary = create_deepfix_common_error_vocabulary( begin_tokens=['<BEGIN>', '<INNER_BEGIN>'], end_tokens=['<END>', '<INNER_END>'], unk_token='<UNK>', addition_tokens=['<PAD>']) transformer = TransformVocabularyAndSLK(tokenize_fn=tokenize_fn, vocab=vocabulary) train_dataset, valid_dataset, test_dataset, _ = load_deepfix_sample_iterative_dataset( is_debug=False, vocabulary=vocabulary, mask_transformer=transformer, do_flatten=True, use_ast=True, do_multi_step_sample=False, merge_action=False) return train_dataset, valid_dataset, test_dataset
def load_customer_code_data_for_iterate(df): vocab = create_deepfix_common_error_vocabulary( begin_tokens=['<BEGIN>', '<INNER_BEGIN>'], end_tokens=['<END>', '<INNER_END>'], unk_token='<UNK>', addition_tokens=['<PAD>']) # df = read_deepfix_error_data() df = convert_deepfix_to_c_code(df) tokenize_fn = tokenize_by_clex_fn() parse_test_param = [vocab, tokenize_fn, True] df_data = parse_test_tokens(df, 'deepfix', *parse_test_param) df = df.loc[df_data[0].index.values] deepfix_dict = { 'error_token_id_list': df_data[0], 'includes': df['includes'], 'distance': df['errorcount'], 'error_token_name_list': df_data[1], 'id': df['code_id'] } return deepfix_dict
test_dataset = IterateErrorDataSet( df, vocabulary, 'deepfix', transformer_vocab_slk=mask_transformer, do_flatten=do_flatten, use_ast=use_ast, do_multi_step_sample=do_multi_step_sample) info_output = "There are {} parsed data in the deepfix dataset".format( len(test_dataset)) print(info_output) return None, None, test_dataset, None if __name__ == '__main__': vocab = create_deepfix_common_error_vocabulary( begin_tokens=['<BEGIN>', '<INNER_BEGIN>'], end_tokens=['<END>', '<INNER_END>'], unk_token='<UNK>', addition_tokens=['<PAD>']) tokenize_fn = tokenize_by_clex_fn() transformer = TransformVocabularyAndSLK(tokenize_fn=tokenize_fn, vocab=vocab) train_dataset = load_deepfix_ac_code_for_generate_dataset( is_debug=True, vocabulary=vocab, mask_transformer=transformer, do_flatten=True, use_ast=False) print(len(train_dataset))
def load_common_error_data(addition_infomation=False, data_type=None): if data_type == 'deepfix': vocab = create_deepfix_common_error_vocabulary( begin_tokens=['<BEGIN>', '<INNER_BEGIN>'], end_tokens=['<END>', '<INNER_END>'], unk_token='<UNK>', addition_tokens=['<PAD>']) train, vaild, test = read_fake_common_deepfix_error_dataset_with_limit_length( 500) train = convert_c_code_fields_to_cpp_fields(train, convert_include=False) vaild = convert_c_code_fields_to_cpp_fields(vaild, convert_include=False) test = convert_c_code_fields_to_cpp_fields(test, convert_include=False) else: vocab = create_common_error_vocabulary(begin_tokens=['<BEGIN>'], end_tokens=['<END>'], unk_token='<UNK>', addition_tokens=['<GAP>']) train, vaild, test = read_fake_common_c_error_dataset_with_limit_length( MAX_TOKEN_LENGTH) train = convert_c_code_fields_to_cpp_fields(train) vaild = convert_c_code_fields_to_cpp_fields(vaild) test = convert_c_code_fields_to_cpp_fields(test) tokenize_fn = tokenize_by_clex_fn() parse_param = [vocab, action_list_sorted, tokenize_fn] parse_test_param = [vocab, tokenize_fn] train_data = parse_error_tokens_and_action_map(train, 'train', *parse_param) vaild_data = parse_error_tokens_and_action_map(vaild, 'valid', *parse_param) test_data = parse_error_tokens_and_action_map(test, 'test', *parse_param) # vaild_data = parse_test_tokens(vaild, 'valid', *parse_test_param) # test_data = parse_test_tokens(test, 'test', *parse_test_param) train = train.loc[train_data[0].index.values] vaild = vaild.loc[vaild_data[0].index.values] test = test.loc[test_data[0].index.values] train_dict = { 'error_code_word_id': train_data[0], 'ac_code_word_id': train_data[1], 'token_map': train_data[2], 'error_mask': train_data[3], 'includes': train['includes'], 'is_copy': train_data[4], 'pointer_map': train_data[5], 'distance': train_data[6], 'error_code_word': train_data[7] } valid_dict = { 'error_code_word_id': vaild_data[0], 'ac_code_word_id': vaild_data[1], 'token_map': vaild_data[2], 'error_mask': vaild_data[3], 'includes': vaild['includes'], 'is_copy': vaild_data[4], 'pointer_map': vaild_data[5], 'distance': vaild_data[6], 'error_code_word': vaild_data[7] } test_dict = { 'error_code_word_id': test_data[0], 'ac_code_word_id': test_data[1], 'token_map': test_data[2], 'error_mask': test_data[3], 'includes': test['includes'], 'is_copy': test_data[4], 'pointer_map': test_data[5], 'distance': test_data[6], 'error_code_word': test_data[7] } if addition_infomation: train_dict = add_c_common_code_original_info(data_dict=train_dict, df=train) valid_dict = add_c_common_code_original_info(data_dict=valid_dict, df=vaild) test_dict = add_c_common_code_original_info(data_dict=test_dict, df=test) # valid_dict = {'error_code_word_id': vaild_data, 'includes': vaild['includes']} # test_dict = {'error_code_word_id': test_data, 'includes': test['includes']} # train_data_set = CCodeErrorDataSet(pd.DataFrame(train_dict), vocab, 'train') # valid_data_set = CCodeErrorDataSet(pd.DataFrame(valid_dict), vocab, 'all_valid') # test_data_set = CCodeErrorDataSet(pd.DataFrame(test_dict), vocab, 'all_test') return train_dict, valid_dict, test_dict