'output_keys': [
'input_1', 'ground_truth_1', 'lan_idx_for_input_1',
'lan_idx_for_gt_1', 'pos_for_gt_1', 'tokenizer'
]
}]
print('\n------------------- Encoding -------------------------')
x, y, lan_x, lan_y, soft_pos_y, tokenizer = utils.pipeline(
preprocess_pipeline=pipeline,
lan_data_1=origin_ro_data,
lan_data_2=origin_en_data,
params={
**params,
# 'tokenizer': tokenizer
})
print('\n----------------------------------------------')
print(x.shape)
print(y.shape)
print(lan_x.shape)
print(lan_y.shape)
print(soft_pos_y.shape)
print('\n------------------- Decoding -------------------------')
x = utils.pipeline(decode_pl(''), x[:2], None, {'tokenizer': tokenizer})
y = utils.pipeline(decode_pl(''), y[:2], None, {'tokenizer': tokenizer})
print(x[0])
print(soft_pos_y[0])
print(x[1])
print(soft_pos_y[1])
pipeline = zh_en.seg_zh_by_jieba_pipeline + noise_pl.remove_noise + tfds_share_pl.train_tokenizer
# pipeline = zh_en.seg_zh_by_jieba_pipeline + noise_pl.remove_noise
pipeline += pl.sent_2_tokens + sample_pl(2.0) + combine_pl(
0.2) + pl.CDLM_encode + [{
'output_keys': [
'input_1', 'ground_truth_1', 'lan_idx_for_input_1',
'lan_idx_for_gt_1', 'pos_for_gt_1', 'tokenizer'
]
}]
print('\n------------------- Encoding -------------------------')
x, y, lan_x, lan_y, soft_pos_y, tokenizer = utils.pipeline(
preprocess_pipeline=pipeline,
lan_data_1=origin_zh_data[:1000],
lan_data_2=origin_en_data[:1000],
params={
**params,
# 'tokenizer': tokenizer
})
print('\n----------------------------------------------')
print(x.shape)
print(y.shape)
print(lan_x.shape)
print(lan_y.shape)
print(soft_pos_y.shape)
print('\n------------------- Decoding -------------------------')
x = utils.pipeline(decode_pl('ner'), x, None, {'tokenizer': tokenizer})
y = utils.pipeline(decode_pl('ner'), y, None, {'tokenizer': tokenizer})