help="model working platform.")
def get_config(config):
config = TransformerConfig.from_json_file(config)
config.compute_type = mstype.float32
config.dtype = mstype.float32
return config
if __name__ == '__main__':
args, _ = parser.parse_known_args()
if args.vocab.endswith("bin"):
vocab = Dictionary.load_from_persisted_dict(args.vocab)
else:
vocab = Dictionary.load_from_text([args.vocab])
_config = get_config(args.config)
device_id = os.getenv('DEVICE_ID', None)
if device_id is None:
device_id = 0
device_id = int(device_id)
context.set_context(
#mode=context.GRAPH_MODE,
mode=context.PYNATIVE_MODE,
device_target=args.platform,
reserve_class_name_in_scope=False,
device_id=device_id)
if args.metric == 'rouge':
result = infer(_config)
raise TypeError("`--processes` must be an integer.")
available_dict = []
args_groups = []
for file in os.listdir(source_folder):
if args.prefix and not file.startswith(args.prefix):
continue
if file.endswith(".txt"):
output_path = os.path.join(output_folder,
file.replace(".txt", "_bpe.txt"))
dict_path = os.path.join(output_folder,
file.replace(".txt", ".dict"))
available_dict.append(dict_path)
args_groups.append(
(codes, os.path.join(source_folder,
file), output_path, dict_path))
kernel_size = 1 if args.processes <= 0 else args.processes
kernel_size = min(kernel_size, cpu_count())
pool = Pool(kernel_size)
for arg in args_groups:
pool.apply_async(bpe_encode, args=arg)
pool.close()
pool.join()
vocab = Dictionary.load_from_text(available_dict)
if args.threshold is not None:
vocab = vocab.shrink(args.threshold)
vocab.persistence(args.vocab_path)
print(f" | Vocabulary Size: {len(vocab)}")