def main():
"""Main program."""
initialize_megatron(extra_args_provider=add_text_generate_args,
args_defaults={
'tokenizer_type': 'GPT2BPETokenizer',
'no_load_rng': True,
'no_load_optim': True
})
args = get_args()
if args.num_layers_per_virtual_pipeline_stage is not None:
print(
"Interleaved pipeline schedule is not yet supported for text generation."
)
exit()
# Set up model and load checkpoint.
model = get_model(model_provider)
if args.load is not None:
_ = load_checkpoint(model, None, None)
assert len(model) == 1, "Above condition should have caught this"
model = model[0]
# Generate samples.
if args.num_samples == 0:
args.micro_batch_size = 1
if args.sample_input_file != None:
generate_samples_input_from_file(model)
else:
generate_samples_interactive(model)
else:
generate_and_write_samples_unconditional(model)
def main():
"""
Generate text/sample model
"""
model, neox_args = setup_for_inference_or_eval(use_cache=True)
if neox_args.recompute:
model.module.inference_mode(
use_cache=False) # don't use kv cache if recomputing
if neox_args.text_gen_type == "unconditional":
print_rank_0(
f"Generating samples unconditionally and saving results to {neox_args.sample_output_file}"
)
generate_samples_unconditional(
neox_args=neox_args,
model=model,
number_of_samples=neox_args.num_samples,
output_file=neox_args.sample_output_file,
maximum_tokens=neox_args.maximum_tokens,
recompute=neox_args.recompute,
temperature=neox_args.temperature,
top_k=neox_args.top_k,
top_p=neox_args.top_p,
)
elif neox_args.text_gen_type == "input-file":
print_rank_0(
f"Generating samples from input file {neox_args.sample_input_file}"
)
assert neox_args.sample_input_file is not None
generate_samples_input_from_file(
neox_args=neox_args,
model=model,
input_file=neox_args.sample_input_file,
output_file=neox_args.sample_output_file,
maximum_tokens=neox_args.maximum_tokens,
recompute=neox_args.recompute,
temperature=neox_args.temperature,
top_k=neox_args.top_k,
top_p=neox_args.top_p,
)
elif neox_args.text_gen_type == "interactive":
generate_samples_interactive(
neox_args=neox_args,
model=model,
recompute=neox_args.recompute,
temperature=neox_args.temperature,
maximum_tokens=neox_args.maximum_tokens,
top_k=neox_args.top_k,
top_p=neox_args.top_p,
)
else:
raise ValueError(
f"`text-gen-type` either not specified or not recognised: {neox_args.text_gen_type}"
)
def main():
"""Main program."""
initialize_megatron(extra_args_provider=add_text_generate_args,
args_defaults={'tokenizer_type': 'GPT2BPETokenizer'})
# Set up model and load checkpoint.
model = get_model(model_provider)
args = get_args()
if args.load is not None:
_ = load_checkpoint(model, None, None)
# Generate samples.
if args.num_samples == 0:
args.batch_size = 1
if args.sample_input_file != "":
generate_samples_input_from_file(model)
else:
generate_samples_interactive(model)
else:
generate_and_write_samples_unconditional(model)
top_p=neox_args.top_p)
elif neox_args.text_gen_type == 'input-file':
print_rank_0(
f'Generating samples from input file {neox_args.sample_input_file}'
)
assert neox_args.sample_input_file is not None
generate_samples_input_from_file(
neox_args=neox_args,
model=model,
input_file=neox_args.sample_input_file,
output_file=neox_args.sample_output_file,
maximum_tokens=neox_args.maximum_tokens,
recompute=neox_args.recompute,
temperature=neox_args.temperature,
top_k=neox_args.top_k,
top_p=neox_args.top_p)
elif neox_args.text_gen_type == 'interactive':
generate_samples_interactive(neox_args=neox_args,
model=model,
recompute=neox_args.recompute,
temperature=neox_args.temperature,
top_k=neox_args.top_k,
top_p=neox_args.top_p)
else:
raise ValueError(
f"`text-gen-type` either not specified or not recognised: {neox_args.text_gen_type}"
)