def main(): """Main program.""" initialize_megatron(extra_args_provider=add_text_generate_args, args_defaults={ 'tokenizer_type': 'GPT2BPETokenizer', 'no_load_rng': True, 'no_load_optim': True }) args = get_args() if args.num_layers_per_virtual_pipeline_stage is not None: print( "Interleaved pipeline schedule is not yet supported for text generation." ) exit() # Set up model and load checkpoint. model = get_model(model_provider) if args.load is not None: _ = load_checkpoint(model, None, None) assert len(model) == 1, "Above condition should have caught this" model = model[0] # Generate samples. if args.num_samples == 0: args.micro_batch_size = 1 if args.sample_input_file != None: generate_samples_input_from_file(model) else: generate_samples_interactive(model) else: generate_and_write_samples_unconditional(model)
def main(): """ Generate text/sample model """ model, neox_args = setup_for_inference_or_eval(use_cache=True) if neox_args.recompute: model.module.inference_mode( use_cache=False) # don't use kv cache if recomputing if neox_args.text_gen_type == "unconditional": print_rank_0( f"Generating samples unconditionally and saving results to {neox_args.sample_output_file}" ) generate_samples_unconditional( neox_args=neox_args, model=model, number_of_samples=neox_args.num_samples, output_file=neox_args.sample_output_file, maximum_tokens=neox_args.maximum_tokens, recompute=neox_args.recompute, temperature=neox_args.temperature, top_k=neox_args.top_k, top_p=neox_args.top_p, ) elif neox_args.text_gen_type == "input-file": print_rank_0( f"Generating samples from input file {neox_args.sample_input_file}" ) assert neox_args.sample_input_file is not None generate_samples_input_from_file( neox_args=neox_args, model=model, input_file=neox_args.sample_input_file, output_file=neox_args.sample_output_file, maximum_tokens=neox_args.maximum_tokens, recompute=neox_args.recompute, temperature=neox_args.temperature, top_k=neox_args.top_k, top_p=neox_args.top_p, ) elif neox_args.text_gen_type == "interactive": generate_samples_interactive( neox_args=neox_args, model=model, recompute=neox_args.recompute, temperature=neox_args.temperature, maximum_tokens=neox_args.maximum_tokens, top_k=neox_args.top_k, top_p=neox_args.top_p, ) else: raise ValueError( f"`text-gen-type` either not specified or not recognised: {neox_args.text_gen_type}" )
def main(): """Main program.""" initialize_megatron(extra_args_provider=add_text_generate_args, args_defaults={'tokenizer_type': 'GPT2BPETokenizer'}) # Set up model and load checkpoint. model = get_model(model_provider) args = get_args() if args.load is not None: _ = load_checkpoint(model, None, None) # Generate samples. if args.num_samples == 0: args.batch_size = 1 if args.sample_input_file != "": generate_samples_input_from_file(model) else: generate_samples_interactive(model) else: generate_and_write_samples_unconditional(model)
top_p=neox_args.top_p) elif neox_args.text_gen_type == 'input-file': print_rank_0( f'Generating samples from input file {neox_args.sample_input_file}' ) assert neox_args.sample_input_file is not None generate_samples_input_from_file( neox_args=neox_args, model=model, input_file=neox_args.sample_input_file, output_file=neox_args.sample_output_file, maximum_tokens=neox_args.maximum_tokens, recompute=neox_args.recompute, temperature=neox_args.temperature, top_k=neox_args.top_k, top_p=neox_args.top_p) elif neox_args.text_gen_type == 'interactive': generate_samples_interactive(neox_args=neox_args, model=model, recompute=neox_args.recompute, temperature=neox_args.temperature, top_k=neox_args.top_k, top_p=neox_args.top_p) else: raise ValueError( f"`text-gen-type` either not specified or not recognised: {neox_args.text_gen_type}" )