for n2, p2 in prunable_model.named_parameters():
# e.g. replace `linear.module.weight` with `linear.weight` when appropriate.
if p2.data_ptr() in sparse_dataptrs:
n1 = n2.replace(".module", "")
else:
n1 = n2
p1 = get_module_attr(dense_model, n1)
p2.data[:] = p1
# Save the prunable model.
new_folder_name = checkpoint_folder + "_prunable"
prunable_model.save_pretrained(new_folder_name)
print(f"Saved prunable model to:\n{new_folder_name}")
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("checkpoint_folder",
type=str,
help="Path to checkpoint to convert")
parser.add_argument("-e",
"--experiment",
choices=list(CONFIGS.keys()),
help="Available experiments",
required=True)
args = parser.parse_args()
convert_to_prunable_checkpoint(**args.__dict__)
config = copy.deepcopy(CONFIGS[args.name])
# Merge configuration with command line arguments
config.update(vars(args))
config = process_args(args, config)
if config is None:
# This may return when a sigopt experiment is created.
print("Nothing to run (config=None).")
return
for trial in create_trials(config):
run_trial(trial)
if __name__ == "__main__":
parser = argparse.ArgumentParser(parents=get_default_parsers(),
argument_default=argparse.SUPPRESS,
description=__doc__)
parser.add_argument("-e",
"--experiment",
dest="name",
help="Experiment to run",
choices=CONFIGS.keys())
args = parser.parse_args()
if "name" not in args:
parser.print_help()
exit(1)
main(args)
print()
print(
tabulate(table_3,
headers="firstrow",
tablefmt=tablefmt,
floatfmt="0.1%",
stralign="left",
numalign="center"))
if __name__ == "__main__":
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description=__doc__)
parser.add_argument("-e",
"--experiments",
dest="experiments",
nargs="+",
choices=list(CONFIGS.keys()),
help="Experiments to analyze",
default=list(CONFIGS.keys()))
parser.add_argument("--tablefmt",
default="grid",
choices=[
"plain", "simple", "grid", "pipe", "orgtbl", "rst",
"mediawiki", "latex", "latex_raw", "latex_booktabs"
],
help="Table format")
args = parser.parse_args()
print_parameters_table(args.experiments, args.tablefmt)
import ray # noqa: F401
from experiments import CONFIGS
from nupic.research.frameworks.ray.run_with_raytune import run
from nupic.research.frameworks.vernon.parser_utils import (
get_default_parsers,
process_args,
)
if __name__ == "__main__":
parser = argparse.ArgumentParser(
parents=get_default_parsers(),
)
parser.add_argument("-e", "--experiment", dest="name", default="default_base",
help="Experiment to run", choices=CONFIGS.keys())
args = parser.parse_args()
if args.name is None:
parser.print_help()
exit(1)
# Get configuration values
config = copy.deepcopy(CONFIGS[args.name])
# Merge configuration with command line arguments
config.update(vars(args))
# Process args and modify config appropriately.
config = process_args(args, config)
def create_cmd_parser():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
argument_default=argparse.SUPPRESS,
add_help=False,
)
# Run options
parser.add_argument(
"-e",
"--exp_name",
help="Experiment to run",
choices=list(CONFIGS.keys())
)
parser.add_argument(
"-n",
"--wandb_run_name",
default="",
help="Name of the wandb run"
)
parser.add_argument(
"-v",
"--verbose",
action="store_true",
default=False,
)
parser.add_argument(
"-l",
"--local_only",
action="store_true",
default=False,
help="Whether or not to log results to wandb"
)
parser.add_argument(
"-c",
"--cpu",
action="store_true",
default=False,
help="Whether to use CPU even if GPU is available",
)
parser.add_argument(
"-r",
"--restore",
action="store_true",
default=False,
help="Whether to restore from existing experiment with same project name",
)
parser.add_argument(
"-p",
"--project_id",
default=None,
help="Alternative way of providing project id",
)
# TODO: evaluate whether or not debugging flag is required
parser.add_argument(
"-d",
"--debugging",
action="store_true",
default=False,
help="Option to use when debugging so not every test run is logged.",
)
return parser
def main():
cmd_parser = argparse.ArgumentParser()
cmd_parser.add_argument("experiments",
nargs="+",
choices=list(CONFIGS.keys()),
help="Available experiments")
cmd_parser.add_argument("--local_rank",
default=None,
help="added by torch.distributed.launch")
cmd_args = cmd_parser.parse_args()
for experiment in cmd_args.experiments:
config_dict = CONFIGS[experiment]
local_rank = int(cmd_args.local_rank or -1)
config_dict["local_rank"] = local_rank
# See all possible arguments in transformers/training_args.py and ./run_args.py
exp_parser = HfArgumentParser(
(ModelArguments, DataTrainingArguments, CustomTrainingArguments))
model_args, data_args, training_args = exp_parser.parse_dict(
config_dict)
# Overrides default behavior of TrainingArguments of setting run name
# equal to output_dir when not available
if training_args.run_name == training_args.output_dir:
training_args.run_name = experiment
# Run name (or experiment name) is added to the output_dir
training_args.output_dir = os.path.join(training_args.output_dir,
training_args.run_name)
# Initialize wandb now to include the logs that follow.
# For now, only support early wandb logging when running one experiment.
if is_wandb_available() and len(cmd_args.experiments) == 1:
CustomWandbCallback.early_init(training_args, local_rank)
# Detecting last checkpoint.
last_checkpoint = None
if (os.path.isdir(training_args.output_dir) and training_args.do_train
and not training_args.overwrite_output_dir):
last_checkpoint = get_last_checkpoint(training_args.output_dir)
logging.warning(f"Loading from checkpoint: {last_checkpoint} ")
if (last_checkpoint is None
and len(os.listdir(training_args.output_dir)) > 0):
raise ValueError(
f"Output directory ({training_args.output_dir}) already exists and "
"is not empty. Use --overwrite_output_dir to overcome.")
elif last_checkpoint is not None:
logging.info(
f"Checkpoint detected, resuming training at {last_checkpoint}. To "
"avoid this behavior, change the `--output_dir` or add "
"`--overwrite_output_dir` to train from scratch.")
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
handlers=[logging.StreamHandler(sys.stdout)],
level=(logging.INFO if is_main_process(training_args.local_rank)
else logging.WARN))
# Log config.
logging.info(f"Running with config:\n{pformat(config_dict, indent=4)}")
# Log on each process the small summary:
logging.warning(
f"Process rank: {training_args.local_rank}, "
f"device: {training_args.device}, n_gpu: {training_args.n_gpu} "
f"distributed training: {bool(training_args.local_rank != -1)}, "
f"16-bits training: {training_args.fp16}")
# Set the verbosity to info of the Transformers logging (on main process only):
if is_main_process(training_args.local_rank):
transformers.utils.logging.set_verbosity_info()
transformers.utils.logging.enable_default_handler()
transformers.utils.logging.enable_explicit_format()
logging.info("Training/evaluation parameters %s", training_args)
logging.info("Model parameters: %s", model_args)
logging.info("Data parameters: %s", data_args)
# Set seed before initializing model.
set_seed(training_args.seed)
logging.info(f"Seed to reproduce: {training_args.seed}")
if model_args.finetuning:
run_finetuning_multiple_tasks(model_args,
data_args,
training_args,
last_checkpoint=last_checkpoint)
else:
run_pretraining(model_args,
data_args,
training_args,
last_checkpoint=last_checkpoint)
# destroy process group before launching another experiment
if cmd_args.local_rank:
torch.distributed.destroy_process_group()