def main():
args = parse_args()
if os.path.exists(args.output_path):
print(f"Output path {args.output_path} exists!!!")
return
random.seed(args.seed)
np.random.seed(args.seed)
if args.limit:
print(
"WARNING: --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT."
)
if args.tasks == "all_tasks":
task_names = tasks.ALL_TASKS
else:
task_names = args.tasks.split(",")
task_dict = tasks.get_task_dict(task_names)
lm = models.get_model(args.model)
train_args = simple_parse_args_string(args.train_args)
model_args = simple_parse_args_string(args.model_args)
if train_args:
train_args.update(model_args)
train_args["seed"] = args.seed
results = evaluator.evaluate(lm, task_dict, args.provide_description,
args.num_fewshot, args.limit, train_args,
args.model_args, args.seed)
results["args"] = args.__dict__
dumped = json.dumps(results, indent=2)
print(dumped)
if args.output_path:
with open(args.output_path, "w") as f:
f.write(dumped)
for task, task_res in results.items():
if task not in task_names:
continue
if "train_args" not in task_res:
experiment = comet_ml.Experiment(
api_key=os.environ.get('COMET_API_KEY'),
project_name=os.environ.get('COMET_PROJECT', "few-shot"),
workspace=os.environ.get('COMET_WORKSPACE', "yuvalkirstain"),
)
experiment.log_asset(args.output_path)
else:
experiment = comet_ml.ExistingExperiment(
api_key=os.environ.get('COMET_API_KEY'),
previous_experiment=task_res["train_args"]
["previous_experiment"])
experiment.log_asset(args.output_path)
def create_from_arg_string(cls, arg_string):
args = utils.simple_parse_args_string(arg_string)
return cls(device=args.get("device", "cpu"))
def create_from_arg_string(cls, arg_string, additional_config=None):
additional_config = {} if additional_config is None else additional_config
args = utils.simple_parse_args_string(arg_string)
args2 = {k: v for k, v in additional_config.items() if v is not None}
return cls(**args, **args2)
def create_from_arg_string(cls, arg_string):
args = utils.simple_parse_args_string(arg_string)
return cls(engine=args.get("engine", "davinci"))
def create_from_arg_string(cls, arg_string):
args = utils.simple_parse_args_string(arg_string)
return cls(device=args.get("device", None), pretrained=args.get("pretrained", "t5-small"))
def create_from_arg_string(cls, arg_string):
args = utils.simple_parse_args_string(arg_string)
return cls(**args)
