def __init__(self, models_to_load=[], device='cpu'): super(FullModelInterface, self).__init__() self.models = models_to_load self.dps = { k:DefaultProcessor( model_info[k][0], Hierarchy.from_dict(read_pickle(os.path.join(model_info[k][1], 'hierarchy.pkl')))\ if model_info[k][1] is not None else self.hierarchy, model_file=os.path.join(model_info[k][1], 'model_state.tpkl') if model_info[k][1] is not None else None, device=device, cluster=True) for k in self.models} self.trained_queries = {k:get_queries(os.path.join(model_info[k][1], 'used_targets.txt')) if model_info[k][1] is not None else list(self.hierarchy.descriptions.keys()) for k in self.models}
hierarchy.to_dict(), os.path.join(args.save_checkpoint_folder, 'hierarchy.pkl')) if os.path.exists(counts_file): copyfile(counts_file, os.path.join(args.save_checkpoint_folder, 'counts.pkl')) if os.path.exists(used_targets_file): copyfile( used_targets_file, os.path.join(args.save_checkpoint_folder, 'used_targets.txt')) if args.expensive_val_every is not None: supervised_val_file = os.path.join(args.supervised_data_dir, 'supervised.data') hierarchy_file = os.path.join(args.supervised_data_dir, 'hierarchy.pkl') supervised_val_hierarchy = Hierarchy.from_dict(read_pickle(hierarchy_file))\ if os.path.exists(hierarchy_file) else hierarchy else: supervised_val_file = None supervised_val_hierarchy = None expensive_val_every = int( args.expensive_val_every ) if args.expensive_val_every is not None else None try: main(args.model_type, train_file, hierarchy, counts_file, val_file=val_file, save_checkpoint_folder=args.save_checkpoint_folder,
parser.add_argument("--supervised_data_dir", default=None) parser.add_argument("--results_folder", default=None) args = parser.parse_args() val_file = os.path.join(p.data_dir, 'val.data') if not args.supervised else os.path.join(args.supervised_data_dir, 'supervised.data') if args.email: email_sender = EmailSender(smtp_server=p.smtp_server, port=p.port, sender_email=p.sender_email, sender_password=args.sender_password, receiver_email=p.receiver_email, subject="%s: testing %s model" % (socket.gethostname(), args.model_type)) email_sender.send_email("Starting to test %s model." % args.model_type) else: email_sender = None if args.supervised: hierarchy_file = os.path.join(args.supervised_data_dir, 'hierarchy.pkl') if not os.path.exists(hierarchy_file): hierarchy_file = os.path.join(args.checkpoint_folder, 'hierarchy.pkl') else: hierarchy_file = os.path.join(args.checkpoint_folder, 'hierarchy.pkl') hierarchy = Hierarchy.from_dict(read_pickle(hierarchy_file)) try: main(args.model_type, val_file, args.checkpoint_folder, hierarchy, supervised=args.supervised, device=args.device, email_sender=email_sender, results_folder=args.results_folder, noload=args.noload) # nprocs = 2 # main_distributed = distributed_wrapper(main, nprocs) # main_distributed(args.model_type, val_file, args.checkpoint_folder, device=args.device) except Exception as e: if email_sender is not None: email_sender("Got an exception:\n%s" % e) raise e