def work(self): project = ProjectProvider(self.session).by_id(self.project) self.info(f'Task = {self.train_task} child_task: {self.child_task}') model = Model(created=now(), name=self.name, project=self.project, equations='', fold=self.fold) provider = ModelProvider(self.session) if self.train_task: task_provider = TaskProvider(self.session) task = task_provider.by_id(self.train_task) model.score_local = task.score task_dir = join(TASK_FOLDER, str(self.child_task or task.id)) src_log = f'{task_dir}/log' models_dir = join(MODEL_FOLDER, project.name) os.makedirs(models_dir, exist_ok=True) model_path_tmp = f'{src_log}/traced.pth' traced = trace_model_from_checkpoint(src_log, self, file=self.file) model_path = f'{models_dir}/{model.name}.pth' model_weight_path = f'{models_dir}/{model.name}_weight.pth' torch.jit.save(traced, model_path_tmp) shutil.copy(model_path_tmp, model_path) file = self.file = 'best_full' shutil.copy(f'{src_log}/checkpoints/{file}.pth', model_weight_path) provider.add(model)
def work(self): project = ProjectProvider(self.session).by_id(self.project) self.info(f'Task = {self.train_task} child_task: {self.child_task}') model = Model( created=now(), name=self.name, project=self.project, equations='', fold=self.fold ) provider = ModelProvider(self.session) if self.train_task: task_provider = TaskProvider(self.session) dag_provider = DagProvider(self.session) task = task_provider.by_id(self.train_task) dag = dag_provider.by_id(task.dag) task_dir = join(TASK_FOLDER, str(self.child_task or task.id)) # get log directory config = yaml_load(dag.config) executor_config = config['executors'][task.executor] catalyst_config_file = executor_config['args']['config'] catalyst_config_file = join(task_dir, catalyst_config_file) catalyst_config = yaml_load(file=catalyst_config_file) catalyst_logdir = catalyst_config['args']['logdir'] model.score_local = task.score src_log = f'{task_dir}/{catalyst_logdir}' models_dir = join(MODEL_FOLDER, project.name) os.makedirs(models_dir, exist_ok=True) model_path_tmp = f'{src_log}/traced.pth' traced = trace_model_from_checkpoint(src_log, self, file=self.file) model_path = f'{models_dir}/{model.name}.pth' model_weight_path = f'{models_dir}/{model.name}_weight.pth' torch.jit.save(traced, model_path_tmp) shutil.copy(model_path_tmp, model_path) file = self.file = 'best_full' shutil.copy(f'{src_log}/checkpoints/{file}.pth', model_weight_path) provider.add(model)
def work(self): task_provider = TaskProvider(self.session) task = task_provider.by_id(self.train_task) dag = DagProvider(self.session).by_id(self.dag_pipe, joined_load=[Dag.project_rel]) task_dir = join(TASK_FOLDER, str(self.child_task or task.id)) src_log = f'{task_dir}/log' models_dir = join(MODEL_FOLDER, dag.project_rel.name) os.makedirs(models_dir, exist_ok=True) self.info(f'Task = {self.task} child_task: {self.child_task}') model_path_tmp = f'{src_log}/traced.pth' traced = trace_model_from_checkpoint(src_log, self) model = Model(dag=self.dag_pipe, interface=self.interface, slot=self.slot, score_local=task.score, created=now(), name=self.name, project=dag.project, interface_params=yaml_dump(self.interface_params)) provider = ModelProvider(self.session) provider.add(model, commit=False) try: model_path = f'{models_dir}/{model.name}.pth' model_weight_path = f'{models_dir}/{model.name}_weight.pth' torch.jit.save(traced, model_path_tmp) shutil.copy(model_path_tmp, model_path) shutil.copy(f'{src_log}/checkpoints/best.pth', model_weight_path) interface_params = yaml_load(model.interface_params) interface_params['file'] = join('models', model.name + '.pth') model.interface_params = yaml_dump(interface_params) provider.update() except Exception as e: provider.rollback() raise e