def work(self): project = ProjectProvider(self.session).by_id(self.project) self.info(f'Task = {self.train_task} child_task: {self.child_task}') model = Model(created=now(), name=self.name, project=self.project, equations='', fold=self.fold) provider = ModelProvider(self.session) if self.train_task: task_provider = TaskProvider(self.session) task = task_provider.by_id(self.train_task) model.score_local = task.score task_dir = join(TASK_FOLDER, str(self.child_task or task.id)) src_log = f'{task_dir}/log' models_dir = join(MODEL_FOLDER, project.name) os.makedirs(models_dir, exist_ok=True) model_path_tmp = f'{src_log}/traced.pth' traced = trace_model_from_checkpoint(src_log, self, file=self.file) model_path = f'{models_dir}/{model.name}.pth' model_weight_path = f'{models_dir}/{model.name}_weight.pth' torch.jit.save(traced, model_path_tmp) shutil.copy(model_path_tmp, model_path) file = self.file = 'best_full' shutil.copy(f'{src_log}/checkpoints/{file}.pth', model_weight_path) provider.add(model)
def model_remove(): data = request_data() provider = ModelProvider(_write_session) model = provider.by_id(data['id'], joined_load=[Model.project_rel]) celery_tasks.remove_model(_write_session, model.project_rel.name, model.name) provider.remove(model.id)
def work(self): self.create_base() parts = self.generate_parts(self.count()) for preds in self.solve(self.key(), parts): score = self.score(preds) if self.layout and self.plot_count > 0: self.plot(preds, score) score = self.score_final() if isnan(score): score = -1 else: score = float(score) if self.layout: self.plot_final(score) self.task.score = score self.task_provider.update() if self.model_id: provider = ModelProvider(self.session) model = provider.by_id(self.model_id) model.score_local = score provider.commit()
def dag_model_start(session: Session, data: dict): provider = ModelProvider(session) model = provider.by_id(data['model_id']) dag_provider = DagProvider(session) dag = dag_provider.by_id(data['dag'], joined_load=[Dag.project_rel]) project = dag.project_rel src_config = Config.from_yaml(dag.config) pipe = src_config['pipes'][data['pipe']['name']] equations = yaml_load(model.equations) versions = data['pipe']['versions'] if len(versions) > 0: version = data['pipe']['version'] pipe_equations = yaml_load(version['equations']) found_version = versions[0] for v in versions: if v['name'] == version['name']: found_version = v break found_version['used'] = now() if len(pipe) == 1: pipe[list(pipe)[0]].update(pipe_equations) else: pipe.update(pipe_equations) equations[data['pipe']['name']] = versions model.equations = yaml_dump(equations) for v in pipe.values(): v['model_id'] = model.id v['model_name'] = model.name config = { 'info': { 'name': data['pipe']['name'], 'project': project.name }, 'executors': pipe } if model.dag: old_dag = dag_provider.by_id(model.dag) if old_dag.name != dag.name: model.dag = dag.id else: model.dag = dag.id provider.commit() dag_standard(session=session, config=config, debug=False, upload_files=False, copy_files_from=data['dag'])
def work(self): project = ProjectProvider(self.session).by_id(self.project) self.info(f'Task = {self.train_task} child_task: {self.child_task}') model = Model( created=now(), name=self.name, project=self.project, equations='', fold=self.fold ) provider = ModelProvider(self.session) if self.train_task: task_provider = TaskProvider(self.session) dag_provider = DagProvider(self.session) task = task_provider.by_id(self.train_task) dag = dag_provider.by_id(task.dag) task_dir = join(TASK_FOLDER, str(self.child_task or task.id)) # get log directory config = yaml_load(dag.config) executor_config = config['executors'][task.executor] catalyst_config_file = executor_config['args']['config'] catalyst_config_file = join(task_dir, catalyst_config_file) catalyst_config = yaml_load(file=catalyst_config_file) catalyst_logdir = catalyst_config['args']['logdir'] model.score_local = task.score src_log = f'{task_dir}/{catalyst_logdir}' models_dir = join(MODEL_FOLDER, project.name) os.makedirs(models_dir, exist_ok=True) model_path_tmp = f'{src_log}/traced.pth' traced = trace_model_from_checkpoint(src_log, self, file=self.file) model_path = f'{models_dir}/{model.name}.pth' model_weight_path = f'{models_dir}/{model.name}_weight.pth' torch.jit.save(traced, model_path_tmp) shutil.copy(model_path_tmp, model_path) file = self.file = 'best_full' shutil.copy(f'{src_log}/checkpoints/{file}.pth', model_weight_path) provider.add(model)
def __init__(self, model_id: int = None, suffix: str = '', max_count=None, part_size: int = None, cache_names: List[str] = (), **kwargs): super().__init__(**kwargs) self.__dict__.update(kwargs) self.model_id = model_id self.suffix = suffix self.max_count = max_count self.part_size = part_size self.part = None self.cache = dict() self.cache_names = cache_names self.model_name = kwargs.get('model_name') self.name = kwargs.get('name') if not self.model_name and self.model_id: self.model_name = ModelProvider(self.session).by_id( self.model_id).name if not self.name: self.name = self.model_name self.suffix = self._solve(suffix)
def dag_pipe(session: Session, config: dict, config_text: str = None): assert 'interfaces' in config, 'interfaces missed' assert 'pipes' in config, 'pipe missed' info = config['info'] storage = Storage(session) dag_provider = DagProvider(session) folder = os.getcwd() project = ProjectProvider(session).by_name(info['project']).id dag = dag_provider.add( Dag( config=config_text, project=project, name=info['name'], docker_img=info.get('docker_img'), type=DagType.Pipe.value ) ) storage.upload(folder, dag) # Change model dags which have the same name ModelProvider(session ).change_dag(project=project, name=info['name'], to=dag.id)
def work(self): self.message = f'Task id = {self.task.id}' prob = np.load(self.prob_file) argmax = prob.argmax(axis=1) pd.DataFrame({ 'ImageId': np.arange(1, len(argmax) + 1), 'Label': argmax }).to_csv(self.out_file, index=False) score = super().work() if self.model_id: provider = ModelProvider(self.session) model = provider.by_id(self.model_id) model.score_public = score provider.commit()
def dag_model_start(session: Session, data: dict): provider = ModelProvider(session) model = provider.by_id(data['model_id']) dag = DagProvider(session ).by_id(data['dag'], joined_load=[Dag.project_rel]) project = dag.project_rel src_config = Config.from_yaml(dag.config) pipe = src_config['pipes'][data['pipe']] for k, v in pipe.items(): if v.get('slot') != data['slot']: continue params = yaml_load(data['interface_params']) slot = { 'interface': data['interface'], 'interface_params': params, 'slot': k, 'name': model.name, 'id': data['model_id'] } v['slot'] = slot config = { 'info': { 'name': data['pipe'], 'project': project.name }, 'executors': pipe } dag_standard( session=session, config=config, debug=False, upload_files=False, copy_files_from=data['dag'] ) model.dag = data['dag'] model.interface = data['interface'] model.interface_params = data['interface_params'] model.slot = data['slot'] provider.commit()
def work(self): submissions = api.competition_submissions(self.competition) submission_refs = {s.ref for s in submissions} if self.submit_type == 'file': self.file_submit() else: self.kernel_submit() self.info('waiting for the submission on Kaggle') step = 10 for i in range(int(self.wait_seconds // step)): try: submissions = api.competition_submissions(self.competition) for s in submissions: if s.ref not in submission_refs: if s.status == 'complete': if s.publicScore is None: raise Exception('Submission is complete, ' 'but publicScore is None') score = float(s.publicScore) if self.model_id: provider = ModelProvider(self.session) model = provider.by_id(self.model_id) model.score_public = score provider.commit() return {'res': score} elif s.status == 'error': raise Exception(f'Submission error ' f'on Kaggle: {s.errorDescription}') break except TypeError: pass time.sleep(step) raise Exception(f'Submission is not ' f'complete after {self.wait_seconds}')
def dag_model_add(session: Session, data: dict): if not data.get('task'): model = Model(name=data['name'], project=data['project'], equations=data['equations'], created=now()) ModelProvider(session).add(model) return task_provider = TaskProvider(session) task = task_provider.by_id(data['task'], options=joinedload(Task.dag_rel, innerjoin=True)) child_tasks = task_provider.children(task.id) computer = task.computer_assigned child_task = None if len(child_tasks) > 0: child_task = child_tasks[0].id computer = child_tasks[0].computer_assigned project = ProjectProvider(session).by_id(task.dag_rel.project) config = { 'info': { 'name': 'model_add', 'project': project.name, 'computer': computer }, 'executors': { 'model_add': { 'type': 'model_add', 'project': data['project'], 'task': data.get('task'), 'name': data['name'], 'file': data['file'], 'child_task': child_task, 'fold': data['fold'] } } } dag_standard(session=session, config=config, debug=False, upload_files=False)
def work(self): task_provider = TaskProvider(self.session) task = task_provider.by_id(self.train_task) dag = DagProvider(self.session).by_id(self.dag_pipe, joined_load=[Dag.project_rel]) task_dir = join(TASK_FOLDER, str(self.child_task or task.id)) src_log = f'{task_dir}/log' models_dir = join(MODEL_FOLDER, dag.project_rel.name) os.makedirs(models_dir, exist_ok=True) self.info(f'Task = {self.task} child_task: {self.child_task}') model_path_tmp = f'{src_log}/traced.pth' traced = trace_model_from_checkpoint(src_log, self) model = Model(dag=self.dag_pipe, interface=self.interface, slot=self.slot, score_local=task.score, created=now(), name=self.name, project=dag.project, interface_params=yaml_dump(self.interface_params)) provider = ModelProvider(self.session) provider.add(model, commit=False) try: model_path = f'{models_dir}/{model.name}.pth' model_weight_path = f'{models_dir}/{model.name}_weight.pth' torch.jit.save(traced, model_path_tmp) shutil.copy(model_path_tmp, model_path) shutil.copy(f'{src_log}/checkpoints/best.pth', model_weight_path) interface_params = yaml_load(model.interface_params) interface_params['file'] = join('models', model.name + '.pth') model.interface_params = yaml_dump(interface_params) provider.update() except Exception as e: provider.rollback() raise e
def models(): data = request_data() options = PaginatorOptions(**data['paginator']) provider = ModelProvider(_read_session) res = provider.get(data, options) return res
def model_start_begin(): data = request_data() return ModelProvider(_read_session).model_start_begin(data['model_id'])