def update_progress_for_optimize(self): job_progress = self.local_progress session = CeleryDBAdapter.session() record = session.query(ModelDownloaderConversionJobsModel).filter_by( job_id=self.job_id).first() record.progress = job_progress write_record(record, session)
def update_progress_in_database(self, progress): session = CeleryDBAdapter.session() int8_record = session.query(Int8AutotuneJobsModel).filter_by( job_id=self.job_id).first() int8_record.progress = progress write_record(int8_record, session) session.close()
def set_task_id(self, task_id: str): session = CeleryDBAdapter.session() record = session.query(self.db_table).get(self.emit_message.job_id) if record: record.task_id = task_id write_record(record, session) session.close()
def convert_downloaded_model(data: dict): topology_id = data['topologyId'] topology = TopologiesModel.query.get(topology_id) topology.precision = ModelPrecisionEnum(data['dataType']) omz_topology = OMZTopologyModel.query.filter_by(name=topology.name).first() convert_job_record = ModelDownloaderConversionJobsModel.query.filter_by( result_model_id=topology_id).first() convert_job_record.conversion_args = json.dumps(({ 'precision': data['dataType'], })) write_record(convert_job_record, get_db().session) weights = JobsWeight.download_source_model() tasks = [ TASK.subtask(args=[ None, JobTypesEnum.model_convert_type.value, convert_job_record.job_id ], kwargs={ 'progress_weight': weights[JobTypesEnum.model_convert_type] }), ] source_path = os.path.join(MODEL_DOWNLOADS_FOLDER, str(topology_id), omz_topology.path) destination_path = topology.path ir_postprocessing(tasks, source_path, destination_path, topology.id, weights) chain(tasks).apply_async() return jsonify({})
def fetch_downloadable_models(): omz_meta = get_metadata_for_omz_models() parameters = InfoDumperParameters() parser = InfoDumperParser() return_code, _ = run_console_tool(parameters, parser) if return_code: return models = json.loads(parser.stdout) for model in models: model_meta = omz_meta.get( model['name'], { 'topology_type': TaskMethodEnum.generic, 'advanced_configuration': None }) for precision in model['precisions']: existing_model = (OMZTopologyModel.query.filter_by( name=model['name'], precision=ModelPrecisionEnum(precision)).first()) if model['framework'] == 'dldt': model['framework'] = SupportedFrameworksEnum.openvino.value if not existing_model: if model_meta['topology_type'] != TaskMethodEnum.generic: task_type = define_topology_task_type(model) else: task_type = TaskEnum.generic record = OMZTopologyModel( data=model, task_type=task_type, topology_type=model_meta['topology_type'], advanced_configuration=model_meta[ 'advanced_configuration'], precision=ModelPrecisionEnum(precision)) write_record(record, get_db().session)
def set_task_id(self, task_id): session = CeleryDBAdapter.session() resulting_topology = session.query(TopologiesModel).get( self.emit_message.config.result_model_id) resulting_topology.task_id = task_id write_record(resulting_topology, session) session.close()
def run(self): session = CeleryDBAdapter.session() winograd_autotune_job = session.query(WinogradAutotuneJobsModel).get(self.emit_message.job_id) new_model = session.query(TopologiesModel).get(winograd_autotune_job.result_model_id) tuned_path = new_model.path set_status_in_db(WinogradAutotuneJobsModel, self.emit_message.job_id, StatusEnum.running, session) session.close() self.emit_message.emit_progress() parameters = WinogradParameters(self.emit_message.config) parameters.set_parameter('o', tuned_path) parser = WinogradConsoleOutputParser(self.emit_message, WinogradToolStages.get_stages()) return_code, message = run_console_tool(parameters, parser) if return_code: message = parser.get_error() if not message else message error_message = WinogradErrorMessageProcessor.recognize_error(message, 'winograd autotuner') self.emit_message.add_error(error_message) raise WinogradAutotuneError(error_message, self.emit_message.job_id) session = CeleryDBAdapter.session() set_status_in_db(WinogradAutotuneJobsModel, self.emit_message.job_id, StatusEnum.ready, session) winograd_autotune_job = session.query(WinogradAutotuneJobsModel).get(self.emit_message.job_id) winograd_autotune_job.progress = 100 write_record(winograd_autotune_job, session) set_status_in_db(WinogradAutotuneJobsModel, self.emit_message.job_id, StatusEnum.ready, session, force=True) session.close() self.emit_message.emit_message()
def total_progress(self): session = CeleryDBAdapter.session() artifact = session.query(TopologiesModel).get(self.job_id) progress = self.local_progress * self.weight + self.previous_progress artifact.progress = progress write_record(artifact, session) session.close() return progress
def set_inference_result_to_record(self, results): session = CeleryDBAdapter.session() infer_result = session.query(InferenceResultsModel).get( self.inference_result_record_id) infer_result.progress = 100 infer_result.update(results) write_record(infer_result, session) session.close()
def update_progress_for_inference_result(self): session = CeleryDBAdapter.session() infer_result = session.query(InferenceResultsModel).get( self.inference_result_record_id) infer_result.progress = self.local_progress infer_result.status = StatusEnum.running write_record(infer_result, session) session.close()
def update_progress_in_database(self): job_progress = self.local_progress session = CeleryDBAdapter.session() record = session.query(ModelDownloaderModel).filter_by( job_id=self.job_id).first() record.progress = job_progress write_record(record, session) session.close()
def update_model_optimizer_scan_result(self, results: str): session = CeleryDBAdapter.session() mo_job_record = (session.query(ModelOptimizerScanJobsModel).filter_by( topology_id=self.config.topology_id).first()) mo_job_record.information = self.cleanup_results(results) mo_job_record.progress = 100 mo_job_record.status = StatusEnum.ready write_record(mo_job_record, session) session.close()
def on_failure(self): super().on_failure() session = CeleryDBAdapter.session() mo_job_record = session.query(ModelOptimizerScanJobsModel).filter_by( topology_id=self.emit_message.config.topology_id).first() mo_job_record.status = StatusEnum.error write_record(mo_job_record, session) set_status_in_db(TopologiesModel, self.emit_message.config.topology_id, StatusEnum.error, session) session.close()
def set_task_id(self, task_id): session = CeleryDBAdapter.session() downloader_job = session.query(ModelDownloaderModel).filter_by( job_id=self.emit_message.job_id).first() topology = session.query(TopologiesModel).get( downloader_job.result_model_id) topology.task_id = task_id write_record(topology, session) downloader_job.task_id = task_id write_record(downloader_job, session) session.close()
def total_progress(self): session = CeleryDBAdapter.session() artifact = session.query(TopologiesModel).get( self.config.result_model_id) current_job_record = session.query( ModelDownloaderConversionJobsModel).get(self.job_id) total_progress = current_job_record.progress * self.weight artifact.progress = self.previous_accumulated_progress + total_progress progress = artifact.progress write_record(artifact, session) session.close() return progress
def set_task_id(self, task_id): session = CeleryDBAdapter.session() topology_convert_job = session.query( ModelDownloaderConversionJobsModel).filter_by( job_id=self.emit_message.job_id).first() topology = topology_convert_job.result_model self.previous_task_id = topology.task_id topology.task_id = task_id write_record(topology, session) topology_convert_job.task_id = task_id write_record(topology_convert_job, session) session.close()
def create_files(cls, files: dict, artifact_id: int, session_id: str) -> dict: result = {} for file_name, file_data in files.items(): file_record = cls(file_data['name'], artifact_id, file_data['size'], session_id) write_record(file_record, get_db().session) file_record.path = os.path.join(file_record.artifact.path, str(file_record.name)) write_record(file_record, get_db().session) result[file_name] = file_record.id return result
def save_pipeline_config(content: str, topology_id: int): topology = TopologiesModel.query.get(topology_id) pipeline_config_path = os.path.join(topology.path, 'pipeline.config') with open(pipeline_config_path, 'w+') as pipeline_config_file: pipeline_config_file.writelines(content) size = get_size_of_files(pipeline_config_path) config_file_record = FilesModel('pipeline.config', topology_id, size, topology.session_id) config_file_record.progress = 100 config_file_record.status = StatusEnum.ready config_file_record.uploaded_blob_size = size config_file_record.path = pipeline_config_path write_record(config_file_record, get_db().session)
def run_accuracy_check(session_id: str): data = request.get_json() data['session_id'] = session_id data['projectId'] = data['projectId'] data['accuracyConfig'] = '' accuracy_job = AccuracyJobsModel(data) write_record(accuracy_job, get_db().session) TASK.apply_async(args=(None, JobTypesEnum.accuracy_type.value, accuracy_job.job_id), task_id=str(accuracy_job.job_id)) return jsonify({'jobId': accuracy_job.job_id})
def set_status_in_db(table: type(BaseModel), item_id: int, status: StatusEnum, session, message: str = None, force: bool = False): record = session.query(table).get(item_id) if record and (force or STATUS_PRIORITY[record.status] < STATUS_PRIORITY[status]): record.status = status if message: record.error_message = message write_record(record, session)
def total_progress(self): if self.from_celery: session = CeleryDBAdapter.session() artifact = session.query(ArtifactsModel).get(self.artifact_id) else: session = get_db().session artifact = ArtifactsModel.query.get(self.artifact_id) progress = self.local_progress * self.weight + self.previous_progress artifact.progress = progress write_record(artifact, session) if self.from_celery: session.close() return progress
def run(self): parameters = self.emit_message.config dataset_id = self.emit_message.job_id current_job = self.emit_message.add_stage( IEmitMessageStage('Setup dataset parameters', weight=0.1)) session = CeleryDBAdapter.session() dataset = session.query(DatasetsModel).get(self.emit_message.job_id) dataset.status = StatusEnum.running dataset_path = dataset.path write_record(dataset, session) session.close() image_size = parameters.width * parameters.height * parameters.channels create_empty_dir(dataset_path) try: random_generator = DistributionLaw( parameters.dist_law, parameters.params_dist).random_generator except AssertionError as exception: self.emit_message.add_error(str(exception)) raise self.emit_message.update_progress(current_job, 100) current_job = self.emit_message.add_stage( IEmitMessageStage('Generate dataset', weight=0.9)) log.debug('Starting of generating dataset %s', dataset_id) index = 0 while index < self.emit_message.config.size: file_name = os.path.join(dataset_path, '{}.jpg'.format(index)) cv2.imwrite( file_name, random_generator(image_size).reshape( parameters.height, parameters.width, parameters.channels).astype(np.uint8)) percent = (index / (parameters.size + 2)) * 100 if index % np.ceil(parameters.size / 10) == 0: self.emit_message.update_progress(current_job, percent) with open(os.path.join(dataset_path, parameters.name + '.txt'), 'a') as desc_file: desc_file.write('{}.jpg 0\n'.format(index)) index += 1 session = CeleryDBAdapter.session() dataset = session.query(DatasetsModel).get(self.emit_message.job_id) dataset.progress = 100 dataset.status = StatusEnum.ready dataset.size = get_size_of_files(dataset_path) write_record(dataset, session) session.close() self.emit_message.update_progress(current_job, 100) log.debug('Finish of generating dataset %s', dataset_id)
def cancel_job_in_db(job_id: int): job = JobsModel.query.get(job_id) if job: job.status = StatusEnum.cancelled write_record(job, get_db().session) compound_inference_job = CompoundInferenceJobsModel.query.get(job_id) if not compound_inference_job: return inference_results = compound_inference_job.inference_results for inference_result in inference_results: if inference_result.status in (StatusEnum.running, StatusEnum.queued): inference_result.status = StatusEnum.cancelled write_record(inference_result, get_db().session)
def _run_python_calibration(self, tuned_path: str, int8_model_id: int): try: self.emit_message.emit_progress() session = CeleryDBAdapter.session() new_int8_model = session.query(TopologiesModel).get(int8_model_id) int8_job = session.query(Int8AutotuneJobsModel).get(self.emit_message.job_id) original_model = session.query(TopologiesModel).get(new_int8_model.optimized_from) project_model = session.query(ProjectsModel).get(int8_job.project_id) dataset_model = session.query(DatasetsModel).get(project_model.dataset_id) config = construct_accuracy_tool_config(original_model, dataset_model, DevicesEnum.cpu) config.dataset.subsample_size = '{}%'.format(int8_job.subset_size) int8_job.status = StatusEnum.running int8_job.calibration_config = json.dumps(config.to_dict()) write_record(int8_job, session) session.close() tuned_model_path = os.path.join(tuned_path, str(self.emit_message.job_id)) yml_file = '{}.yml'.format(tuned_model_path) config.dump_to_yml(yml_file) cli_params = AccuracyCheckerCliParameters() cli_params.exe = os.path.join(PYTHON_CLI_FOLDER, os.path.join('calibration', 'calibrate.py')) cli_params.set_parameter('y', yml_file) cli_params.set_parameter('th', self.emit_message.config.threshold) cli_params.set_parameter('tp', tuned_model_path) self.emit_message.add_stage(IEmitMessageStage(job_type='int8_tuning')) cli_parser = ProgressParser(self.emit_message, None) code, error = run_console_tool(cli_params, cli_parser, self) if code: self.emit_message.add_error('Calibration tool failed') raise Int8AutotuneError(error, self.emit_message.job_id) self._update_db_on_success() self.emit_message.emit_message() except Exception as exc: log.debug('[ INT8 python ] ERROR: calibration job failed') log.debug(exc) remove_dir(tuned_path) self.emit_message.add_error('Calibration tool failed') raise Int8AutotuneError(str(exc), self.emit_message.job_id)
def generate_queue_of_single_inference_tasks(data: dict, job_id: int, start_tasks: list = None, previous_weight=0): min_nireq = data['minNireq'] max_nireq = data['maxNireq'] step_nireq = data['stepNireq'] min_batch = data['minBatch'] max_batch = data['maxBatch'] step_batch = data['stepBatch'] queue = [] if start_tasks: for task in start_tasks: queue.append(task) num_runs = math.ceil((max_batch - min_batch + 1) / step_batch) * math.ceil( (max_nireq - min_nireq + 1) / step_nireq) weight_single_run = (1 - previous_weight) / num_runs for batch in range(min_batch, max_batch + 1, step_batch): for nireq in range(min_nireq, max_nireq + 1, step_nireq): queue.append( TASK.subtask(args=(JobTypesEnum.single_inference_type.value, job_id), kwargs={ 'data': ExecInfo(batch, nireq).json(), 'progress_weight': weight_single_run })) inference_result = InferenceResultsModel({ 'jobId': job_id, 'execInfo': { 'batch': batch, 'nireq': nireq } }) write_record(inference_result, get_db().session) if not start_tasks: queue.pop(0) queue.insert( 0, TASK.subtask(args=tuple( [None, JobTypesEnum.single_inference_type.value, job_id]), kwargs={ 'data': ExecInfo(min_batch, min_nireq).json(), 'progress_weight': weight_single_run })) get_db().session().commit() return queue
def total_progress(self): session = CeleryDBAdapter.session() infer_results = session.query(InferenceResultsModel).filter_by( job_id=self.job_id).all() compound_infer_record = session.query( CompoundInferenceJobsModel).filter_by(job_id=self.job_id).first() num_single_inferences = compound_infer_record.num_single_inferences progress = 0.0 for infer_result in infer_results: progress += infer_result.progress total_progress = progress / num_single_inferences compound_infer_record.progress = total_progress write_record(compound_infer_record, session) session.close() return total_progress
def rename_mxnet_files(artifact_id: int): model = TopologiesModel.query.get(artifact_id) if model and model.framework == SupportedFrameworksEnum.mxnet: files = model.files for file in files: old_path = Path(file.path) new_name = model.name + { '.params': '-00001.params', '.json': '-symbol.json' }[old_path.suffix] new_path = old_path.parent / new_name os.rename(str(old_path), str(new_path)) file.path = str(new_path) file.name = new_name write_record(file, get_db().session)
def write_chunk(upload_id, request): file_record = FilesModel.query.get(upload_id) artifact = file_record.artifact chunk = request.files['file'].stream.read() file_name = os.path.join(artifact.path, file_record.name) with open(file_name, "ab") as file: file.write(chunk) if file_record.uploaded_blob_size: file_record.uploaded_blob_size += len(chunk) else: file_record.uploaded_blob_size = len(chunk) file_record.progress = file_record.uploaded_blob_size / file_record.size * 100 write_record(file_record, get_db().session)
def update_inference_result(self, results): for job in self.jobs: job.progress = 100 self.set_inference_result_to_record(results) if self.total_progress >= 100: session = CeleryDBAdapter.session() job = session.query(CompoundInferenceJobsModel).get(self.job_id) infer_results = session.query(InferenceResultsModel).filter_by( job_id=self.job_id).all() for infer_result in infer_results: infer_result.status = StatusEnum.ready write_record(infer_result, session) job.status = StatusEnum.ready write_record(job, session) session.close() self.emit_message()
def set_model_advanced_configuration(model_id: int): config = request.get_json() try_load_configuration(config) model = TopologiesModel.query.get(model_id) if not model: return 'Model with id {} was not found in the database'.format( model_id), 404 model.meta.task_type = config['taskType'] model.meta.topology_type = config['taskMethod'] model.meta.advanced_configuration = json.dumps(config) write_record(model, get_db().session) return jsonify(model.short_json())