def new(self, name: str = None, model: str = None, train: str = None, ds: str = None, path: str = None, pretrained: str = None, skip_upload=False, lightweight=False, **kwargs): if path is None: raise NhaAPIError( "Cannot publish model version if path to model files is not provided" ) model = Model.find_one(name=model) if lightweight: model.assert_movers_can_be_lightweight() if ds is not None: kwargs['ds'] = Dataset.find_one(name=ds, model=model).to_embedded() if train is not None: if self.proj is None: raise NhaAPIError( "Cannot determine parent training if no working project is set" ) else: kwargs['train'] = Training.find_one( name=train, proj=self.proj.name).to_embedded() if pretrained is not None: kwargs['pretrained'] = ModelVersion.find_by_pk( pretrained).to_embedded() LOG.info( "Model version used pre-trained model '{}'".format(pretrained)) mv: ModelVersion = super().new(name=name, model=model, lightweight=lightweight, **kwargs, _duplicate_filter=dict(name=name, model=model)) barrel = None try: if not skip_upload: barrel = self._store(mv, path) except Exception as e: LOG.warn("Reverting creation of model version '{}'".format( mv.name)) mv.delete() if barrel is not None: barrel.purge(ignore=True) raise e return mv
def new(self, repo=None, models: list = None, home_dir: str = None, **kwargs): if models: finder = Model().find_one models = [finder(name=model_name) for model_name in models] else: models = [] LOG.warn( "No models specified for the new project. " "When publishing a model version this project must specify its model name." ) if home_dir is None: LOG.warn("No home directory was specified for the new project.") if self._decide( "Would you like to use the current working directory?", default=False): home_dir = os.getcwd() LOG.info("Using as home directory: {}".format(home_dir)) return super().new(home_dir=None if home_dir is None else LocalRepository(home_dir).address, models=models, **kwargs)
def new(self, name: str = None, model: str = None, path: str = None, files: dict = None, skip_upload=False, lightweight=False, **kwargs): model = Model.find_one(name=model) if lightweight: model.assert_datasets_can_be_lightweight() barrel = None ds = super().new(name=name, model=model, lightweight=lightweight, **kwargs, _duplicate_filter=dict(name=name, model=model)) try: if not skip_upload: barrel = self._store(ds, path, files) except Exception as e: LOG.warn("Reverting creation of dataset '{}'".format(ds.name)) ds.delete() if barrel is not None: barrel.purge(ignore=True) raise e else: return ds
def _store(self, mv: ModelVersion, path: str = None): barrel = MoversBarrel(mv) if barrel.schema is None: LOG.warn( "Publishing model version '{}' without a strict file definition" .format(mv.get_pk())) barrel.store_from_path(path) return barrel
def new(self, **kwargs): if not kwargs.get('model_files'): LOG.warn( "Creating model without a strict model persistence definition") if not kwargs.get('data_files'): LOG.warn( "Creating model without a strict dataset files definition") return super().new(**kwargs)
def push_image(self): if self.img_spec.pushable: LOG.info("Pushing {}".format(self.img_spec.target)) log = self.docker.push(self.img_spec.repo, tag=self.img_spec.tag) outcome = json.loads(Regex.LINE_BREAK.split(log.strip())[-1]) if 'error' in outcome: raise NhaDockerError( "Failed to push image '{}'. Error: {}".format( self.img_spec.target, outcome.get('errorDetail'))) else: LOG.warn("Docker registry is not configured. Skipping image push.")
def __call__(self, src_path: str = Paths.TMP, details: dict = None, version_name: str = None, model_name: str = None, uses_dataset: bool = True, dataset_name: str = None, uses_pretrained: bool = False, pretrained_with: str = None, lightweight: bool = False): version_name = version_name or self.train.name model_name = self._infer_parent_model(model_name) ds = self._infer_dataset(model_name, uses_dataset, dataset_name) mv = None err = None try: mv = self.mv_api.new(name=version_name, model=model_name, ds=ds.name if ds else None, train=self.train.name, path=src_path, details=details or {}, pretrained=self._infer_pretrained( uses_pretrained, pretrained_with), lightweight=lightweight, _replace=True) except Exception as e: LOG.warn("Model version {}:{} publish failed".format( model_name, version_name)) err = e if self.train.name: self.train.update(mover=mv, ds=ds) if err: raise err if get_purpose() == DockerConst.Section.IDE: LOG.info( "For testing purposes, model files will be moved to the deployed model path" ) MetaCargo(docs=[mv], section=DockerConst.Section.IDE).deploy() MoversCargo(mv, local=True, section=DockerConst.Section.IDE).move(src_path) return mv
def clean(self): if not isinstance(self, EmbeddedDocument): self.modified = datetime.now() if hasattr(self, 'name') and self.name is None: name = random_name.generate_name(separator='-') setattr(self, 'name', name) LOG.warn("{} is anonymous. Using random name: {}".format( self.__class__.__name__, name)) for key, val in self._fields.items(): if isinstance(val, ReferenceField): getattr(self, key) # assure that referenced document exists self._id = self.get_pk()
def rm(self, name): LOG.warn( "All datasets and model versions for the model '{}' will be deleted" .format(name)) self._decide("Would you like to proceed?", interrupt=True, default=True) report = {'Removed Datasets': [], 'Removed ModelVersions': []} for key, api in zip(report.keys(), [DatasetAPI(), ModelVersionAPI()]): for obj in api.lyst(model=name): api.rm(model=name, name=obj.name) report[key].append(obj.name) report.update(super().rm(name=name)) return report
def update(self, name, **kwargs): if kwargs.get('model_files'): LOG.warn( "If 'model_files' definition is changed, old model versions may become unusable" ) self._decide("Do you want to proceed?", default=True, interrupt=True) if kwargs.get('data_files'): LOG.warn( "If 'data_files' definition is changed, old datasets may become unusable" ) self._decide("Do you want to proceed?", default=True, interrupt=True) return super().update(filter_kwargs=dict(name=name), update_kwargs=kwargs)
def _store(self, ds: Dataset, path: str = None, files: dict = None): if path or files: # either is not None barrel = DatasetBarrel(ds) if barrel.schema is None: LOG.warn( "Publishing dataset '{}' without a strict file definition". format(ds.get_pk())) if path: barrel.store_from_path(path) elif files: barrel.store_from_dict(files) else: raise NotImplementedError() return barrel else: LOG.warn( "Dataset '{}' for model '{}' is not being stored by the framework" .format(ds.name, ds.model.name)) ds.update(stored=False) return None