def new(self,
name: str = None,
model: str = None,
train: str = None,
ds: str = None,
path: str = None,
pretrained: str = None,
skip_upload=False,
lightweight=False,
**kwargs):
if path is None:
raise NhaAPIError(
"Cannot publish model version if path to model files is not provided"
)
model = Model.find_one(name=model)
if lightweight:
model.assert_movers_can_be_lightweight()
if ds is not None:
kwargs['ds'] = Dataset.find_one(name=ds, model=model).to_embedded()
if train is not None:
if self.proj is None:
raise NhaAPIError(
"Cannot determine parent training if no working project is set"
)
else:
kwargs['train'] = Training.find_one(
name=train, proj=self.proj.name).to_embedded()
if pretrained is not None:
kwargs['pretrained'] = ModelVersion.find_by_pk(
pretrained).to_embedded()
LOG.info(
"Model version used pre-trained model '{}'".format(pretrained))
mv: ModelVersion = super().new(name=name,
model=model,
lightweight=lightweight,
**kwargs,
_duplicate_filter=dict(name=name,
model=model))
barrel = None
try:
if not skip_upload:
barrel = self._store(mv, path)
except Exception as e:
LOG.warn("Reverting creation of model version '{}'".format(
mv.name))
mv.delete()
if barrel is not None:
barrel.purge(ignore=True)
raise e
return mv
def new(self,
repo=None,
models: list = None,
home_dir: str = None,
**kwargs):
if models:
finder = Model().find_one
models = [finder(name=model_name) for model_name in models]
else:
models = []
LOG.warn(
"No models specified for the new project. "
"When publishing a model version this project must specify its model name."
)
if home_dir is None:
LOG.warn("No home directory was specified for the new project.")
if self._decide(
"Would you like to use the current working directory?",
default=False):
home_dir = os.getcwd()
LOG.info("Using as home directory: {}".format(home_dir))
return super().new(home_dir=None if home_dir is None else
LocalRepository(home_dir).address,
models=models,
**kwargs)
def new(self,
name: str = None,
model: str = None,
path: str = None,
files: dict = None,
skip_upload=False,
lightweight=False,
**kwargs):
model = Model.find_one(name=model)
if lightweight:
model.assert_datasets_can_be_lightweight()
barrel = None
ds = super().new(name=name,
model=model,
lightweight=lightweight,
**kwargs,
_duplicate_filter=dict(name=name, model=model))
try:
if not skip_upload:
barrel = self._store(ds, path, files)
except Exception as e:
LOG.warn("Reverting creation of dataset '{}'".format(ds.name))
ds.delete()
if barrel is not None:
barrel.purge(ignore=True)
raise e
else:
return ds
def _store(self, mv: ModelVersion, path: str = None):
barrel = MoversBarrel(mv)
if barrel.schema is None:
LOG.warn(
"Publishing model version '{}' without a strict file definition"
.format(mv.get_pk()))
barrel.store_from_path(path)
return barrel
def new(self, **kwargs):
if not kwargs.get('model_files'):
LOG.warn(
"Creating model without a strict model persistence definition")
if not kwargs.get('data_files'):
LOG.warn(
"Creating model without a strict dataset files definition")
return super().new(**kwargs)
def push_image(self):
if self.img_spec.pushable:
LOG.info("Pushing {}".format(self.img_spec.target))
log = self.docker.push(self.img_spec.repo, tag=self.img_spec.tag)
outcome = json.loads(Regex.LINE_BREAK.split(log.strip())[-1])
if 'error' in outcome:
raise NhaDockerError(
"Failed to push image '{}'. Error: {}".format(
self.img_spec.target, outcome.get('errorDetail')))
else:
LOG.warn("Docker registry is not configured. Skipping image push.")
def __call__(self,
src_path: str = Paths.TMP,
details: dict = None,
version_name: str = None,
model_name: str = None,
uses_dataset: bool = True,
dataset_name: str = None,
uses_pretrained: bool = False,
pretrained_with: str = None,
lightweight: bool = False):
version_name = version_name or self.train.name
model_name = self._infer_parent_model(model_name)
ds = self._infer_dataset(model_name, uses_dataset, dataset_name)
mv = None
err = None
try:
mv = self.mv_api.new(name=version_name,
model=model_name,
ds=ds.name if ds else None,
train=self.train.name,
path=src_path,
details=details or {},
pretrained=self._infer_pretrained(
uses_pretrained, pretrained_with),
lightweight=lightweight,
_replace=True)
except Exception as e:
LOG.warn("Model version {}:{} publish failed".format(
model_name, version_name))
err = e
if self.train.name:
self.train.update(mover=mv, ds=ds)
if err:
raise err
if get_purpose() == DockerConst.Section.IDE:
LOG.info(
"For testing purposes, model files will be moved to the deployed model path"
)
MetaCargo(docs=[mv], section=DockerConst.Section.IDE).deploy()
MoversCargo(mv, local=True,
section=DockerConst.Section.IDE).move(src_path)
return mv
def clean(self):
if not isinstance(self, EmbeddedDocument):
self.modified = datetime.now()
if hasattr(self, 'name') and self.name is None:
name = random_name.generate_name(separator='-')
setattr(self, 'name', name)
LOG.warn("{} is anonymous. Using random name: {}".format(
self.__class__.__name__, name))
for key, val in self._fields.items():
if isinstance(val, ReferenceField):
getattr(self, key) # assure that referenced document exists
self._id = self.get_pk()
def rm(self, name):
LOG.warn(
"All datasets and model versions for the model '{}' will be deleted"
.format(name))
self._decide("Would you like to proceed?",
interrupt=True,
default=True)
report = {'Removed Datasets': [], 'Removed ModelVersions': []}
for key, api in zip(report.keys(), [DatasetAPI(), ModelVersionAPI()]):
for obj in api.lyst(model=name):
api.rm(model=name, name=obj.name)
report[key].append(obj.name)
report.update(super().rm(name=name))
return report
def update(self, name, **kwargs):
if kwargs.get('model_files'):
LOG.warn(
"If 'model_files' definition is changed, old model versions may become unusable"
)
self._decide("Do you want to proceed?",
default=True,
interrupt=True)
if kwargs.get('data_files'):
LOG.warn(
"If 'data_files' definition is changed, old datasets may become unusable"
)
self._decide("Do you want to proceed?",
default=True,
interrupt=True)
return super().update(filter_kwargs=dict(name=name),
update_kwargs=kwargs)
def _store(self, ds: Dataset, path: str = None, files: dict = None):
if path or files: # either is not None
barrel = DatasetBarrel(ds)
if barrel.schema is None:
LOG.warn(
"Publishing dataset '{}' without a strict file definition".
format(ds.get_pk()))
if path:
barrel.store_from_path(path)
elif files:
barrel.store_from_dict(files)
else:
raise NotImplementedError()
return barrel
else:
LOG.warn(
"Dataset '{}' for model '{}' is not being stored by the framework"
.format(ds.name, ds.model.name))
ds.update(stored=False)
return None
