def get_routes(self):
new_routes = dict(
update=dict(
func=self._update_route,
methods=['POST']
)
)
return join_dicts(super().get_routes(), new_routes)
def new(self,
name: str = None,
tag=DockerConst.LATEST,
notebook: str = None,
params: dict = None,
details: dict = None,
datasets: list = None,
movers: list = None,
_replace: bool = None,
target_deploy: str = None,
**kwargs):
self.set_logger(name)
bv = BuildVersion.find_one_or_none(tag=tag, proj=self.proj)
movers = [
ModelVersion.find_by_pk(mv).to_embedded() for mv in movers or []
]
datasets = [Dataset.find_by_pk(ds) for ds in datasets or []]
if name is None:
all_names = set([ds.name for ds in datasets])
if len(all_names) == 1:
name = all_names.pop()
for mv in movers:
self.LOG.info(
"Pre-trained model '{}' will be available in this training".
format(mv.show()))
train: Training = super().new(
name=name,
proj=self.proj,
bvers=None if bv is None else bv.to_embedded(),
notebook=assert_extension(notebook, Extension.IPYNB),
details=join_dicts(details or {},
dict(params=params or {}),
allow_overwrite=False),
_duplicate_filter=dict(name=name, proj=self.proj))
exp = TrainingExp(train=train,
tag=tag,
datasets=datasets,
movers=movers,
resource_profile=kwargs.pop('resource_profile',
None),
log=self.LOG)
exp.launch(**kwargs)
train.reload()
if target_deploy is not None and train.task.state == Task.State.FINISHED:
self._update_deploy(target_deploy, train)
self.reset_logger()
return train.reload()
def kube_resources(self):
if self.resources is None:
return None
res = {}
for key in ['requests', 'limits']:
res[key] = dict(cpu=self.resources[key]['cpu'],
memory=self.kube_memory(
self.resources[key]['memory']))
if self.resources.get('enable_gpu', False):
res['limits'] = join_dicts(res['limits'], {"nvidia.com/gpu": 1})
return res
def new(self,
name: str = None,
tag=DockerConst.LATEST,
notebook: str = 'predict',
details: dict = None,
params: dict = None,
movers: list = None,
port: int = None,
_replace: bool = None,
**kwargs):
self.set_logger(name)
bv = BuildVersion.find_one_or_none(tag=tag, proj=self.proj)
depl = super().new(name=name,
proj=self.proj,
movers=[
ModelVersion.find_by_pk(mv).to_embedded()
for mv in movers or []
],
bvers=None if bv is None else bv.to_embedded(),
notebook=assert_extension(notebook,
Extension.IPYNB),
details=join_dicts(details or {},
dict(params=params or {}),
allow_overwrite=False),
replicas=kwargs.get('tasks'),
host_port=port,
_duplicate_filter=dict(name=name, proj=self.proj))
# TODO: check consistensy of project, docker tag and git version between depl.bvers and movers.bvers
exp = DeploymentExp(depl,
port,
tag,
resource_profile=kwargs.pop(
'resource_profile', None),
log=self.LOG)
exp.launch(**kwargs)
if port is None:
node_port = exp.captain.get_node_port(exp.make_name())
depl.reload()
depl.update(host_port=node_port)
self.reset_logger()
return depl
def launch(self,
env_vars: dict = None,
mounts: list = None,
ports: list = None,
**kwargs):
self.launcher(
name=self.make_name(),
img=self.img_spec,
env_vars=join_dicts(env_vars or {},
self.make_env_vars(),
allow_overwrite=False),
mounts=(mounts or []),
cargos=self.cargos,
ports=(ports or []) + self.make_ports(),
cmd=DockerConst.HANG_CMD if self.mock else self.make_cmd(),
**self.additional_launch_kwargs,
**kwargs)
return True
def load(self):
default_conf, user_conf, local_conf = [
None if not os.path.exists(src) else Kaptan(
handler=Config.FMT).import_config(src) for src in self.sources
]
assert default_conf is not None,\
ConfigurationError("Default configuration not found at {}".format(self.sources[0]))
conf_opts = [local_conf, user_conf,
{}] # if local conf was found, user conf is ignored
child_conf = filter(
lambda x: x is not None,
conf_opts).__next__() # returns first non empty option
super().__init__(**join_dicts(default_conf.get(self.namespace, {}),
child_conf.get(self.namespace, {}),
allow_overwrite=True))
return self
def make_env_vars(self):
return join_dicts(super().make_env_vars(), {EnvVar.OPEN_SEA: 'Yes'})
def get_config(self):
return join_dicts(self.compass.get_extra_conf(), self.model_conf, allow_overwrite=True)