def validate(self):
if not self._values or len(self._values) == 0:
raise ArenaException(
ArenaErrorType,
"failed to validate flag {},values are null".format(
self._flag))
return True
def get_logs(self, logger: LoggerBuilder) -> None:
cmds = list()
cmds.append(ARENA_BINARY)
cmds.append("serve")
cmds.append("logs")
cmds.append(self.get_owner())
cmds.append("-T=" + self.get_owner_type().value[0])
cmds.append("--version=" + self.get_owner_version())
for opt in logger.get_args():
cmds.append(opt)
cmds.append("-i=" + self._name)
kubeconfig = os.getenv("KUBECONFIG")
if kubeconfig and kubeconfig != "":
cmds.append("--config=" + kubeconfig)
cmds.append("--namespace=" + self._namespace)
arena_namespace = os.getenv("ARENA_NAMESPACE")
if arena_namespace and arena_namespace != "":
cmds.append("--arena-namespace=" + arena_namespace)
try:
status, stdout, stderr = Command(*cmds).run_with_communicate(
logger.get_accepter())
if status != 0:
raise ArenaException(ArenaErrorType.LogsServingJobError,
stdout + stderr)
except ArenaException as e:
raise e
def submit(self,job: TrainingJob) -> str:
cmds = self.__generate_commands("serve")
cmds.append(job.get_type().value[0])
for opt in job.get_args():
cmds.append(opt)
cmds.append(job.get_command())
try:
status,stdout,stderr = Command(*cmds).run()
if status != 0:
err_message = "the job {} is already exist, please delete it first.".format(job.get_name())
if stdout.find(err_message) != -1 or stderr.find(err_message) != -1:
raise ArenaException(ArenaErrorType.TrainingJobExistError,stdout + stderr)
else:
raise ArenaException(ArenaErrorType.SubmitTrainingJobError,stdout + stderr)
return stdout
except ArenaException as e:
raise e
def build(self) -> ServingJob:
args = list()
try:
for field in self._options:
if not isinstance(field,Field):
raise ArenaException(ArenaErrorType.Unknown,"the type of option {} is not Field".format(field))
field.validate()
for opt in field.options():
args.append(opt)
return ServingJob(self._job_name,self._job_type,self._version,args,self._command)
except ArenaException as e:
raise e
def traffic_router_split(self,job_name: str,job_type: ServingJobType,version_weights: Dict[str,int]):
cmds = self.__generate_commands("serve","traffic-router-split")
cmds.append("--name=" + job_name)
for version,weight in version_weights.items():
cmds.append("-v=" + version + ":" + str(weight))
try:
status,stdout,stderr = Command(*cmds).run()
if status != 0:
raise ArenaException(ArenaErrorType.TrafficRouterSplitServingJobError,stdout + stderr)
return stdout
except ArenaException as e:
raise e
def _filter(self, node_type: NodeType, *node_names: List[str]) -> NodeSet:
cmds = self.__generate_commands("top", "node", "-d", "-o", "json")
if node_type != NodeType.AllNodeType and node_type != NodeType.UnknownNodeType:
cmds.append("-m=" + node_type.value[0])
try:
status, stdout, stderr = Command(*cmds).run()
if status != 0:
raise ArenaException(ArenaErrorType.TopNodeError,
stdout + stderr)
data = json.loads(stdout)
return build_nodes(data)
except ArenaException as e:
raise e
def delete(self, job_name: str, job_type: TrainingJobType) -> str:
cmds = self.__generate_commands("delete")
cmds.append(job_name)
if job_type != TrainingJobType.AllTrainingJob and job_type != TrainingJobType.UnknownTrainingJob:
cmds.append("--type=" + job_type.value)
try:
status, stdout, stderr = Command(*cmds).run()
if status != 0:
raise ArenaException(ArenaErrorType.DeleteTrainingJobError,
stdout + stderr)
return stdout
except ArenaException as e:
raise e
def prune(self, since: str, all_namespaces: bool) -> str:
cmds = self.__generate_commands("prune")
cmds.append("--since=" + since)
if all_namespaces:
cmds.append("-A")
try:
status, stdout, stderr = Command(*cmds).run()
if status != 0:
raise ArenaException(ArenaErrorType.PruneTrainingJobError,
stdout + stderr)
return stdout
except ArenaException as e:
raise e
def scaleout(self, job: TrainingJob) -> str:
cmds = self.__generate_commands("scaleout")
cmds.append(job.get_type().value)
for opt in job.get_args():
cmds.append(opt)
cmds.append(job.get_command())
try:
status, stdout, stderr = Command(*cmds).run()
if status != 0:
raise ArenaException(ArenaErrorType.ScaleoutTrainingJobError,
stdout + stderr)
return stdout
except ArenaException as e:
raise e
def list(self,job_type: ServingJobType,all_namespaces: bool) -> List[TrainingJobInfo]:
def convert(json_object):
job_infos = list()
for obj in json_object:
job_infos.append(generate_serving_job_info(obj))
return job_infos
cmds = self.__generate_commands("list")
if job_type != ServingJobType.AllServingJob and job_type != ServingJobType.UnknownServingJob:
cmds.append("--type=" + job_type.value)
if all_namespaces:
cmds.append("-A")
cmds.append("-o")
cmds.append("json")
try:
status,stdout,stderr = Command(*cmds).run()
if status != 0:
raise ArenaException(ArenaErrorType.ListServingJobError,stdout + stderr)
return json.loads(stdout,object_hook=convert)
except ArenaException as e:
raise e
def get(self,job_name: str,job_type: ServingJobType,version: str) -> TrainingJobInfo:
cmds = self.__generate_commands("serve","get")
cmds.append(job_name)
if job_type != ServingJobType.AllServingJob and job_type != ServingJobType.UnknownServingJob:
cmds.append("--type=" + job_type.value[0])
if version != "":
cmds.append("--version=" + version)
cmds.append("-o")
cmds.append("json")
try:
status,stdout,stderr = Command(*cmds).run()
if status != 0:
err_message = "Not found serving job {}".format(job_name)
if stdout.find(err_message) != -1 or stderr.find(err_message) != -1:
return None
else:
raise ArenaException(ArenaErrorType.GetServingJobError,stdout + stderr)
data = json.loads(stdout)
return generate_serving_job_info(data)
except ArenaException as e:
raise e
def get(self, job_name: str, job_type: TrainingJobType) -> TrainingJobInfo:
cmds = self.__generate_commands("get")
cmds.append(job_name)
if job_type != TrainingJobType.AllTrainingJob and job_type != TrainingJobType.UnknownTrainingJob:
cmds.append("--type=" + job_type.value)
cmds.append("-o")
cmds.append("json")
try:
status, stdout, stderr = Command(*cmds).run()
if status != 0:
err_message = "Not found training job {} in namespace {},please use 'arena submit' to create it.".format(
job_name, self.namespace)
if stdout.find(err_message) != -1 or stderr.find(
err_message) != -1:
return None
else:
raise ArenaException(ArenaErrorType.GetTrainingJobError,
stdout + stderr)
data = json.loads(stdout)
return generate_training_job_info(data)
except ArenaException as e:
raise e
def validate(self):
if not self._flag or self._value == "":
raise ArenaException(
ArenaErrorType.ValidateArgsError,
"failed to validate flag {},value is null".format(self._flag))
return True
