def _translate_pkg_to_id(self, pkgtype, pkg):
uai_logger.info("Start download {0} package info".format(pkgtype))
api_op = GetUAITrainEnvPkgAPIOp(self.pub_key,
self.pri_key,
pkgtype,
self.project_id,
self.region,
self.zone)
succ, result = api_op.call_api()
if succ is False:
raise RuntimeError("Error get {0} info from server".format(pkgtype))
for avpkg in result['PkgSet']:
if pkgtype == 'OS' or pkgtype == 'Python' or pkgtype == 'AIFrame':
versionsplit = pkg.rfind('-')
if versionsplit > 0:
if avpkg["PkgName"] == pkg[:versionsplit] and (
avpkg["PkgVersion"] == "" or avpkg["PkgVersion"] == pkg[versionsplit + 1:]):
return avpkg["PkgId"]
elif versionsplit < 0:
if avpkg["PkgName"] == pkg:
return avpkg["PkgId"]
else:
if avpkg["PkgName"] == pkg:
return avpkg["PkgId"]
uai_logger.error("Some {0} package is not supported: {1}".format(pkgtype, pkg))
raise RuntimeError("Some {0} package is not supported: {1}".format(pkgtype, pkg))
return None
def _delete_request(self):
private_key = self.conf_params['private_key']
task_id = self.delete_params['TaskId']
self.delete_params['Signature'] = _verfy_ac(private_key,
self.delete_params)
uai_logger.debug('==' * 10)
uai_logger.info(
"Start to make delete task {0} request".format(task_id))
r = requests.get(self.delete_url, params=self.delete_params)
if r.status_code != requests.codes.ok:
uai_logger.debug('================================')
uai_logger.warning('Delete Error, PLS check your connection')
sys.exit(0)
json_data = json.loads(r.text)
r_info = {}
for k, v in json_data.items():
if isinstance(k, unicode):
k = k.encode('utf-8')
if isinstance(v, unicode):
v = v.encode('utf-8')
r_info[k] = v
if check_retcode(r_info) != NORMAL_CONDITION:
uai_logger.error('Delete Error: {0}'.format(r_info['Message']))
sys.exit(0)
else:
uai_logger.info('Delete task id {0} successful'.format(task_id))
def _upload_code(self):
""" Upload the all code files
"""
code_files_list = self.conf_params['docker']['ufile']['code_files']
for file in code_files_list:
self._upload_file(file)
uai_logger.info('Upload code {0} successful'.format(code_files_list))
def _translate_pkg_to_id(self, pkgtype, pkglist):
resultlist = []
uai_logger.info(
"Start translate {0} package to their id, packages: {1}".format(
pkgtype, pkglist))
for avpkg in self.rsp['PkgSet']:
for pkg in pkglist:
if pkgtype == 'os' or pkgtype == 'python_v' or pkgtype == 'ai_arch_v':
versionsplit = pkg.rfind('-')
if versionsplit >= 0:
if avpkg["PkgName"] == pkg[:versionsplit] and (
avpkg["PkgVersion"] == "" or
avpkg["PkgVersion"] == pkg[versionsplit + 1:]):
pkglist.remove(pkg)
resultlist.append(avpkg["PkgId"])
elif versionsplit < 0:
if avpkg["PkgName"] == pkg:
pkglist.remove(pkg)
resultlist.append(avpkg["PkgId"])
else:
if avpkg["PkgName"] == pkg:
pkglist.remove(pkg)
resultlist.append(avpkg["PkgId"])
if len(pkglist) != 0:
uai_logger.error("Some {0} package is not supported: {1}".format(
pkgtype, pkglist))
raise RuntimeError("Some {0} package is not supported: {1}".format(
pkgtype, pkglist))
uai_logger.info(
"End translate {0} package to their id, result: {1}".format(
pkgtype, resultlist))
return resultlist
def _push_userimage(self):
uai_logger.info("Push user image")
retcode = subprocess.check_call(["docker", "push", self.userimage],
stderr=subprocess.STDOUT)
if retcode != 0:
raise RuntimeError(
"Error push image {0}, Please check your network".format(
self.userimage))
def _upload_model(self):
""" Upload the all model files in model dir
"""
model_dir = self.conf_params['docker']['ufile']['model_dir']
model_files_list = self._get_all_files(model_dir)
for model_file in model_files_list:
self._upload_file(model_file)
uai_logger.info('Upload model {0} successful'.format(model_dir))
def _build_gpu_userimage(self):
'''
Build actual image for training
'''
uai_logger.info("Build training docker image")
uai_logger.info("Pull base image from " + self.dcoker_register)
retcode = subprocess.check_call(["docker", "pull", self.acc_image],
stderr=subprocess.STDOUT)
if retcode != 0:
raise RuntimeError("Error pull image: {0}, Please check your network".format(self.acc_image))
uai_logger.info("Create GPU Dockerfile")
dockerbuf = []
dockerbuf.append("From " + self.acc_image + "\n")
dockerbuf.append("ADD " + "./" + self.code_path + " /data/\n")
with open(TMP_DOCKER_FILE, 'w') as f:
f.write(''.join(dockerbuf))
uai_logger.info("Build user image")
print(self.dcoker_register)
userimage = self.dcoker_register + "/" + self.uhub_registry + "/" + self.uhub_imagename
if self.uhub_imagetag != None and self.uhub_imagetag != "":
userimage = userimage + ":" + self.uhub_imagetag
else:
userimage = userimage + ":" + DOCKER_TAG_SUFFIX
retcode = subprocess.check_call(["docker", "build", "-t", userimage, "-f", TMP_DOCKER_FILE, "."],
stderr=subprocess.STDOUT)
if retcode != 0:
raise RuntimeError("Error build image: {0}, Please retry".format(userimage))
print(userimage)
self.user_gpu_image = userimage
def _build_userimage(self):
uai_logger.info("Docker login on " + self.dcoker_register)
retcode = subprocess.check_call(
["docker", "login", "-u", self.uhub_username, "-p", self.uhub_password, self.dcoker_register],
stderr=subprocess.STDOUT)
if retcode != 0:
raise RuntimeError("Error login to uhub, Please check your username and password, or try with sudo")
self._build_cpu_userimage()
if self.acc_id_name == 'gpu':
self._build_gpu_userimage()
self._push_gpu_userimage()
self._gen_run_cmd()
def _gen_jsonfile(self):
""" Reformat the conf params and generate jsonfile
"""
uai_logger.info('Generating uflie.json in filepath: {0}'.format(
self.params['pack_file_path']))
self.conf_params['docker']['ufile']['files'] = [
self.params['upload_name']
]
self.conf_params['docker']['ufile'].pop('code_files')
self.conf_params['docker']['ufile'].pop('model_dir')
self.conf_params['docker']['ufile'].pop('upload_name')
# self.conf_params['docker']['ufile'].pop('upload_prefix')
with open(os.path.join(self.params['pack_file_path'], 'ufile.json'),
'w') as f:
json.dump(self.conf_params, f)
def _cmd_common_request(self):
if 'Signature' in self.cmd_params:
self.cmd_params.pop('Signature')
self.cmd_params['Signature'] = _verfy_ac(self.private_key,
self.cmd_params)
uai_logger.info("Call http request: {0} ".format(get_request(self.cmd_url, params=self.cmd_params)))
r = requests.get(self.cmd_url, params=self.cmd_params)
print(r.text)
self.rsp = json.loads(r.text, encoding="utf-8")
if self.rsp["RetCode"] != 0:
uai_logger.error("{0} Fail: [{1}]{2}".format(self.cmd_params["Action"], self.rsp["RetCode"],
self.rsp["Message"].encode('utf-8')))
return False
else:
del self.rsp['Action']
uai_logger.info("{0} Success: {1}".format(self.cmd_params["Action"], get_response(self.rsp, 0)))
return True
def _translate_pkg_to_id(self, pkgtype, pkglist):
if not os.path.exists(pkgtype):
raise RuntimeError(
"{0} file doesn't found, please download from github "
"and put it under the same directory as deploy tool".format(
pkgtype))
# uai_logger.info("Start download {0} package info".format(pkgtype))
# self.conf_params['pkg_type'] = pkgtype
# self._format_availableenv_param()
# self.cmd_url = UCLOUD_API_URL
# self._cmd_writefile_package(pkgtype)
resultlist = []
uai_logger.info(
"Start translate {0} package to their id, packages: {1}".format(
pkgtype, pkglist))
for avpkg in json.load(open(pkgtype), 'utf-8'):
for pkg in pkglist:
if pkgtype == 'os' or pkgtype == 'language' or pkgtype == 'ai_arch':
versionsplit = pkg.rfind('-')
if versionsplit >= 0:
if avpkg["PkgName"] == pkg[:versionsplit] and (
avpkg["PkgVersion"] == "" or
avpkg["PkgVersion"] == pkg[versionsplit + 1:]):
pkglist.remove(pkg)
resultlist.append(avpkg["PkgId"])
elif versionsplit < 0:
if avpkg["PkgName"] == pkg:
pkglist.remove(pkg)
resultlist.append(avpkg["PkgId"])
else:
if avpkg["PkgName"] == pkg:
pkglist.remove(pkg)
resultlist.append(avpkg["PkgId"])
if len(pkglist) != 0:
uai_logger.error("Some {0} package is not supported: {1}".format(
pkgtype, pkglist))
raise RuntimeError("Some {0} package is not supported: {1}".format(
pkgtype, pkglist))
uai_logger.info(
"End translate {0} package to their id, result: {1}".format(
pkgtype, resultlist))
return resultlist
def _check_request(self):
max_poll_steps = MAX_POLL_STEPS
uai_logger.debug('=' * 10 + 'Start poll the state of deploy' +
'=' * 10)
uai_logger.debug('the url to check the state of deploy: {0}'.format(
self.check_url))
self.check_params['Signature'] = _verfy_ac(
self.request_params['PrivateKey'], self.check_params)
for step in range(max_poll_steps):
uai_logger.debug(
'the step {0} to poll the state of the deploy'.format(step))
r = requests.get(self.check_url, params=self.check_params)
uai_logger.debug('the result of step {0}'.format(step))
if r.status_code != requests.codes.ok:
uai_logger.warning('================================')
uai_logger.warning('Check Error, PLS check your connection')
break
json_data = json.loads(r.text)
r_info = {}
for k, v in json_data.items():
if isinstance(k, unicode):
k = k.encode('utf-8')
if isinstance(v, unicode):
v = v.encode('utf-8')
r_info[k] = v
if check_retcode(r_info) != NORMAL_CONDITION:
uai_logger.warning('Deploy Error: {0}'.format(
r_info['Message']))
break
else:
if r_info['Status'] == 'SUCCESS':
uai_logger.info(
'Building Success, the domain of your deploy application is {0}'
.format(r_info['Domain']))
self.id_file.write('URL: {0}\n'.format(r_info['Domain']))
break
elif r_info['Status'] == 'BUILDING':
uai_logger.info('Building Image Now')
time.sleep(5)
continue
elif r_info['Status'] == 'PUSHIMAGE':
uai_logger.info('Pushing Image Now')
time.sleep(5)
continue
elif r_info['Status'] == 'PAASDEPLOY':
uai_logger.info('Creating App On PAAS Now')
time.sleep(5)
continue
elif r_info['Status'] == 'FAILED':
uai_logger.error('Error Message: {0} '.format(
r_info['Message']))
break
def _cmd_writefile_package(self, filepath):
if ('Signature' in self.cmd_params) is True:
self.cmd_params.pop('Signature')
self.cmd_params['Signature'] = _verfy_ac(
self.conf_params['private_key'], self.cmd_params)
uai_logger.info("Call http request: {0} ".format(
get_request(self.cmd_url, params=self.cmd_params)))
r = requests.get(self.cmd_url, params=self.cmd_params)
rsp = json.loads(r.text, encoding='utf-8')
if rsp["RetCode"] != 0:
uai_logger.error("{0} Fail: [{1}]{2}".format(
self.cmd_params["Action"], rsp["RetCode"],
rsp["Message"].encode('utf-8')))
raise RuntimeError("{0} Fail: [{1}]{2}".format(
self.cmd_params["Action"], rsp["RetCode"],
rsp["Message"].encode('utf-8')))
else:
with open(filepath, 'w') as f:
json.dump(rsp["PkgSet"], f)
def _upload_file(self, file_path):
""" Upload the file to the ufile
"""
public_key = self.conf_params['docker']['ufile']['public_key']
private_key = self.conf_params['docker']['ufile']['private_key']
bucket = self.conf_params['docker']['ufile']['bucket']
uai_logger.debug('Start upload file to the bucket {0}'.format(
self.conf_params['docker']['ufile']['bucket']))
handler = putufile.PutUFile(public_key, private_key)
local_file = file_path
key = local_file
uai_logger.info('Upload >> key: {0}, local file: {1}'.format(
key, local_file))
ret, resp = handler.putfile(bucket, key, local_file)
assert resp.status_code == 200, 'upload seems something error'
uai_logger.debug(
'upload local file :{0} to ufile key= {1} successful'.format(
local_file, key))
self.files.append(key)
def _pack_file(self):
uai_logger.info('Start packing files in the target tar path.')
os.chdir(os.path.join(os.getcwd(), self.pack_file_path))
tar = tarfile.open(self.tar_name, 'w')
for i in self.filelist:
try:
tar.add(i)
except OSError as e:
uai_logger.info('{0} : {1}'.format(OSError, e))
uai_logger.info('The package process is interrupted.')
sys.exit(0)
tar.close()
uai_logger.info('Finish packing the files.')
def _pack_file(self):
""" Pack files in the target path
"""
uai_logger.info('Start packing files in the target tar path.')
os.chdir(self.params['pack_file_path'])
tar = tarfile.open(self.params['upload_name'], 'w')
for i in self.filelist:
try:
tar.add(i)
except OSError as e:
uai_logger.info('{0} : {1}'.format(OSError, e))
uai_logger.info('The package process is interrupted.')
sys.exit(0)
uai_logger.info('Finish packing the files.')
def _upload_to_ufile(self):
public_key = self.public_key
private_key = self.private_key
bucket = self.bucket
uai_logger.debug('Start upload file to the bucket {0}'.format(bucket))
handler = putufile.PutUFile(public_key, private_key)
local_file = os.path.join(CURRENT_PATH, self.pack_file_path,
self.upload_name)
local_file = local_file.replace('\\', '/')
key = self.upload_name
uai_logger.info('Upload >> key: {0}, local file: {1}'.format(
key, local_file))
ret, resp = handler.putfile(bucket, key, local_file)
uai_logger.debug('Ufile response: {0}'.format(resp))
assert resp.status_code == 200, 'upload seems something error'
uai_logger.debug(
'upload local file :{0} to ufile key= {1} successful'.format(
local_file, key))
current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
self.ufile_info.append('Upload Time: {0}\n'.format(current_time))
self.ufile_info.append('Path of Upload File: {0}\n'.format(key))
def _upload_file(self):
""" Upload tar file to certain bucket
"""
public_key = self.conf_params['docker']['ufile']['public_key']
private_key = self.conf_params['docker']['ufile']['private_key']
bucket = self.conf_params['docker']['ufile']['bucket']
uai_logger.debug('Start upload file to the bucket {0}'.format(bucket))
handler = putufile.PutUFile(public_key, private_key)
local_file = os.path.join(self.params['pack_file_path'],
self.params['upload_name'])
key = self.params['upload_name']
uai_logger.info('Upload >> key: {0}, local file: {1}'.format(
key, local_file))
ret, resp = handler.putfile(bucket, key, local_file)
uai_logger.debug('Ufile response: {0}'.format(resp))
assert resp.status_code == 200, 'upload seems something error'
uai_logger.debug(
'upload local file :{0} to ufile key= {1} successful'.format(
local_file, key))
current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
self.ufile_info.write('Upload Time: {0}\n'.format(current_time))
self.ufile_info.write('Path of Upload File: {0}\n'.format(key))
def _bulid_userimage(self):
uai_logger.info("Docker login on " + self.docker_register)
retcode = subprocess.check_call([
"docker", "login", "-u", self.uhub_username, "-p",
self.uhub_password, self.docker_register
],
stderr=subprocess.STDOUT)
if retcode != 0:
raise RuntimeError(
"Error login to uhub, Please check your username and password, or try with sudo"
)
# uai_logger.info("Pull base image from " + self.docker_register)
# retcode = subprocess.check_call(["docker", "pull", self.baseimage], stderr=subprocess.STDOUT)
# if retcode != 0:
# raise RuntimeError("Error pull image: {0}, Please check your network".format(self.baseimage))
uai_logger.info("Create Dockerfile")
dockerbuf = []
dockerbuf.append("From " + self.baseimage + "\n")
if self.platform == 'caffe':
dockerbuf.append("ENV PYTHONPATH /root/caffe/python\n")
dockerbuf.append("EXPOSE 8080\n")
dockerbuf.append("ADD " + "./" + self.tar_name +
" /ai-ucloud-client-django/\n")
dockerbuf.append(
"ENV UAI_SERVICE_CONFIG /ai-ucloud-client-django/ufile.json\n")
dockerbuf.append(
"CMD cd /ai-ucloud-client-django && gunicorn -c gunicorn.conf.py httpserver.wsgi"
+ "\n")
with open(TMP_DOCKER_FILE, 'w') as f:
f.write(''.join(dockerbuf))
'''
Build user image
'''
uai_logger.info("Build user image")
userimage = self.docker_register + "/" + self.uhub_registry + "/" + self.uhub_imagename
if self.uhub_imagetag == '':
# userimage = userimage + ":" + self.uhub_imagetag
# else:
userimage = userimage + ":" + DOCKER_TAG_SUFFIX
retcode = subprocess.check_call(
["docker", "build", "-t", userimage, "-f", TMP_DOCKER_FILE, "."],
stderr=subprocess.STDOUT)
if retcode != 0:
raise RuntimeError(
"Error build image: {0}, Please retry".format(userimage))
self.userimage = userimage
def _cmd_common_request(self):
if ('Signature' in self.cmd_params) is True:
self.cmd_params.pop('Signature')
self.cmd_params['Signature'] = _verfy_ac(self.priv_key,
self.cmd_params)
uai_logger.info("Signature: {0}".format(self.cmd_params['Signature']))
uai_logger.info(self.cmd_params)
uai_logger.info("Call http request: {0} ".format(
get_request(self.cmd_url, params=self.cmd_params)))
r = requests.get(self.cmd_url, params=self.cmd_params)
rsp = json.loads(r.text, encoding='utf-8')
if rsp['RetCode'] != 0:
uai_logger.error("{0} Fail: [{1}]{2}".format(
self.cmd_params[PARAM_ACTION], rsp['RetCode'],
rsp['Message'].encode('utf-8')))
return False, rsp
else:
del rsp[PARAM_ACTION]
#uai_logger.info("{0} Success: {1}".format(self.cmd_params[PARAM_ACTION], get_response(rsp, 0)))
return True, rsp
"modifynodecount": modify_node_count_op,
"start": start_op,
"stop": stop_op,
"tar": tar_op_dic,
"pack": pack_op_dic,
"packdocker": docker_pack_op_dic,
}
return cmd_op_dic
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='UAI Inference Platform Commander',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
subparser = parser.add_subparsers(dest='commands', help='commands')
cmd_op_dic = parse_args(subparser)
cmd_args = param_filter(vars(parser.parse_args()))
uai_logger.info("cmd_args: {0}".format(cmd_args))
if cmd_args['commands'] == 'packdocker':
cmd_op_dic.get('packdocker').get(
cmd_args['ai_arch_type']).cmd_run(cmd_args)
elif cmd_args['commands'] == 'pack':
cmd_op_dic.get('pack').get(cmd_args['ai_arch_type']).cmd_run(cmd_args)
elif cmd_args['commands'] == 'tar':
cmd_op_dic.get('tar').get(cmd_args['ai_arch_type']).cmd_run(cmd_args)
else:
cmd_op_dic.get(cmd_args['commands']).cmd_run(cmd_args)
def _make_request(self):
self.request_params['Signature'] = _verfy_ac(
self.request_params['PrivateKey'], self.request_params)
uai_logger.info('=====================================')
uai_logger.info('Send AI Service Deploy Request')
uai_logger.debug(self.request_url)
uai_logger.debug(self.request_params)
r = requests.get(self.request_url,
params=self.request_params,
timeout=600)
uai_logger.info('=====================================')
uai_logger.info('Get API Request Response:')
uai_logger.info(r.status_code)
uai_logger.debug('the deploy request url: {0}'.format(r.url))
uai_logger.debug('the all info {0}'.format(r.text))
if r.status_code != requests.codes.ok:
uai_logger.info('Deploy Error! Please Retry')
sys.exit(0)
json_data = json.loads(r.text)
r_info = {}
for k, v in json_data.items():
if isinstance(k, unicode):
k = k.encode('utf-8')
if isinstance(v, unicode):
v = v.encode('utf-8')
r_info[k] = v
self.deploy_response = r_info
uai_logger.info('Start Setting Up, Your application ID is {0}'.format(
r_info['Id']))
self.id_file.write('Id: {0}\n'.format(r_info['Id']))
