def install_deps(self, args):
package_name = args.package_name
requirements = args.requirements
upgrade = args.upgrade
force_reinstall = args.force_reinstall
pkgs = ' '.join(package_name)
config_path = self.get_model_config_path()
config = get_config_parser(config_path)
section_name = DEFAULT_MODEL_SECTION
model_id = config_get(config.get, section_name, 'model_id')
runtime = config_get(config.get, section_name, 'runtime')
client = self._get_docker_client()
command = ['pip install', '--cache-dir', '/tmp/.cache/']
if upgrade:
command.append('--upgrade')
if force_reinstall:
command.append('--force-reinstall')
deps = []
if pkgs:
deps.append(pkgs)
if requirements:
deps.append('-r {}'.format(requirements))
command.append('-t /code/lib {}'.format(' '.join(deps)))
command = ' '.join(command)
image_name = RUNTIMES[runtime]
self._get_or_pull_image(client, image_name)
with console.status('开始安装模型依赖包...', spinner='earth'):
command = ('sh -c "{}"').format(command)
container_name = generate_container_name(model_id)
self._force_remove_container(client, container_name)
self._run_by_container(client, image_name, command, container_name)
output('[green]安装完成[/green]')
def handler(self, args):
access_token = args.access_token
filepath = args.filepath
project = get_active_project()
oauth2_section = DEFAULT_OAUTH2_SECTION
token_section = DEFAULT_TOKEN_SECTION
config = read_config(project)
if not access_token:
access_token = config_get(config.get, token_section,
'access_token')
endpoint = config_get(config.get, oauth2_section, 'endpoint')
api = API(access_token, endpoint=endpoint, timeout=18.)
if not posixpath.isfile(filepath):
output('[red]只能上传文件[/red]')
sys.exit(1)
filename = args.filename
cmk_id = args.cmk_id
if not filename:
filename = posixpath.split(filepath)[1]
message = '正在上传文件 {}'.format(filepath)
with console.status(message, spinner='earth'):
try:
with open(filepath, 'rb') as fp:
object_name = api.upload(filename, fp, cmk_id=cmk_id)
except APIError as e:
output('[red]请求失败：[/red]')
output_json(e.result)
sys.exit(1)
output('[green]文件上传成功，存储位置：[/green]{}'.format(object_name))
def _get_or_pull_image(self, client, image_name):
try:
client.images.get(image_name)
except docker.errors.NotFound:
output(
'[cyan]本地 docker 镜像 {} 不存在，开始拉取...[/cyan]'.format(image_name))
with console.status('拉取 docker 镜像中...', spinner='earth'):
return_code = os.system('docker pull {}'.format(image_name))
if return_code != 0:
output('[red]拉取镜像 {} 失败，请重试[/red]'.format(image_name))
sys.exit(1)
output('[green]拉取镜像 {} 成功[/green]'.format(image_name))
else:
with console.status('更新 docker 镜像中...', spinner='earth'):
repository, tag = image_name.split(':')
try:
client.images.pull(repository, tag)
except docker.errors.APIError:
output('[red]更新镜像 {} 失败，请重试[/red]'.format(image_name))
sys.exit(1)
output('[green]更新镜像 {} 成功[/green]'.format(image_name))
def _zip_codedir(self, home, ziph):
home = home.rstrip('/')
total_files = {}
zip_relpaths = set()
for root, _, files in os.walk(home):
for file in files:
filepath = join(root, file)
zip_relpath = relpath(filepath, home)
zip_relpaths.add(zip_relpath)
total_files[zip_relpath] = filepath
src_zip_relpaths = zip_relpaths.copy()
ignore_patterns = self.get_ignore_patterns(home)
for ignore_pattern in ignore_patterns:
if ignore_pattern.startswith('!'):
ignore_pattern = ignore_pattern[1:]
zip_relpaths.update(
fnmatch.filter(src_zip_relpaths, ignore_pattern))
continue
zip_relpaths -= set(fnmatch.filter(zip_relpaths, ignore_pattern))
minify_relpaths = set()
minify_patterns = self.get_minify_patterns(home)
for minify_pattern in minify_patterns:
if minify_pattern.startswith('!'):
minify_pattern = minify_pattern[1:]
minify_relpaths -= set(
fnmatch.filter(minify_relpaths, minify_pattern))
continue
minify_relpaths.update(
fnmatch.filter(src_zip_relpaths, minify_pattern))
# 仅支持混淆 .py 文件
minify_relpaths = fnmatch.filter(minify_relpaths, '*.py')
with console.status('正在打包模型源码...', spinner='dots'):
for zip_relpath in sorted(zip_relpaths):
filepath = total_files[zip_relpath]
if zip_relpath in minify_relpaths:
continue
ziph.write(filepath, zip_relpath)
output('\t{}'.format(zip_relpath))
for zip_relpath in sorted(minify_relpaths):
filepath = total_files[zip_relpath]
with open(filepath, 'r') as fp:
content = fp.read()
with tempfile.NamedTemporaryFile(mode='w') as temp_fp:
content = python_minifier.minify(content)
temp_fp.write(content)
temp_fp.flush()
temp_fp.seek(0)
ziph.write(temp_fp.name, zip_relpath)
output('\t{} [green]MINIFIED[/green]'.format(zip_relpath))
def _install_sdk(self):
with console.status("正在安装 bge-python-sdk", spinner="earth"):
config_path = self.get_model_config_path()
config = get_config_parser(config_path)
section_name = DEFAULT_MODEL_SECTION
model_id = config_get(config.get, section_name, 'model_id')
runtime = config_get(config.get, section_name, 'runtime')
client = self._get_docker_client()
command = ('pip install --cache-dir /tmp/.cache/ --no-deps '
'bge-python-sdk pimento requests_toolbelt -t /code/lib')
image_name = RUNTIMES[runtime]
self._get_or_pull_image(client, image_name)
output('开始安装模型依赖包...')
command = ('sh -c "{}"').format(command)
container_name = generate_container_name(model_id)
self._force_remove_container(client, container_name)
self._run_by_container(client, image_name, command, container_name)
output('[green]安装完成[/green]')
def handler(self, args):
access_token = args.access_token
object_name = args.object_name
mode = args.mode
project = get_active_project()
oauth2_section = DEFAULT_OAUTH2_SECTION
token_section = DEFAULT_TOKEN_SECTION
config = read_config(project)
if not access_token:
access_token = config_get(config.get, token_section,
'access_token')
endpoint = config_get(config.get, oauth2_section, 'endpoint')
api = API(access_token, endpoint=endpoint, timeout=18.)
filename = posixpath.split(object_name)[1]
if filename == '':
output('[red]下载失败，object_name 不是文件：{}[/red]'.format(
repr(object_name)))
sys.exit(1)
if posixpath.exists(filename):
basename, suffix = posixpath.splitext(filename)
i = 1
while True:
new_basename = '{} ({})'.format(basename, i)
filename = ''.join((new_basename, suffix))
if not posixpath.exists(filename):
break
i += 1
message = '正在下载 {}'.format(object_name)
with console.status(message, spinner='earth'):
try:
with open(filename, mode) as fp:
api.download(object_name,
fp,
region=args.region,
expiration_time=args.expiration_time,
chunk_size=args.chunk_size)
except APIError as e:
output('[red]请求失败：[/red]')
output_json(e.result)
sys.exit(1)
output('[green]文件下载成功：[/green]{}'.format(filename))
def handler(self, args):
access_token = args.access_token
project = get_active_project()
oauth2_section = DEFAULT_OAUTH2_SECTION
token_section = DEFAULT_TOKEN_SECTION
config = read_config(project)
if not access_token:
access_token = config_get(config.get, token_section,
'access_token')
endpoint = config_get(config.get, oauth2_section, 'endpoint')
api = API(access_token, endpoint=endpoint, timeout=18.)
dirpath = args.dirpath
cmk_id = args.cmk_id
files = []
for filename in os.listdir(dirpath):
filepath = join(dirpath, filename)
if isfile(filepath):
files.append(filepath)
if not files:
output('[red]文件夹中没有可上传的文件[/red]')
sys.exit(1)
message = '正在上传目录 {}'.format(dirpath)
with console.status(message, spinner='earth'):
try:
object_names = api.upload_dir(dirpath, cmk_id=cmk_id)
except APIError as e:
output('[red]请求失败：[/red]')
output_json(e.result)
sys.exit(1)
output('[green]文件上传成功：[/green]')
table = Table(title="文件列表",
expand=True,
show_header=True,
header_style="magenta")
table.add_column("序号", style="cyan", no_wrap=True)
table.add_column("存储位置", style="magenta", overflow='fold')
for i, object_name in enumerate(object_names, 1):
table.add_row(str(i), object_name)
console.print(table)
def deploy_model(self, args):
"""部署模型"""
ignore_source = args.ignore_source
home = get_home()
project = get_active_project()
config_path = self.get_model_config_path()
config = get_config_parser(config_path)
section_name = DEFAULT_MODEL_SECTION
model_id = config_get(config.get, section_name, 'model_id')
timestr = datetime.now().strftime('%Y%m%d%H%M%S%f')
randstr = uuid4().hex
zip_filename = '{}.{}.{}.zip'.format(model_id, timestr, randstr)
params = {}
params['runtime'] = config_get(config.get, section_name, 'runtime')
params['memory_size'] = config_get(config.getint, section_name,
'memory_size')
params['timeout'] = config_get(config.getint, section_name, 'timeout')
oauth2_section = DEFAULT_OAUTH2_SECTION
token_section = DEFAULT_TOKEN_SECTION
config = read_config(project)
access_token = config_get(config.get, token_section, 'access_token')
endpoint = config_get(config.get, oauth2_section, 'endpoint')
api = API(access_token,
endpoint=endpoint,
timeout=DEFAULT_MODEL_TIMEOUT)
object_name = None
if not ignore_source:
ignore_path = join(home, BGE_IGNORE_FILE)
if not exists(ignore_path):
output('未发现 .bgeignore 文件，初始化 {} ...'.format(ignore_path))
open(ignore_path, 'w').write(BGEIGNORE_TEMPLATE)
minify_path = join(home, BGE_MINIFY_FILE)
if not exists(minify_path):
output('未发现 .bgeminify 文件，初始化 {} ...'.format(minify_path))
open(minify_path, 'w').write(BGEMINIFY_TEMPLATE)
output('开始打包模型源码...')
zip_tmpdir = join(home, '.bge', 'tmp')
if not exists(zip_tmpdir):
os.makedirs(zip_tmpdir)
with tempfile.NamedTemporaryFile(suffix='.zip',
prefix='model-',
dir=zip_tmpdir,
delete=False) as tmp:
with zipfile.ZipFile(tmp.name, 'w', ZIP_COMPRESSION) as zf:
self._zip_codedir(home, zf)
tmp.flush()
tmp.seek(0, 2)
size = tmp.tell()
tmp.seek(0)
human_size = human_byte(size)
if size > 100 * 1024 * 1024:
output('打包后 zip 文件大小为 {}，最大限制 100MB'.format(human_size))
exit(1)
output('打包成功：{}'.format(tmp.name))
output('文件大小：{}'.format(human_size))
output('开始上传模型源码...')
try:
object_name = api.upload(zip_filename, tmp)
except APIError as e:
output('[red]上传模型源码失败：[/red]')
output_json(e.result)
sys.exit(1)
output('[green]上传成功[green]')
with console.status('模型部署中...', spinner='earth'):
try:
result = api.deploy_model(model_id,
object_name=object_name,
**params)
except APIError as e:
output('[red]部署模型失败：[/red]')
output_json(e.result)
sys.exit(1)
task_id = result.task_id
output('模型部署任务：{}'.format(task_id))
task_path = join(home, '.bge', 'task_id')
with open(task_path, 'w') as f:
f.write(task_id)
output('模型部署任务返回结果：')
progress = self._wait_model_task(api, task_id, task_path)
if 'SUCCESS' == progress:
output('[green]模型 {} 灰度部署成功。'.format(model_id))
elif 'FAILURE' == progress:
output('[red]模型 {} 灰度部署失败。任务结果：{}'.format(model_id, result))
elif 'REVOKED' == progress:
output('[white]模型 {} 灰度部署任务已被撤销。'.format(model_id))