def upload_model_expfs(self, args):
home = get_home()
project = get_active_project()
config_path = self.get_model_config_path()
config = get_config_parser(config_path)
section_name = DEFAULT_MODEL_SECTION
model_id = config_get(config.get, section_name, 'model_id')
oauth2_section = DEFAULT_OAUTH2_SECTION
token_section = DEFAULT_TOKEN_SECTION
config = read_config(project)
access_token = config_get(config.get, token_section, 'access_token')
endpoint = config_get(config.get, oauth2_section, 'endpoint')
api = API(access_token,
endpoint=endpoint,
timeout=DEFAULT_MODEL_TIMEOUT)
try:
result = api.upload_model_expfs(model_id, 'uploadtest.zip')
except APIError as e:
output('[red]上传模型扩展集失败：[/red]')
output_json(e.result)
sys.exit(1)
task_id = result.task_id
output('上传模型扩展集任务：{}'.format(task_id))
task_path = join(home, '.bge', 'expfs.task_id')
with open(task_path, 'w') as f:
f.write(task_id)
output('上传模型扩展集任务返回结果：')
progress = self._wait_model_task(api, task_id, task_path)
if 'SUCCESS' == progress:
output('[green]模型 {} 上传模型扩展集成功。'.format(model_id))
elif 'FAILURE' == progress:
output('[red]模型 {} 上传模型扩展集失败。'.format(model_id))
elif 'REVOKED' == progress:
output('[white]模型 {} 上传模型扩展集任务已被撤销。'.format(model_id))
def _run_by_container(self, client, image_name, command, container_name):
home = get_home()
user = get_sys_user()
container = client.containers.run(
image_name,
command=command,
name=container_name,
volumes={home: {
'bind': WORKDIR,
'mode': 'rw'
}},
stop_signal='SIGINT',
user=user,
detach=True,
auto_remove=True)
try:
logs = container.logs(stream=True)
for log in logs:
output_syntax(log.strip().decode('utf-8'), line_numbers=False)
finally:
if container.status != 'exited':
try:
container.remove(force=True)
except Exception:
pass
def add_arguments(self, parser):
home = get_home()
try:
doc_ps = parser.add_subparsers(
help='对模型文档进行预览和发布'
)
except TypeError:
# required: Whether or not a subcommand must be provided,
# by default False (added in 3.7)
doc_ps = parser.add_subparsers(
help='对模型文档进行预览和发布',
required=False
)
init_p = doc_ps.add_parser(
'init',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
help='初始化 docsify 项目。'
)
init_p.add_argument(
'name',
type=str,
default='docs',
help='docsify 项目名称'
)
init_p.add_argument(
'--home',
type=str,
default=home,
help='docsify 项目生成的父级目录，默认为当前目录'
)
init_p.set_defaults(method=self.init_docsify, parser=init_p)
pre_p = doc_ps.add_parser(
'preview',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
help='对模型文档进行预览'
)
pre_p.add_argument(
'path',
type=str,
help='模型文档的 json 文件路径。'
)
pre_p.add_argument(
'--home',
type=str,
default=home,
help='docsify 项目的根目录，默认为当前目录'
)
pre_p.set_defaults(method=self.preview, parser=pre_p)
release_p = doc_ps.add_parser(
'release',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
help='发布模型文档'
)
release_p.set_defaults(method=self.release_doc, parser=release_p)
def clear_model(self, args):
home = get_home()
zip_tmpdir = join(home, '.bge', 'tmp')
if confirm('是否清空目录 {} 下打包模型生成的临时文件？'.format(zip_tmpdir)):
shutil.rmtree(zip_tmpdir, ignore_errors=True)
os.mkdir(zip_tmpdir)
output('[green]成功删除[/green]')
else:
output('[white]已取消[/white]')
def get_model_config_path(self, home=None):
if home is None:
home = get_home()
config_dir = join(home, '.bge')
model_config_path = join(home, 'model.ini')
if not exists(model_config_path) or not exists(config_dir):
output('[red]请确认当前是否为模型项目根目录。[/red]')
sys.exit(1)
return model_config_path
def get_docs_dir(self, home=None):
if home is None:
home = get_home()
doc_path = join(home, 'index.html')
if not exists(doc_path):
output(
'[red]请确认当前目录或者输入的项目路径是否为 docsify 项目根目录。[/red]'
)
sys.exit(1)
return home
def init_scaffold(self, args):
scaffold_name = args.scaffold_name
home = args.home
if home is None:
home = get_home()
scaffold_dir = join(home, scaffold_name)
if exists(scaffold_dir):
output('[red]错误！{} 已存在[/red]'.format(scaffold_dir))
sys.exit(1)
if not exists(home):
output('[red]错误！无法找到 home 目录 {}。[/red]'.format(home))
sys.exit(1)
model_id, runtime, memory_size, timeout = self._config_model()
os.makedirs(scaffold_dir)
bge_dir = join(scaffold_dir, '.bge')
lib_dir = join(scaffold_dir, 'lib')
for dir_ in [scaffold_dir, bge_dir, lib_dir]:
output(CREATE_MESSAGE.format(dir_))
if not exists(dir_):
os.makedirs(dir_)
output('[green]完成[/green]')
elif not isdir(dir_):
output('[red]失败！{} 存在但不是目录。[/red]'.format(dir_))
sys.exit(1)
else:
output('[red]已存在[/red]')
model_config_path = join(scaffold_dir, 'model.ini')
output(CREATE_MESSAGE.format(model_config_path))
if not exists(model_config_path):
open(model_config_path, 'w').write(MODEL_CONFIG_TEMPLATE)
# 模型源码打包忽略规则文件
ignore_path = join(scaffold_dir, BGE_IGNORE_FILE)
output(CREATE_MESSAGE.format(ignore_path))
if not exists(ignore_path):
open(ignore_path, 'w').write(BGEIGNORE_TEMPLATE)
# 模型源码打包混淆规则文件
minify_path = join(scaffold_dir, BGE_MINIFY_FILE)
output(CREATE_MESSAGE.format(minify_path))
if not exists(minify_path):
open(minify_path, 'w').write(BGEMINIFY_TEMPLATE)
script_name = 'main.py'
script_path = join(scaffold_dir, script_name)
with open(script_path, 'wb') as file_out:
file_out.write(MAIN_PY.encode())
st = os.stat(script_path)
os.chmod(script_path, st.st_mode | stat.S_IEXEC)
self._save_model_config(model_id,
runtime,
memory_size,
timeout,
home=scaffold_dir)
if confirm('是否安装 bge-python-sdk？'):
os.chdir(scaffold_dir)
self._install_sdk()
output('[green]成功创建模型项目脚手架[/green]')
def start_model(self, args):
port = args.port
home = get_home()
config_path = self.get_model_config_path()
config = get_config_parser(config_path)
section_name = DEFAULT_MODEL_SECTION
model_id = config_get(config.get, section_name, 'model_id')
runtime = config_get(config.get, section_name, 'runtime')
client = self._get_docker_client()
image_name = RUNTIMES[runtime]
self._get_or_pull_image(client, image_name)
command = 'python -u /server/app.py'
user = get_sys_user()
container_name = generate_container_name(model_id)
self._force_remove_container(client, container_name)
container = client.containers.run(
image_name,
command=command,
name=container_name,
volumes={home: {
'bind': WORKDIR,
'mode': 'rw'
}},
stop_signal='SIGINT',
ports={TEST_SERVER_PORT: port},
user=user,
detach=True,
stream=True,
auto_remove=True)
output('Model debug server is starting at {}...'.format(port))
output('Model {} was registered'.format(model_id))
output('\n\tURL: {}:{}/model/{}'.format(TEST_SERVER_ENDPOINT, port,
model_id))
output('\tMethod: GET\n')
try:
logs = container.logs(stream=True, follow=True)
for log in logs:
output(log.strip().decode('utf-8'))
finally:
if container.status != 'exited':
try:
container.remove(force=True)
except Exception:
pass
def init_docsify(self, args):
name = args.name
home = args.home
if home is None:
home = get_home()
docs_dir = join(home, name)
if exists(docs_dir):
output('[red]错误！{} 已存在[/red] '.format(docs_dir))
sys.exit(1)
if not exists(home):
output('[red]错误！无法找到 home 目录 {}。[/red]'.format(home))
sys.exit(1)
with os.popen('docsify init {}'.format(docs_dir)) as f:
content = f.read()
if content:
output(
'[green]docsify 项目已初始化，路径为：[/green]{}'.format(docs_dir)
)
output('[green]请跳转至项目目录下。[/green]')
def rollback_model(self, args):
version = args.version
home = get_home()
project = get_active_project()
config_path = self.get_model_config_path()
config = get_config_parser(config_path)
section_name = DEFAULT_MODEL_SECTION
model_id = config_get(config.get, section_name, 'model_id')
oauth2_section = DEFAULT_OAUTH2_SECTION
token_section = DEFAULT_TOKEN_SECTION
config = read_config(project)
access_token = config_get(config.get, token_section, 'access_token')
endpoint = config_get(config.get, oauth2_section, 'endpoint')
api = API(access_token,
endpoint=endpoint,
timeout=DEFAULT_MODEL_TIMEOUT)
try:
result = api.rollback_model(model_id, version)
except APIError as e:
output('[red]模型回滚失败：[/red]')
output_json(e.result)
sys.exit(1)
task_id = result.task_id
output('模型回滚任务：{}'.format(task_id))
task_path = join(home, '.bge', 'task_id')
with open(task_path, 'w') as f:
f.write(task_id)
progress = self._wait_model_task(api, task_id, task_path)
if 'SUCCESS' == progress:
output('[green]模型 {} 灰度版已成功回滚至版本 {}。[/green]'.format(
model_id, version))
elif 'FAILURE' == progress:
output('[red]模型 {} 灰度回滚至版本 {} 失败。任务结果：{}[/red]'.format(
model_id, version, result))
elif 'REVOKED' == progress:
output('[white]模型 {} 灰度回滚至版本 {} 任务已被撤销失败。[/white]'.format(
model_id, version))
def deploy_model(self, args):
"""部署模型"""
ignore_source = args.ignore_source
home = get_home()
project = get_active_project()
config_path = self.get_model_config_path()
config = get_config_parser(config_path)
section_name = DEFAULT_MODEL_SECTION
model_id = config_get(config.get, section_name, 'model_id')
timestr = datetime.now().strftime('%Y%m%d%H%M%S%f')
randstr = uuid4().hex
zip_filename = '{}.{}.{}.zip'.format(model_id, timestr, randstr)
params = {}
params['runtime'] = config_get(config.get, section_name, 'runtime')
params['memory_size'] = config_get(config.getint, section_name,
'memory_size')
params['timeout'] = config_get(config.getint, section_name, 'timeout')
oauth2_section = DEFAULT_OAUTH2_SECTION
token_section = DEFAULT_TOKEN_SECTION
config = read_config(project)
access_token = config_get(config.get, token_section, 'access_token')
endpoint = config_get(config.get, oauth2_section, 'endpoint')
api = API(access_token,
endpoint=endpoint,
timeout=DEFAULT_MODEL_TIMEOUT)
object_name = None
if not ignore_source:
ignore_path = join(home, BGE_IGNORE_FILE)
if not exists(ignore_path):
output('未发现 .bgeignore 文件，初始化 {} ...'.format(ignore_path))
open(ignore_path, 'w').write(BGEIGNORE_TEMPLATE)
minify_path = join(home, BGE_MINIFY_FILE)
if not exists(minify_path):
output('未发现 .bgeminify 文件，初始化 {} ...'.format(minify_path))
open(minify_path, 'w').write(BGEMINIFY_TEMPLATE)
output('开始打包模型源码...')
zip_tmpdir = join(home, '.bge', 'tmp')
if not exists(zip_tmpdir):
os.makedirs(zip_tmpdir)
with tempfile.NamedTemporaryFile(suffix='.zip',
prefix='model-',
dir=zip_tmpdir,
delete=False) as tmp:
with zipfile.ZipFile(tmp.name, 'w', ZIP_COMPRESSION) as zf:
self._zip_codedir(home, zf)
tmp.flush()
tmp.seek(0, 2)
size = tmp.tell()
tmp.seek(0)
human_size = human_byte(size)
if size > 100 * 1024 * 1024:
output('打包后 zip 文件大小为 {}，最大限制 100MB'.format(human_size))
exit(1)
output('打包成功：{}'.format(tmp.name))
output('文件大小：{}'.format(human_size))
output('开始上传模型源码...')
try:
object_name = api.upload(zip_filename, tmp)
except APIError as e:
output('[red]上传模型源码失败：[/red]')
output_json(e.result)
sys.exit(1)
output('[green]上传成功[green]')
with console.status('模型部署中...', spinner='earth'):
try:
result = api.deploy_model(model_id,
object_name=object_name,
**params)
except APIError as e:
output('[red]部署模型失败：[/red]')
output_json(e.result)
sys.exit(1)
task_id = result.task_id
output('模型部署任务：{}'.format(task_id))
task_path = join(home, '.bge', 'task_id')
with open(task_path, 'w') as f:
f.write(task_id)
output('模型部署任务返回结果：')
progress = self._wait_model_task(api, task_id, task_path)
if 'SUCCESS' == progress:
output('[green]模型 {} 灰度部署成功。'.format(model_id))
elif 'FAILURE' == progress:
output('[red]模型 {} 灰度部署失败。任务结果：{}'.format(model_id, result))
elif 'REVOKED' == progress:
output('[white]模型 {} 灰度部署任务已被撤销。'.format(model_id))
def add_arguments(self, parser):
home = get_home()
try:
model_subparsers = parser.add_subparsers(dest='subcommand',
help='可选子命令。')
except TypeError:
# required: Whether or not a subcommand must be provided,
# by default False (added in 3.7)
model_subparsers = parser.add_subparsers(dest='subcommand',
help='可选子命令。',
required=False)
# 初始化脚手架命令
init_p = model_subparsers.add_parser(
'init',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
help='初始化一个新的模型开发脚手架项目。')
init_p.add_argument('scaffold_name', type=str, help='脚手架名字。')
init_p.add_argument('--home',
type=str,
default=home,
help='脚手架项目生成的父级目录，默认为当前目录。')
init_p.set_defaults(method=self.init_scaffold, parser=init_p)
# 模型配置
config_p = model_subparsers.add_parser(
'config',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
help='配置模型。')
config_p.add_argument('-s',
'--show',
default=False,
action='store_true',
help='打印显示当前模型脚手架的配置。')
config_p.set_defaults(method=self.config_model, parser=config_p)
# 安装 Python 依赖包
install_p = model_subparsers.add_parser(
'install',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
help='部署灰度版本模型。（开发中，仅支持 python2.7/3.6 的 docker 环境）')
install_p.add_argument('package_name',
nargs="+",
type=str,
help='要安装的 Python 软件包，如 numpy 或 numpy==v1.19.5')
install_p.add_argument('-r',
'--requirements',
nargs=1,
type=str,
help='要安装的 Python 软件包依赖文件')
install_p.add_argument('-U',
'--upgrade',
default=False,
action='store_true',
help='是否升级依赖包，同 pip install -U')
install_p.add_argument(
'--force-reinstall',
default=False,
action='store_true',
help='是否强制重新安装依赖包，同 pip install --force-reinstall')
install_p.set_defaults(method=self.install_deps, parser=install_p)
start_p = model_subparsers.add_parser(
'start',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
help='启动本地 HTTP 测试服务器')
start_p.add_argument('-p',
'--port',
default=TEST_SERVER_PORT,
type=int,
choices=range(1, 65536),
metavar="[1-65535]",
help='服务器监听端口')
start_p.set_defaults(method=self.start_model, parser=start_p)
expfs_p = model_subparsers.add_parser(
'expfs',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
help='扩展模型文件集')
expfs_p.set_defaults(method=self.upload_model_expfs, parser=expfs_p)
# 部署子命令
deploy_p = model_subparsers.add_parser(
'deploy',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
help='部署灰度版本模型。')
deploy_p.add_argument('-i',
'--ignore-source',
default=False,
action="store_true",
help='部署时不包含模型源代码')
deploy_p.set_defaults(method=self.deploy_model, parser=deploy_p)
publish_p = model_subparsers.add_parser(
'publish',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
help='发布最新部署的灰度版本模型为稳定版。')
publish_p.add_argument('-m', '--message', type=str, help='模型发布说明。')
publish_p.set_defaults(method=self.publish_model, parser=publish_p)
# 运行模型子命令
run_p = model_subparsers.add_parser(
'run',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
help='调用线上稳定版模型。')
group = run_p.add_mutually_exclusive_group()
draft_a = group.add_argument('-d',
'--draft',
default=False,
action="store_true",
help='调用最新灰度版本模型。')
group.add_argument('-t',
'--test',
default=False,
action="store_true",
help='调用本地启动的测试服务器运行模型。')
group_2 = run_p.add_mutually_exclusive_group()
group_2.add_argument('-p',
'--port',
default=TEST_SERVER_PORT,
type=int,
action=TestServerPortAction,
choices=range(1, 65536),
metavar="[1-65535]",
help='服务器监听端口')
group_2._group_actions.append(draft_a) # 禁止 -p 与 -d 参数同时提供
group = run_p.add_mutually_exclusive_group()
group.add_argument('-a',
'--args',
nargs=2,
action='append',
help='参数对，示例：--args f1 v1 --args f2 v2')
group.add_argument('-f',
'--file',
help='JSON 格式参数文件，内容示例：{ "f1": "v1", "f2": "v2" }。')
run_p.set_defaults(method=self.run_model, parser=run_p)
# 回滚子命令
rollback_p = model_subparsers.add_parser(
'rollback',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
help='模型回滚至某个稳定版本。')
rollback_p.add_argument('-v', '--version', type=str, help='要回滚的模型版本。')
rollback_p.set_defaults(method=self.rollback_model, parser=rollback_p)
# 回滚版本列表子命令
versions_p = model_subparsers.add_parser(
'versions',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
help='模型发布版本列表。（开发中）')
versions_p.add_argument('-l',
'--limit',
default=5,
type=int,
help='每页返回数量。')
versions_p.add_argument('-p', '--next_page', type=int, help='下一页。')
versions_p.set_defaults(method=self.model_versions, parser=versions_p)
# 清空临时文件
clear_p = model_subparsers.add_parser(
'clear',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
help='清空部署模型时生成的打包临时文件')
clear_p.set_defaults(method=self.clear_model, parser=clear_p)
