def upload_model_expfs(self, args): home = get_home() project = get_active_project() config_path = self.get_model_config_path() config = get_config_parser(config_path) section_name = DEFAULT_MODEL_SECTION model_id = config_get(config.get, section_name, 'model_id') oauth2_section = DEFAULT_OAUTH2_SECTION token_section = DEFAULT_TOKEN_SECTION config = read_config(project) access_token = config_get(config.get, token_section, 'access_token') endpoint = config_get(config.get, oauth2_section, 'endpoint') api = API(access_token, endpoint=endpoint, timeout=DEFAULT_MODEL_TIMEOUT) try: result = api.upload_model_expfs(model_id, 'uploadtest.zip') except APIError as e: output('[red]上传模型扩展集失败:[/red]') output_json(e.result) sys.exit(1) task_id = result.task_id output('上传模型扩展集任务:{}'.format(task_id)) task_path = join(home, '.bge', 'expfs.task_id') with open(task_path, 'w') as f: f.write(task_id) output('上传模型扩展集任务返回结果:') progress = self._wait_model_task(api, task_id, task_path) if 'SUCCESS' == progress: output('[green]模型 {} 上传模型扩展集成功。'.format(model_id)) elif 'FAILURE' == progress: output('[red]模型 {} 上传模型扩展集失败。'.format(model_id)) elif 'REVOKED' == progress: output('[white]模型 {} 上传模型扩展集任务已被撤销。'.format(model_id))
def _run_by_container(self, client, image_name, command, container_name): home = get_home() user = get_sys_user() container = client.containers.run( image_name, command=command, name=container_name, volumes={home: { 'bind': WORKDIR, 'mode': 'rw' }}, stop_signal='SIGINT', user=user, detach=True, auto_remove=True) try: logs = container.logs(stream=True) for log in logs: output_syntax(log.strip().decode('utf-8'), line_numbers=False) finally: if container.status != 'exited': try: container.remove(force=True) except Exception: pass
def add_arguments(self, parser): home = get_home() try: doc_ps = parser.add_subparsers( help='对模型文档进行预览和发布' ) except TypeError: # required: Whether or not a subcommand must be provided, # by default False (added in 3.7) doc_ps = parser.add_subparsers( help='对模型文档进行预览和发布', required=False ) init_p = doc_ps.add_parser( 'init', formatter_class=argparse.ArgumentDefaultsHelpFormatter, help='初始化 docsify 项目。' ) init_p.add_argument( 'name', type=str, default='docs', help='docsify 项目名称' ) init_p.add_argument( '--home', type=str, default=home, help='docsify 项目生成的父级目录,默认为当前目录' ) init_p.set_defaults(method=self.init_docsify, parser=init_p) pre_p = doc_ps.add_parser( 'preview', formatter_class=argparse.ArgumentDefaultsHelpFormatter, help='对模型文档进行预览' ) pre_p.add_argument( 'path', type=str, help='模型文档的 json 文件路径。' ) pre_p.add_argument( '--home', type=str, default=home, help='docsify 项目的根目录,默认为当前目录' ) pre_p.set_defaults(method=self.preview, parser=pre_p) release_p = doc_ps.add_parser( 'release', formatter_class=argparse.ArgumentDefaultsHelpFormatter, help='发布模型文档' ) release_p.set_defaults(method=self.release_doc, parser=release_p)
def clear_model(self, args): home = get_home() zip_tmpdir = join(home, '.bge', 'tmp') if confirm('是否清空目录 {} 下打包模型生成的临时文件?'.format(zip_tmpdir)): shutil.rmtree(zip_tmpdir, ignore_errors=True) os.mkdir(zip_tmpdir) output('[green]成功删除[/green]') else: output('[white]已取消[/white]')
def get_model_config_path(self, home=None): if home is None: home = get_home() config_dir = join(home, '.bge') model_config_path = join(home, 'model.ini') if not exists(model_config_path) or not exists(config_dir): output('[red]请确认当前是否为模型项目根目录。[/red]') sys.exit(1) return model_config_path
def get_docs_dir(self, home=None): if home is None: home = get_home() doc_path = join(home, 'index.html') if not exists(doc_path): output( '[red]请确认当前目录或者输入的项目路径是否为 docsify 项目根目录。[/red]' ) sys.exit(1) return home
def init_scaffold(self, args): scaffold_name = args.scaffold_name home = args.home if home is None: home = get_home() scaffold_dir = join(home, scaffold_name) if exists(scaffold_dir): output('[red]错误!{} 已存在[/red]'.format(scaffold_dir)) sys.exit(1) if not exists(home): output('[red]错误!无法找到 home 目录 {}。[/red]'.format(home)) sys.exit(1) model_id, runtime, memory_size, timeout = self._config_model() os.makedirs(scaffold_dir) bge_dir = join(scaffold_dir, '.bge') lib_dir = join(scaffold_dir, 'lib') for dir_ in [scaffold_dir, bge_dir, lib_dir]: output(CREATE_MESSAGE.format(dir_)) if not exists(dir_): os.makedirs(dir_) output('[green]完成[/green]') elif not isdir(dir_): output('[red]失败!{} 存在但不是目录。[/red]'.format(dir_)) sys.exit(1) else: output('[red]已存在[/red]') model_config_path = join(scaffold_dir, 'model.ini') output(CREATE_MESSAGE.format(model_config_path)) if not exists(model_config_path): open(model_config_path, 'w').write(MODEL_CONFIG_TEMPLATE) # 模型源码打包忽略规则文件 ignore_path = join(scaffold_dir, BGE_IGNORE_FILE) output(CREATE_MESSAGE.format(ignore_path)) if not exists(ignore_path): open(ignore_path, 'w').write(BGEIGNORE_TEMPLATE) # 模型源码打包混淆规则文件 minify_path = join(scaffold_dir, BGE_MINIFY_FILE) output(CREATE_MESSAGE.format(minify_path)) if not exists(minify_path): open(minify_path, 'w').write(BGEMINIFY_TEMPLATE) script_name = 'main.py' script_path = join(scaffold_dir, script_name) with open(script_path, 'wb') as file_out: file_out.write(MAIN_PY.encode()) st = os.stat(script_path) os.chmod(script_path, st.st_mode | stat.S_IEXEC) self._save_model_config(model_id, runtime, memory_size, timeout, home=scaffold_dir) if confirm('是否安装 bge-python-sdk?'): os.chdir(scaffold_dir) self._install_sdk() output('[green]成功创建模型项目脚手架[/green]')
def start_model(self, args): port = args.port home = get_home() config_path = self.get_model_config_path() config = get_config_parser(config_path) section_name = DEFAULT_MODEL_SECTION model_id = config_get(config.get, section_name, 'model_id') runtime = config_get(config.get, section_name, 'runtime') client = self._get_docker_client() image_name = RUNTIMES[runtime] self._get_or_pull_image(client, image_name) command = 'python -u /server/app.py' user = get_sys_user() container_name = generate_container_name(model_id) self._force_remove_container(client, container_name) container = client.containers.run( image_name, command=command, name=container_name, volumes={home: { 'bind': WORKDIR, 'mode': 'rw' }}, stop_signal='SIGINT', ports={TEST_SERVER_PORT: port}, user=user, detach=True, stream=True, auto_remove=True) output('Model debug server is starting at {}...'.format(port)) output('Model {} was registered'.format(model_id)) output('\n\tURL: {}:{}/model/{}'.format(TEST_SERVER_ENDPOINT, port, model_id)) output('\tMethod: GET\n') try: logs = container.logs(stream=True, follow=True) for log in logs: output(log.strip().decode('utf-8')) finally: if container.status != 'exited': try: container.remove(force=True) except Exception: pass
def init_docsify(self, args): name = args.name home = args.home if home is None: home = get_home() docs_dir = join(home, name) if exists(docs_dir): output('[red]错误!{} 已存在[/red] '.format(docs_dir)) sys.exit(1) if not exists(home): output('[red]错误!无法找到 home 目录 {}。[/red]'.format(home)) sys.exit(1) with os.popen('docsify init {}'.format(docs_dir)) as f: content = f.read() if content: output( '[green]docsify 项目已初始化,路径为:[/green]{}'.format(docs_dir) ) output('[green]请跳转至项目目录下。[/green]')
def rollback_model(self, args): version = args.version home = get_home() project = get_active_project() config_path = self.get_model_config_path() config = get_config_parser(config_path) section_name = DEFAULT_MODEL_SECTION model_id = config_get(config.get, section_name, 'model_id') oauth2_section = DEFAULT_OAUTH2_SECTION token_section = DEFAULT_TOKEN_SECTION config = read_config(project) access_token = config_get(config.get, token_section, 'access_token') endpoint = config_get(config.get, oauth2_section, 'endpoint') api = API(access_token, endpoint=endpoint, timeout=DEFAULT_MODEL_TIMEOUT) try: result = api.rollback_model(model_id, version) except APIError as e: output('[red]模型回滚失败:[/red]') output_json(e.result) sys.exit(1) task_id = result.task_id output('模型回滚任务:{}'.format(task_id)) task_path = join(home, '.bge', 'task_id') with open(task_path, 'w') as f: f.write(task_id) progress = self._wait_model_task(api, task_id, task_path) if 'SUCCESS' == progress: output('[green]模型 {} 灰度版已成功回滚至版本 {}。[/green]'.format( model_id, version)) elif 'FAILURE' == progress: output('[red]模型 {} 灰度回滚至版本 {} 失败。任务结果:{}[/red]'.format( model_id, version, result)) elif 'REVOKED' == progress: output('[white]模型 {} 灰度回滚至版本 {} 任务已被撤销失败。[/white]'.format( model_id, version))
def deploy_model(self, args): """部署模型""" ignore_source = args.ignore_source home = get_home() project = get_active_project() config_path = self.get_model_config_path() config = get_config_parser(config_path) section_name = DEFAULT_MODEL_SECTION model_id = config_get(config.get, section_name, 'model_id') timestr = datetime.now().strftime('%Y%m%d%H%M%S%f') randstr = uuid4().hex zip_filename = '{}.{}.{}.zip'.format(model_id, timestr, randstr) params = {} params['runtime'] = config_get(config.get, section_name, 'runtime') params['memory_size'] = config_get(config.getint, section_name, 'memory_size') params['timeout'] = config_get(config.getint, section_name, 'timeout') oauth2_section = DEFAULT_OAUTH2_SECTION token_section = DEFAULT_TOKEN_SECTION config = read_config(project) access_token = config_get(config.get, token_section, 'access_token') endpoint = config_get(config.get, oauth2_section, 'endpoint') api = API(access_token, endpoint=endpoint, timeout=DEFAULT_MODEL_TIMEOUT) object_name = None if not ignore_source: ignore_path = join(home, BGE_IGNORE_FILE) if not exists(ignore_path): output('未发现 .bgeignore 文件,初始化 {} ...'.format(ignore_path)) open(ignore_path, 'w').write(BGEIGNORE_TEMPLATE) minify_path = join(home, BGE_MINIFY_FILE) if not exists(minify_path): output('未发现 .bgeminify 文件,初始化 {} ...'.format(minify_path)) open(minify_path, 'w').write(BGEMINIFY_TEMPLATE) output('开始打包模型源码...') zip_tmpdir = join(home, '.bge', 'tmp') if not exists(zip_tmpdir): os.makedirs(zip_tmpdir) with tempfile.NamedTemporaryFile(suffix='.zip', prefix='model-', dir=zip_tmpdir, delete=False) as tmp: with zipfile.ZipFile(tmp.name, 'w', ZIP_COMPRESSION) as zf: self._zip_codedir(home, zf) tmp.flush() tmp.seek(0, 2) size = tmp.tell() tmp.seek(0) human_size = human_byte(size) if size > 100 * 1024 * 1024: output('打包后 zip 文件大小为 {},最大限制 100MB'.format(human_size)) exit(1) output('打包成功:{}'.format(tmp.name)) output('文件大小:{}'.format(human_size)) output('开始上传模型源码...') try: object_name = api.upload(zip_filename, tmp) except APIError as e: output('[red]上传模型源码失败:[/red]') output_json(e.result) sys.exit(1) output('[green]上传成功[green]') with console.status('模型部署中...', spinner='earth'): try: result = api.deploy_model(model_id, object_name=object_name, **params) except APIError as e: output('[red]部署模型失败:[/red]') output_json(e.result) sys.exit(1) task_id = result.task_id output('模型部署任务:{}'.format(task_id)) task_path = join(home, '.bge', 'task_id') with open(task_path, 'w') as f: f.write(task_id) output('模型部署任务返回结果:') progress = self._wait_model_task(api, task_id, task_path) if 'SUCCESS' == progress: output('[green]模型 {} 灰度部署成功。'.format(model_id)) elif 'FAILURE' == progress: output('[red]模型 {} 灰度部署失败。任务结果:{}'.format(model_id, result)) elif 'REVOKED' == progress: output('[white]模型 {} 灰度部署任务已被撤销。'.format(model_id))
def add_arguments(self, parser): home = get_home() try: model_subparsers = parser.add_subparsers(dest='subcommand', help='可选子命令。') except TypeError: # required: Whether or not a subcommand must be provided, # by default False (added in 3.7) model_subparsers = parser.add_subparsers(dest='subcommand', help='可选子命令。', required=False) # 初始化脚手架命令 init_p = model_subparsers.add_parser( 'init', formatter_class=argparse.ArgumentDefaultsHelpFormatter, help='初始化一个新的模型开发脚手架项目。') init_p.add_argument('scaffold_name', type=str, help='脚手架名字。') init_p.add_argument('--home', type=str, default=home, help='脚手架项目生成的父级目录,默认为当前目录。') init_p.set_defaults(method=self.init_scaffold, parser=init_p) # 模型配置 config_p = model_subparsers.add_parser( 'config', formatter_class=argparse.ArgumentDefaultsHelpFormatter, help='配置模型。') config_p.add_argument('-s', '--show', default=False, action='store_true', help='打印显示当前模型脚手架的配置。') config_p.set_defaults(method=self.config_model, parser=config_p) # 安装 Python 依赖包 install_p = model_subparsers.add_parser( 'install', formatter_class=argparse.ArgumentDefaultsHelpFormatter, help='部署灰度版本模型。(开发中,仅支持 python2.7/3.6 的 docker 环境)') install_p.add_argument('package_name', nargs="+", type=str, help='要安装的 Python 软件包,如 numpy 或 numpy==v1.19.5') install_p.add_argument('-r', '--requirements', nargs=1, type=str, help='要安装的 Python 软件包依赖文件') install_p.add_argument('-U', '--upgrade', default=False, action='store_true', help='是否升级依赖包,同 pip install -U') install_p.add_argument( '--force-reinstall', default=False, action='store_true', help='是否强制重新安装依赖包,同 pip install --force-reinstall') install_p.set_defaults(method=self.install_deps, parser=install_p) start_p = model_subparsers.add_parser( 'start', formatter_class=argparse.ArgumentDefaultsHelpFormatter, help='启动本地 HTTP 测试服务器') start_p.add_argument('-p', '--port', default=TEST_SERVER_PORT, type=int, choices=range(1, 65536), metavar="[1-65535]", help='服务器监听端口') start_p.set_defaults(method=self.start_model, parser=start_p) expfs_p = model_subparsers.add_parser( 'expfs', formatter_class=argparse.ArgumentDefaultsHelpFormatter, help='扩展模型文件集') expfs_p.set_defaults(method=self.upload_model_expfs, parser=expfs_p) # 部署子命令 deploy_p = model_subparsers.add_parser( 'deploy', formatter_class=argparse.ArgumentDefaultsHelpFormatter, help='部署灰度版本模型。') deploy_p.add_argument('-i', '--ignore-source', default=False, action="store_true", help='部署时不包含模型源代码') deploy_p.set_defaults(method=self.deploy_model, parser=deploy_p) publish_p = model_subparsers.add_parser( 'publish', formatter_class=argparse.ArgumentDefaultsHelpFormatter, help='发布最新部署的灰度版本模型为稳定版。') publish_p.add_argument('-m', '--message', type=str, help='模型发布说明。') publish_p.set_defaults(method=self.publish_model, parser=publish_p) # 运行模型子命令 run_p = model_subparsers.add_parser( 'run', formatter_class=argparse.ArgumentDefaultsHelpFormatter, help='调用线上稳定版模型。') group = run_p.add_mutually_exclusive_group() draft_a = group.add_argument('-d', '--draft', default=False, action="store_true", help='调用最新灰度版本模型。') group.add_argument('-t', '--test', default=False, action="store_true", help='调用本地启动的测试服务器运行模型。') group_2 = run_p.add_mutually_exclusive_group() group_2.add_argument('-p', '--port', default=TEST_SERVER_PORT, type=int, action=TestServerPortAction, choices=range(1, 65536), metavar="[1-65535]", help='服务器监听端口') group_2._group_actions.append(draft_a) # 禁止 -p 与 -d 参数同时提供 group = run_p.add_mutually_exclusive_group() group.add_argument('-a', '--args', nargs=2, action='append', help='参数对,示例:--args f1 v1 --args f2 v2') group.add_argument('-f', '--file', help='JSON 格式参数文件,内容示例:{ "f1": "v1", "f2": "v2" }。') run_p.set_defaults(method=self.run_model, parser=run_p) # 回滚子命令 rollback_p = model_subparsers.add_parser( 'rollback', formatter_class=argparse.ArgumentDefaultsHelpFormatter, help='模型回滚至某个稳定版本。') rollback_p.add_argument('-v', '--version', type=str, help='要回滚的模型版本。') rollback_p.set_defaults(method=self.rollback_model, parser=rollback_p) # 回滚版本列表子命令 versions_p = model_subparsers.add_parser( 'versions', formatter_class=argparse.ArgumentDefaultsHelpFormatter, help='模型发布版本列表。(开发中)') versions_p.add_argument('-l', '--limit', default=5, type=int, help='每页返回数量。') versions_p.add_argument('-p', '--next_page', type=int, help='下一页。') versions_p.set_defaults(method=self.model_versions, parser=versions_p) # 清空临时文件 clear_p = model_subparsers.add_parser( 'clear', formatter_class=argparse.ArgumentDefaultsHelpFormatter, help='清空部署模型时生成的打包临时文件') clear_p.set_defaults(method=self.clear_model, parser=clear_p)