def handler(self, args): access_token = args.access_token filepath = args.filepath project = get_active_project() oauth2_section = DEFAULT_OAUTH2_SECTION token_section = DEFAULT_TOKEN_SECTION config = read_config(project) if not access_token: access_token = config_get(config.get, token_section, 'access_token') endpoint = config_get(config.get, oauth2_section, 'endpoint') api = API(access_token, endpoint=endpoint, timeout=18.) if not posixpath.isfile(filepath): output('[red]只能上传文件[/red]') sys.exit(1) filename = args.filename cmk_id = args.cmk_id if not filename: filename = posixpath.split(filepath)[1] message = '正在上传文件 {}'.format(filepath) with console.status(message, spinner='earth'): try: with open(filepath, 'rb') as fp: object_name = api.upload(filename, fp, cmk_id=cmk_id) except APIError as e: output('[red]请求失败:[/red]') output_json(e.result) sys.exit(1) output('[green]文件上传成功,存储位置:[/green]{}'.format(object_name))
def handler(self, args): access_token = args.access_token namespace = args.namespace biosample_id = args.biosample_id project = get_active_project() oauth2_section = DEFAULT_OAUTH2_SECTION token_section = DEFAULT_TOKEN_SECTION config = read_config(project) if not access_token: access_token = config_get(config.get, token_section, 'access_token') endpoint = config_get(config.get, oauth2_section, 'endpoint') api = API(access_token, endpoint=endpoint, timeout=18.) try: result = api.get_data_items(namespace, biosample_id, collection_id=args.collection_id, data_element_ids=args.data_element_ids, next_page=args.next_page) except APIError as e: output('[red]请求失败:[/red]') output_json(e.result) sys.exit(1) output('[green]请求成功:[/green]') output_json(result, cls=ModelEncoder)
def upload_model_expfs(self, args): home = get_home() project = get_active_project() config_path = self.get_model_config_path() config = get_config_parser(config_path) section_name = DEFAULT_MODEL_SECTION model_id = config_get(config.get, section_name, 'model_id') oauth2_section = DEFAULT_OAUTH2_SECTION token_section = DEFAULT_TOKEN_SECTION config = read_config(project) access_token = config_get(config.get, token_section, 'access_token') endpoint = config_get(config.get, oauth2_section, 'endpoint') api = API(access_token, endpoint=endpoint, timeout=DEFAULT_MODEL_TIMEOUT) try: result = api.upload_model_expfs(model_id, 'uploadtest.zip') except APIError as e: output('[red]上传模型扩展集失败:[/red]') output_json(e.result) sys.exit(1) task_id = result.task_id output('上传模型扩展集任务:{}'.format(task_id)) task_path = join(home, '.bge', 'expfs.task_id') with open(task_path, 'w') as f: f.write(task_id) output('上传模型扩展集任务返回结果:') progress = self._wait_model_task(api, task_id, task_path) if 'SUCCESS' == progress: output('[green]模型 {} 上传模型扩展集成功。'.format(model_id)) elif 'FAILURE' == progress: output('[red]模型 {} 上传模型扩展集失败。'.format(model_id)) elif 'REVOKED' == progress: output('[white]模型 {} 上传模型扩展集任务已被撤销。'.format(model_id))
def _get_config_value(self, config_parser, key, type_): oauth2_section = constants.DEFAULT_OAUTH2_SECTION if 'str' == type_: saved_value = config_get( config_parser.get, oauth2_section, key) elif 'int' == type_: saved_value = config_get( config_parser.getint, oauth2_section, key) elif 'bool' == type_: saved_value = config_get( config_parser.getboolean, oauth2_section, key) else: raise ValueError('invalid type: {}'.format(type_)) return saved_value
def run_model(self, args): params = {} if args.file: filepath = args.file if not exists(filepath): output('文件不存在:{}'.format(filepath)) sys.exit(1) if isdir(filepath): output('存在文件夹:{}'.format(filepath)) sys.exit(1) with open(filepath, 'r') as fp: params = json.load(fp) elif args.args: params = dict(args.args) draft = args.draft test = args.test project = get_active_project() config_path = self.get_model_config_path() config = get_config_parser(config_path) section_name = DEFAULT_MODEL_SECTION model_id = config_get(config.get, section_name, 'model_id') oauth2_section = DEFAULT_OAUTH2_SECTION token_section = DEFAULT_TOKEN_SECTION config = read_config(project) access_token = config_get(config.get, token_section, 'access_token') endpoint = config_get(config.get, oauth2_section, 'endpoint') api = API(access_token, endpoint=endpoint, timeout=DEFAULT_MODEL_TIMEOUT) try: if draft is True: result = api.invoke_draft_model(model_id, **params) elif test is True: port = args.port api = API(access_token, endpoint='{}:{}'.format(TEST_SERVER_ENDPOINT, port), timeout=DEFAULT_MODEL_TIMEOUT) result = api.invoke_model(model_id, **params) else: result = api.invoke_model(model_id, **params) except APIError as e: output('[red]模型运行失败:[/red]') output_json(e.result) sys.exit(1) output('[green]模型返回值:[/green]') output_json(result.json())
def _config_model(self, config=None): output('开始配置解读模型...') default_model_id = '' default_runtime = 'python3' default_memory_size = 128 default_timeout = 900 if config is not None: section_name = DEFAULT_MODEL_SECTION default_model_id = config_get(config.get, section_name, 'model_id') default_runtime = config_get(config.get, section_name, 'runtime') default_memory_size = config_get(config.getint, section_name, 'memory_size') default_timeout = config_get(config.getint, section_name, 'timeout') model_id = self._input_model_id(default=default_model_id) try: index = RUNTIME_CHOICES.index(default_runtime) except IndexError: index = 0 prompt = '请选择运行环境 [{}]'.format(default_runtime) input_prompt = "请选择" runtime = qprompt.enum_menu(RUNTIME_CHOICES).show(header=prompt, dft=index + 1, returns="desc", msg=input_prompt) try: index = MEMORY_SIZE_CHOICES.index(default_memory_size) except IndexError: index = 0 prompt = '请选择内存占用(MB) [{}]'.format(default_memory_size) input_prompt = "请选择" memory_size = qprompt.enum_menu(MEMORY_SIZE_CHOICES).show( header=prompt, dft=index + 1, returns="desc", msg=input_prompt) prompt = '请输入模型运行超时时间,类型为整数' while True: timeout = qprompt.ask_int(msg=prompt, dft=default_timeout) if (MIN_TIMEOUT and timeout < MIN_TIMEOUT) \ or (MAX_TIMEOUT and timeout > MAX_TIMEOUT): output('超出范围 [{},{}],请重新输入'.format(MIN_TIMEOUT or '', MAX_TIMEOUT or '')) continue break if not timeout: timeout = default_timeout return model_id, runtime, memory_size, timeout
def _install_sdk(self): with console.status("正在安装 bge-python-sdk", spinner="earth"): config_path = self.get_model_config_path() config = get_config_parser(config_path) section_name = DEFAULT_MODEL_SECTION model_id = config_get(config.get, section_name, 'model_id') runtime = config_get(config.get, section_name, 'runtime') client = self._get_docker_client() command = ('pip install --cache-dir /tmp/.cache/ --no-deps ' 'bge-python-sdk pimento requests_toolbelt -t /code/lib') image_name = RUNTIMES[runtime] self._get_or_pull_image(client, image_name) output('开始安装模型依赖包...') command = ('sh -c "{}"').format(command) container_name = generate_container_name(model_id) self._force_remove_container(client, container_name) self._run_by_container(client, image_name, command, container_name) output('[green]安装完成[/green]')
def start_model(self, args): port = args.port home = get_home() config_path = self.get_model_config_path() config = get_config_parser(config_path) section_name = DEFAULT_MODEL_SECTION model_id = config_get(config.get, section_name, 'model_id') runtime = config_get(config.get, section_name, 'runtime') client = self._get_docker_client() image_name = RUNTIMES[runtime] self._get_or_pull_image(client, image_name) command = 'python -u /server/app.py' user = get_sys_user() container_name = generate_container_name(model_id) self._force_remove_container(client, container_name) container = client.containers.run( image_name, command=command, name=container_name, volumes={home: { 'bind': WORKDIR, 'mode': 'rw' }}, stop_signal='SIGINT', ports={TEST_SERVER_PORT: port}, user=user, detach=True, stream=True, auto_remove=True) output('Model debug server is starting at {}...'.format(port)) output('Model {} was registered'.format(model_id)) output('\n\tURL: {}:{}/model/{}'.format(TEST_SERVER_ENDPOINT, port, model_id)) output('\tMethod: GET\n') try: logs = container.logs(stream=True, follow=True) for log in logs: output(log.strip().decode('utf-8')) finally: if container.status != 'exited': try: container.remove(force=True) except Exception: pass
def model_versions(self, args): limit = args.limit next_page = args.next_page project = get_active_project() config_path = self.get_model_config_path() config = get_config_parser(config_path) section_name = DEFAULT_MODEL_SECTION model_id = config_get(config.get, section_name, 'model_id') oauth2_section = DEFAULT_OAUTH2_SECTION token_section = DEFAULT_TOKEN_SECTION config = read_config(project) access_token = config_get(config.get, token_section, 'access_token') endpoint = config_get(config.get, oauth2_section, 'endpoint') api = API(access_token, endpoint=endpoint, timeout=DEFAULT_MODEL_TIMEOUT) try: result = api.model_versions(model_id, limit=limit, next_page=next_page) except APIError as e: output('[red]获取模型版本列表失败:[/red]') output_json(e.result) sys.exit(1) output('第 {} 页,模型 {} 已发布版本:'.format(next_page or 1, model_id)) output('...') length = None items = result['result'] for item in items: version = item['version'] message = item['message'] create_time = datetime.fromtimestamp(item['create_time']) if length is None: length = len(str(version)) output('发布时间:{},版本号:{},发布记录:{}'.format(create_time, str(version).rjust(length), message)) output('...') size = len(items) if size == 0 or size < limit or next_page == result['next_page']: output('下一页:无') else: output('下一页:{}'.format(result['next_page']))
def handler(self, args): access_token = args.access_token project = get_active_project() oauth2_section = DEFAULT_OAUTH2_SECTION token_section = DEFAULT_TOKEN_SECTION config = read_config(project) if not access_token: access_token = config_get(config.get, token_section, 'access_token') endpoint = config_get(config.get, oauth2_section, 'endpoint') api = API(access_token, endpoint=endpoint, timeout=18.) try: result = api.get_user() except APIError as e: output('[red]请求失败:[/red]') output_json(e.result) sys.exit(1) output('[green]请求成功:[/green]') output_json(result, cls=ModelEncoder)
def handler(self, args): project = get_active_project() config_path = get_config_path(project) config_parser = get_config_parser(config_path) oauth2_section = constants.DEFAULT_OAUTH2_SECTION client_id = config_get(config_parser.get, oauth2_section, 'client_id') client_secret = config_get(config_parser.get, oauth2_section, 'client_secret') endpoint = config_get(config_parser.get, oauth2_section, 'endpoint') redirect_uri = args.redirect_uri oauth2 = OAuth2(client_id, client_secret, endpoint=endpoint, timeout=60.) authorization_url = oauth2.get_authorization_url(redirect_uri, state=args.state, scopes=args.scopes) output('授权页面地址为:{}'.format(authorization_url)) output('浏览器启动中...') webbrowser.open(authorization_url, new=1, autoraise=False) output('请在浏览器登录并完成授权,复制跳转后页面链接的 code 参数并关闭浏览器。\n') while True: code = Prompt.ask('请输入浏览器回调地址携带的参数 [cyan]code[/cyan]') code = code.strip() if code: break output('[red]参数 code 不能为空[/red]') try: token_result = oauth2.exchange_authorization_code( code, redirect_uri=redirect_uri) except APIError as e: output('[red]令牌获取出错:[/red]') output_json(e.result) sys.exit(1) self._write_token_config(project, token_result) output('[green]令牌内容如下[/green]') content = [] for key in [ 'access_token', 'token_type', 'expires_in', 'scope', 'refresh_token' ]: content.append('{} = {}'.format(key, token_result[key])) output_syntax('\n'.join(content), lexer='INI')
def handler(self, args): access_token = args.access_token object_name = args.object_name mode = args.mode project = get_active_project() oauth2_section = DEFAULT_OAUTH2_SECTION token_section = DEFAULT_TOKEN_SECTION config = read_config(project) if not access_token: access_token = config_get(config.get, token_section, 'access_token') endpoint = config_get(config.get, oauth2_section, 'endpoint') api = API(access_token, endpoint=endpoint, timeout=18.) filename = posixpath.split(object_name)[1] if filename == '': output('[red]下载失败,object_name 不是文件:{}[/red]'.format( repr(object_name))) sys.exit(1) if posixpath.exists(filename): basename, suffix = posixpath.splitext(filename) i = 1 while True: new_basename = '{} ({})'.format(basename, i) filename = ''.join((new_basename, suffix)) if not posixpath.exists(filename): break i += 1 message = '正在下载 {}'.format(object_name) with console.status(message, spinner='earth'): try: with open(filename, mode) as fp: api.download(object_name, fp, region=args.region, expiration_time=args.expiration_time, chunk_size=args.chunk_size) except APIError as e: output('[red]请求失败:[/red]') output_json(e.result) sys.exit(1) output('[green]文件下载成功:[/green]{}'.format(filename))
def handler(self, args): access_token = args.access_token project = get_active_project() oauth2_section = DEFAULT_OAUTH2_SECTION token_section = DEFAULT_TOKEN_SECTION config = read_config(project) if not access_token: access_token = config_get(config.get, token_section, 'access_token') endpoint = config_get(config.get, oauth2_section, 'endpoint') api = API(access_token, endpoint=endpoint, timeout=18.) dirpath = args.dirpath cmk_id = args.cmk_id files = [] for filename in os.listdir(dirpath): filepath = join(dirpath, filename) if isfile(filepath): files.append(filepath) if not files: output('[red]文件夹中没有可上传的文件[/red]') sys.exit(1) message = '正在上传目录 {}'.format(dirpath) with console.status(message, spinner='earth'): try: object_names = api.upload_dir(dirpath, cmk_id=cmk_id) except APIError as e: output('[red]请求失败:[/red]') output_json(e.result) sys.exit(1) output('[green]文件上传成功:[/green]') table = Table(title="文件列表", expand=True, show_header=True, header_style="magenta") table.add_column("序号", style="cyan", no_wrap=True) table.add_column("存储位置", style="magenta", overflow='fold') for i, object_name in enumerate(object_names, 1): table.add_row(str(i), object_name) console.print(table)
def handler(self, args): project = get_active_project() config_path = get_config_path(project) config_parser = get_config_parser(config_path) oauth2_section = constants.DEFAULT_OAUTH2_SECTION client_id = config_get(config_parser.get, oauth2_section, 'client_id') client_secret = config_get( config_parser.get, oauth2_section, 'client_secret') endpoint = config_get(config_parser.get, oauth2_section, 'endpoint') oauth2 = OAuth2( client_id, client_secret, endpoint=endpoint, timeout=60.) try: token_result = oauth2.get_credentials_token() except APIError as e: output('[red]令牌获取出错:[/red]') output_json(e.result) sys.exit(1) self._write_token_config(project, token_result) console.rule('[green]令牌内容如下[/green]') content = [] for key in ['access_token', 'token_type', 'expires_in', 'scope']: content.append('{} = {}'.format(key, token_result[key])) output_syntax('\n'.join(content), lexer='INI')
def handler(self, args): access_token = args.access_token biosample_id = args.biosample_id project = get_active_project() oauth2_section = DEFAULT_OAUTH2_SECTION token_section = DEFAULT_TOKEN_SECTION config = read_config(project) if not access_token: access_token = config_get(config.get, token_section, 'access_token') endpoint = config_get(config.get, oauth2_section, 'endpoint') api = API(access_token, endpoint=endpoint, timeout=18.) try: result = api.get_taxon_abundance(biosample_id, taxon_ids=args.taxon_ids, next_page=args.next_page, limit=args.limit) except APIError as e: output('[red]请求失败:[/red]') output_json(e.result) sys.exit(1) output('[green]请求成功:[/green]') output_json(result, cls=ModelEncoder)
def rollback_model(self, args): version = args.version home = get_home() project = get_active_project() config_path = self.get_model_config_path() config = get_config_parser(config_path) section_name = DEFAULT_MODEL_SECTION model_id = config_get(config.get, section_name, 'model_id') oauth2_section = DEFAULT_OAUTH2_SECTION token_section = DEFAULT_TOKEN_SECTION config = read_config(project) access_token = config_get(config.get, token_section, 'access_token') endpoint = config_get(config.get, oauth2_section, 'endpoint') api = API(access_token, endpoint=endpoint, timeout=DEFAULT_MODEL_TIMEOUT) try: result = api.rollback_model(model_id, version) except APIError as e: output('[red]模型回滚失败:[/red]') output_json(e.result) sys.exit(1) task_id = result.task_id output('模型回滚任务:{}'.format(task_id)) task_path = join(home, '.bge', 'task_id') with open(task_path, 'w') as f: f.write(task_id) progress = self._wait_model_task(api, task_id, task_path) if 'SUCCESS' == progress: output('[green]模型 {} 灰度版已成功回滚至版本 {}。[/green]'.format( model_id, version)) elif 'FAILURE' == progress: output('[red]模型 {} 灰度回滚至版本 {} 失败。任务结果:{}[/red]'.format( model_id, version, result)) elif 'REVOKED' == progress: output('[white]模型 {} 灰度回滚至版本 {} 任务已被撤销失败。[/white]'.format( model_id, version))
def publish_model(self, args): """发布模型到稳定版""" message = args.message project = get_active_project() config_path = self.get_model_config_path() config = get_config_parser(config_path) section_name = DEFAULT_MODEL_SECTION model_id = config_get(config.get, section_name, 'model_id') oauth2_section = DEFAULT_OAUTH2_SECTION token_section = DEFAULT_TOKEN_SECTION config = read_config(project) access_token = config_get(config.get, token_section, 'access_token') endpoint = config_get(config.get, oauth2_section, 'endpoint') api = API(access_token, endpoint=endpoint, timeout=DEFAULT_MODEL_TIMEOUT) try: result = api.publish_model(model_id, message) except APIError as e: output('[red]模型发布失败:[/red]') output_json(e.result) sys.exit(1) output('[green]模型 {} 稳定版已成功发布。\n版本号:{}。[/green]'.format( model_id, result['version']))
def handler(self, args): access_token = args.access_token params = vars(args) for field in NOT_PARAM_FIELDS: params.pop(field, None) project = get_active_project() oauth2_section = DEFAULT_OAUTH2_SECTION token_section = DEFAULT_TOKEN_SECTION config = read_config(project) if not access_token: access_token = config_get(config.get, token_section, 'access_token') endpoint = config_get(config.get, oauth2_section, 'endpoint') api = API(access_token, endpoint=endpoint, timeout=18.) try: result = api.get_samples(**params) except APIError as e: output('[red]请求失败:[/red]') output_json(e.result) sys.exit(1) output('[green]请求成功:[/green]') output_json( result, cls=ModelEncoder )
def deploy_model(self, args): """部署模型""" ignore_source = args.ignore_source home = get_home() project = get_active_project() config_path = self.get_model_config_path() config = get_config_parser(config_path) section_name = DEFAULT_MODEL_SECTION model_id = config_get(config.get, section_name, 'model_id') timestr = datetime.now().strftime('%Y%m%d%H%M%S%f') randstr = uuid4().hex zip_filename = '{}.{}.{}.zip'.format(model_id, timestr, randstr) params = {} params['runtime'] = config_get(config.get, section_name, 'runtime') params['memory_size'] = config_get(config.getint, section_name, 'memory_size') params['timeout'] = config_get(config.getint, section_name, 'timeout') oauth2_section = DEFAULT_OAUTH2_SECTION token_section = DEFAULT_TOKEN_SECTION config = read_config(project) access_token = config_get(config.get, token_section, 'access_token') endpoint = config_get(config.get, oauth2_section, 'endpoint') api = API(access_token, endpoint=endpoint, timeout=DEFAULT_MODEL_TIMEOUT) object_name = None if not ignore_source: ignore_path = join(home, BGE_IGNORE_FILE) if not exists(ignore_path): output('未发现 .bgeignore 文件,初始化 {} ...'.format(ignore_path)) open(ignore_path, 'w').write(BGEIGNORE_TEMPLATE) minify_path = join(home, BGE_MINIFY_FILE) if not exists(minify_path): output('未发现 .bgeminify 文件,初始化 {} ...'.format(minify_path)) open(minify_path, 'w').write(BGEMINIFY_TEMPLATE) output('开始打包模型源码...') zip_tmpdir = join(home, '.bge', 'tmp') if not exists(zip_tmpdir): os.makedirs(zip_tmpdir) with tempfile.NamedTemporaryFile(suffix='.zip', prefix='model-', dir=zip_tmpdir, delete=False) as tmp: with zipfile.ZipFile(tmp.name, 'w', ZIP_COMPRESSION) as zf: self._zip_codedir(home, zf) tmp.flush() tmp.seek(0, 2) size = tmp.tell() tmp.seek(0) human_size = human_byte(size) if size > 100 * 1024 * 1024: output('打包后 zip 文件大小为 {},最大限制 100MB'.format(human_size)) exit(1) output('打包成功:{}'.format(tmp.name)) output('文件大小:{}'.format(human_size)) output('开始上传模型源码...') try: object_name = api.upload(zip_filename, tmp) except APIError as e: output('[red]上传模型源码失败:[/red]') output_json(e.result) sys.exit(1) output('[green]上传成功[green]') with console.status('模型部署中...', spinner='earth'): try: result = api.deploy_model(model_id, object_name=object_name, **params) except APIError as e: output('[red]部署模型失败:[/red]') output_json(e.result) sys.exit(1) task_id = result.task_id output('模型部署任务:{}'.format(task_id)) task_path = join(home, '.bge', 'task_id') with open(task_path, 'w') as f: f.write(task_id) output('模型部署任务返回结果:') progress = self._wait_model_task(api, task_id, task_path) if 'SUCCESS' == progress: output('[green]模型 {} 灰度部署成功。'.format(model_id)) elif 'FAILURE' == progress: output('[red]模型 {} 灰度部署失败。任务结果:{}'.format(model_id, result)) elif 'REVOKED' == progress: output('[white]模型 {} 灰度部署任务已被撤销。'.format(model_id))