def handler(self, args): access_token = args.access_token data_element_id = args.data_element_id project = get_active_project() oauth2_section = DEFAULT_OAUTH2_SECTION token_section = DEFAULT_TOKEN_SECTION config = read_config(project) if not access_token: access_token = config_get(config.get, token_section, 'access_token') endpoint = config_get(config.get, oauth2_section, 'endpoint') api = API(access_token, endpoint=endpoint, timeout=18.) try: result = api.get_range_stream(data_element_id, biosample_id=args.biosample_id, start_time=args.start_time, end_time=args.end_time, sort_direction=args.sort_direction, next_page=args.next_page, limit=args.limit) except APIError as e: output('[red]请求失败:[/red]') output_json(e.result) sys.exit(1) output('[green]请求成功:[/green]') output_json(result, cls=ModelEncoder)
def install_deps(self, args): package_name = args.package_name requirements = args.requirements upgrade = args.upgrade force_reinstall = args.force_reinstall pkgs = ' '.join(package_name) config_path = self.get_model_config_path() config = get_config_parser(config_path) section_name = DEFAULT_MODEL_SECTION model_id = config_get(config.get, section_name, 'model_id') runtime = config_get(config.get, section_name, 'runtime') client = self._get_docker_client() command = ['pip install', '--cache-dir', '/tmp/.cache/'] if upgrade: command.append('--upgrade') if force_reinstall: command.append('--force-reinstall') deps = [] if pkgs: deps.append(pkgs) if requirements: deps.append('-r {}'.format(requirements)) command.append('-t /code/lib {}'.format(' '.join(deps))) command = ' '.join(command) image_name = RUNTIMES[runtime] self._get_or_pull_image(client, image_name) with console.status('开始安装模型依赖包...', spinner='earth'): command = ('sh -c "{}"').format(command) container_name = generate_container_name(model_id) self._force_remove_container(client, container_name) self._run_by_container(client, image_name, command, container_name) output('[green]安装完成[/green]')
def handler(self, args): access_token = args.access_token project = get_active_project() oauth2_section = DEFAULT_OAUTH2_SECTION token_section = DEFAULT_TOKEN_SECTION config = read_config(project) if not access_token: access_token = config_get(config.get, token_section, 'access_token') endpoint = config_get(config.get, oauth2_section, 'endpoint') api = API(access_token, endpoint=endpoint, timeout=18.) try: result = api.aggregate_omics_data(args.data_element_id, args.time_dimension, args.start_time, end_time=args.end_time, biosample_id=args.biosample_id, interval=args.interval, periods=args.periods) except APIError as e: output('[red]请求失败:[/red]') output_json(e.result) sys.exit(1) output('[green]请求成功:[/green]') output_json(result, cls=ModelEncoder)
def handler(self, args): access_token = args.access_token namespace = args.namespace biosample_id = args.biosample_id project = get_active_project() oauth2_section = DEFAULT_OAUTH2_SECTION token_section = DEFAULT_TOKEN_SECTION config = read_config(project) if not access_token: access_token = config_get(config.get, token_section, 'access_token') endpoint = config_get(config.get, oauth2_section, 'endpoint') api = API(access_token, endpoint=endpoint, timeout=18.) try: result = api.get_data_items(namespace, biosample_id, collection_id=args.collection_id, data_element_ids=args.data_element_ids, next_page=args.next_page) except APIError as e: output('[red]请求失败:[/red]') output_json(e.result) sys.exit(1) output('[green]请求成功:[/green]') output_json(result, cls=ModelEncoder)
def workon_project(self, project): config_path = get_config_path(project) output('配置文件位置:{}'.format(config_path)) active_path = get_active_path() with open(active_path, 'w') as fp: fp.write(project) output('[green]已激活 {} 的项目配置'.format(project))
def add_project(self, args): project = args.project.lower() config_path = get_config_path(project, check_exists=False) if exists(config_path): output('[red]配置 {} 已存在'.format(project)) sys.exit(1) self._new_project(config_path)
def _new_project(self, config_path): config_parser = get_config_parser(config_path) oauth2_section = constants.DEFAULT_OAUTH2_SECTION if oauth2_section not in config_parser.sections(): config_parser.add_section(oauth2_section) for key, conf in constants.CLIENT_CREDENTIALS_CONFIGS: type_ = conf['type'] saved_value = self._get_config_value(config_parser, key, type_) secure = conf.get('secure') description = conf.get('description', '') value = saved_value or conf.get('default', '') show_value = value if secure and value: show_value = secure_str(value) input_value = Prompt.ask( '请输入{} {} ([bold cyan]{}[/bold cyan])'.format( description, key, show_value ), show_default=False, default=value ) if input_value: config_parser.set(oauth2_section, key, input_value) elif saved_value is None: if conf.get('default') is not None: config_parser.set( oauth2_section, key, conf.get('default')) else: config_parser.set(oauth2_section, key, '') with open(config_path, 'w') as config_file: config_parser.write(config_file) output('') output('[green]配置已保存至:[/green]{}'.format(config_path))
def _join_content(self, content): line_feed = '\n' templates = content.get('templates', None) arguments = content.get('arguments', None) if templates: description_lines = [] params = arguments.get('image', None) target_array = 0 for tem in templates: tem = str(tem) word = '{%s}' % 'image' if word in tem: word_count = tem.count(word) for index in range(target_array, target_array + word_count): try: data = params[index] except: output( '[red]arguments中的关键词数组应大于' '等于templates中的关键词数组[/red]') sys.exit(1) image_md = '''{}<br><center><img src="{}" width="386" height="386" /><div style="color: #999; ">{}</div></center><br>{}'''.format( line_feed, data['uri'], data['caption'], line_feed) tem = tem.replace(word, image_md, 1) target_array += word_count description_lines.append(tem) return description_lines return ['未提及', 'Not mentioned']
def handler(self, args): access_token = args.access_token biosample_id = args.biosample_id catalog = args.catalog data_type = args.data_type project = get_active_project() oauth2_section = DEFAULT_OAUTH2_SECTION token_section = DEFAULT_TOKEN_SECTION config = read_config(project) if not access_token: access_token = config_get(config.get, token_section, 'access_token') endpoint = config_get(config.get, oauth2_section, 'endpoint') api = API(access_token, endpoint=endpoint, timeout=18.) try: result = api.get_gene_abundance(biosample_id, catalog, data_type, ids=args.ids, next_page=args.next_page, limit=args.limit) except APIError as e: output('[red]请求失败:[/red]') output_json(e.result) sys.exit(1) output('[green]请求成功:[/green]') output_json(result, cls=ModelEncoder)
def clear_model(self, args): home = get_home() zip_tmpdir = join(home, '.bge', 'tmp') if confirm('是否清空目录 {} 下打包模型生成的临时文件?'.format(zip_tmpdir)): shutil.rmtree(zip_tmpdir, ignore_errors=True) os.mkdir(zip_tmpdir) output('[green]成功删除[/green]') else: output('[white]已取消[/white]')
def get_model_config_path(self, home=None): if home is None: home = get_home() config_dir = join(home, '.bge') model_config_path = join(home, 'model.ini') if not exists(model_config_path) or not exists(config_dir): output('[red]请确认当前是否为模型项目根目录。[/red]') sys.exit(1) return model_config_path
def get_docs_dir(self, home=None): if home is None: home = get_home() doc_path = join(home, 'index.html') if not exists(doc_path): output( '[red]请确认当前目录或者输入的项目路径是否为 docsify 项目根目录。[/red]' ) sys.exit(1) return home
def _input_model_id(self, default=''): prompt = '[?] 请输入解读模型编号' while True: model_id = Prompt.ask(prompt, default=default) if not model_id: if default: model_id = default break output('[red]必须提供模型编号[/red]') else: if model_id_match(model_id): break output('[red]模型编号只能包含数字、大小写字母[/red]') return model_id
def _read_model_project(self, args): config_path = self.get_model_config_path() output('配置文件:{}'.format(config_path)) output('配置详情:') output('') with open(config_path, 'r') as config_file: output(config_file.read())
def release_doc(self, args): project = get_active_project() config = read_config(project) access_token = config_get( config.get, DEFAULT_TOKEN_SECTION, 'access_token') endpoint = config_get(config.get, DEFAULT_OAUTH2_SECTION, 'endpoint') api = API( access_token, endpoint=endpoint, timeout=DEFAULT_MODEL_TIMEOUT) input_value = input('?请输入需要发布的 json 文件路径:') if input_value: if not exists(input_value): output('[red]文件路径:{} 有误,请检查。[/red]'.format(input_value)) sys.exit(1) doc_data = json.load(open(input_value)) doc_tab = doc_data['doc_tab'] model_id = doc_data['model_id'] doc_content = doc_data['doc_content'] try: result = api.upload_model_doc(doc_tab, model_id, doc_content) except APIError as e: output('[red]模型文档上传失败:[/red]') output_json(e.result) sys.exit(1) output('模型文档上传结果:') output_json(result) else: output('[red]请输入需要预览的文档路径[/red]') sys.exit(1)
def _zip_codedir(self, home, ziph): home = home.rstrip('/') total_files = {} zip_relpaths = set() for root, _, files in os.walk(home): for file in files: filepath = join(root, file) zip_relpath = relpath(filepath, home) zip_relpaths.add(zip_relpath) total_files[zip_relpath] = filepath src_zip_relpaths = zip_relpaths.copy() ignore_patterns = self.get_ignore_patterns(home) for ignore_pattern in ignore_patterns: if ignore_pattern.startswith('!'): ignore_pattern = ignore_pattern[1:] zip_relpaths.update( fnmatch.filter(src_zip_relpaths, ignore_pattern)) continue zip_relpaths -= set(fnmatch.filter(zip_relpaths, ignore_pattern)) minify_relpaths = set() minify_patterns = self.get_minify_patterns(home) for minify_pattern in minify_patterns: if minify_pattern.startswith('!'): minify_pattern = minify_pattern[1:] minify_relpaths -= set( fnmatch.filter(minify_relpaths, minify_pattern)) continue minify_relpaths.update( fnmatch.filter(src_zip_relpaths, minify_pattern)) # 仅支持混淆 .py 文件 minify_relpaths = fnmatch.filter(minify_relpaths, '*.py') with console.status('正在打包模型源码...', spinner='dots'): for zip_relpath in sorted(zip_relpaths): filepath = total_files[zip_relpath] if zip_relpath in minify_relpaths: continue ziph.write(filepath, zip_relpath) output('\t{}'.format(zip_relpath)) for zip_relpath in sorted(minify_relpaths): filepath = total_files[zip_relpath] with open(filepath, 'r') as fp: content = fp.read() with tempfile.NamedTemporaryFile(mode='w') as temp_fp: content = python_minifier.minify(content) temp_fp.write(content) temp_fp.flush() temp_fp.seek(0) ziph.write(temp_fp.name, zip_relpath) output('\t{} [green]MINIFIED[/green]'.format(zip_relpath))
def _write_token_config(self, project, token_result): access_token = token_result['access_token'] token_type = token_result['token_type'] expires_in = str(token_result['expires_in']) scope = token_result['scope'] config_path = get_config_path(project) config_parser = get_config_parser(config_path) section_name = constants.DEFAULT_TOKEN_SECTION if section_name not in config_parser.sections(): config_parser.add_section(section_name) config_parser.set(section_name, 'access_token', access_token) config_parser.set(section_name, 'token_type', token_type) config_parser.set(section_name, 'expires_in', expires_in) config_parser.set(section_name, 'scope', scope) with open(config_path, 'w') as config_file: config_parser.write(config_file) output('[green]令牌已保存至:[/green]{}'.format(config_path))
def _config_model(self, config=None): output('开始配置解读模型...') default_model_id = '' default_runtime = 'python3' default_memory_size = 128 default_timeout = 900 if config is not None: section_name = DEFAULT_MODEL_SECTION default_model_id = config_get(config.get, section_name, 'model_id') default_runtime = config_get(config.get, section_name, 'runtime') default_memory_size = config_get(config.getint, section_name, 'memory_size') default_timeout = config_get(config.getint, section_name, 'timeout') model_id = self._input_model_id(default=default_model_id) try: index = RUNTIME_CHOICES.index(default_runtime) except IndexError: index = 0 prompt = '请选择运行环境 [{}]'.format(default_runtime) input_prompt = "请选择" runtime = qprompt.enum_menu(RUNTIME_CHOICES).show(header=prompt, dft=index + 1, returns="desc", msg=input_prompt) try: index = MEMORY_SIZE_CHOICES.index(default_memory_size) except IndexError: index = 0 prompt = '请选择内存占用(MB) [{}]'.format(default_memory_size) input_prompt = "请选择" memory_size = qprompt.enum_menu(MEMORY_SIZE_CHOICES).show( header=prompt, dft=index + 1, returns="desc", msg=input_prompt) prompt = '请输入模型运行超时时间,类型为整数' while True: timeout = qprompt.ask_int(msg=prompt, dft=default_timeout) if (MIN_TIMEOUT and timeout < MIN_TIMEOUT) \ or (MAX_TIMEOUT and timeout > MAX_TIMEOUT): output('超出范围 [{},{}],请重新输入'.format(MIN_TIMEOUT or '', MAX_TIMEOUT or '')) continue break if not timeout: timeout = default_timeout return model_id, runtime, memory_size, timeout
def _install_sdk(self): with console.status("正在安装 bge-python-sdk", spinner="earth"): config_path = self.get_model_config_path() config = get_config_parser(config_path) section_name = DEFAULT_MODEL_SECTION model_id = config_get(config.get, section_name, 'model_id') runtime = config_get(config.get, section_name, 'runtime') client = self._get_docker_client() command = ('pip install --cache-dir /tmp/.cache/ --no-deps ' 'bge-python-sdk pimento requests_toolbelt -t /code/lib') image_name = RUNTIMES[runtime] self._get_or_pull_image(client, image_name) output('开始安装模型依赖包...') command = ('sh -c "{}"').format(command) container_name = generate_container_name(model_id) self._force_remove_container(client, container_name) self._run_by_container(client, image_name, command, container_name) output('[green]安装完成[/green]')
def start_model(self, args): port = args.port home = get_home() config_path = self.get_model_config_path() config = get_config_parser(config_path) section_name = DEFAULT_MODEL_SECTION model_id = config_get(config.get, section_name, 'model_id') runtime = config_get(config.get, section_name, 'runtime') client = self._get_docker_client() image_name = RUNTIMES[runtime] self._get_or_pull_image(client, image_name) command = 'python -u /server/app.py' user = get_sys_user() container_name = generate_container_name(model_id) self._force_remove_container(client, container_name) container = client.containers.run( image_name, command=command, name=container_name, volumes={home: { 'bind': WORKDIR, 'mode': 'rw' }}, stop_signal='SIGINT', ports={TEST_SERVER_PORT: port}, user=user, detach=True, stream=True, auto_remove=True) output('Model debug server is starting at {}...'.format(port)) output('Model {} was registered'.format(model_id)) output('\n\tURL: {}:{}/model/{}'.format(TEST_SERVER_ENDPOINT, port, model_id)) output('\tMethod: GET\n') try: logs = container.logs(stream=True, follow=True) for log in logs: output(log.strip().decode('utf-8')) finally: if container.status != 'exited': try: container.remove(force=True) except Exception: pass
def handler(self, args): access_token = args.access_token project = get_active_project() oauth2_section = DEFAULT_OAUTH2_SECTION token_section = DEFAULT_TOKEN_SECTION config = read_config(project) if not access_token: access_token = config_get(config.get, token_section, 'access_token') endpoint = config_get(config.get, oauth2_section, 'endpoint') api = API(access_token, endpoint=endpoint, timeout=18.) try: result = api.get_user() except APIError as e: output('[red]请求失败:[/red]') output_json(e.result) sys.exit(1) output('[green]请求成功:[/green]') output_json(result, cls=ModelEncoder)
def _get_or_pull_image(self, client, image_name): try: client.images.get(image_name) except docker.errors.NotFound: output( '[cyan]本地 docker 镜像 {} 不存在,开始拉取...[/cyan]'.format(image_name)) with console.status('拉取 docker 镜像中...', spinner='earth'): return_code = os.system('docker pull {}'.format(image_name)) if return_code != 0: output('[red]拉取镜像 {} 失败,请重试[/red]'.format(image_name)) sys.exit(1) output('[green]拉取镜像 {} 成功[/green]'.format(image_name)) else: with console.status('更新 docker 镜像中...', spinner='earth'): repository, tag = image_name.split(':') try: client.images.pull(repository, tag) except docker.errors.APIError: output('[red]更新镜像 {} 失败,请重试[/red]'.format(image_name)) sys.exit(1) output('[green]更新镜像 {} 成功[/green]'.format(image_name))
def handler(self, args): access_token = args.access_token filepath = args.filepath project = get_active_project() oauth2_section = DEFAULT_OAUTH2_SECTION token_section = DEFAULT_TOKEN_SECTION config = read_config(project) if not access_token: access_token = config_get(config.get, token_section, 'access_token') endpoint = config_get(config.get, oauth2_section, 'endpoint') api = API(access_token, endpoint=endpoint, timeout=18.) if not posixpath.isfile(filepath): output('[red]只能上传文件[/red]') sys.exit(1) filename = args.filename cmk_id = args.cmk_id if not filename: filename = posixpath.split(filepath)[1] message = '正在上传文件 {}'.format(filepath) with console.status(message, spinner='earth'): try: with open(filepath, 'rb') as fp: object_name = api.upload(filename, fp, cmk_id=cmk_id) except APIError as e: output('[red]请求失败:[/red]') output_json(e.result) sys.exit(1) output('[green]文件上传成功,存储位置:[/green]{}'.format(object_name))
def _wait_model_task(self, api, task_id, task_path): seconds = 0 while True: result = api.task(task_id) progress = result.progress if progress not in TOTAL_PROGRESS: output(TASK_ERROR_MESSAGE.format(seconds, progress)) try: os.unlink(task_path) except (IOError, OSError): pass sys.exit(1) output(WAIT_MESSAGE.format(seconds, TOTAL_PROGRESS[progress])) if progress in ('SUCCESS', 'FAILURE', 'REVOKED'): break sleep(1) seconds += 1 try: os.unlink(task_path) except (IOError, OSError): pass return progress
def handler(self, args): project = get_active_project() config_path = get_config_path(project) config_parser = get_config_parser(config_path) oauth2_section = constants.DEFAULT_OAUTH2_SECTION client_id = config_get(config_parser.get, oauth2_section, 'client_id') client_secret = config_get( config_parser.get, oauth2_section, 'client_secret') endpoint = config_get(config_parser.get, oauth2_section, 'endpoint') oauth2 = OAuth2( client_id, client_secret, endpoint=endpoint, timeout=60.) try: token_result = oauth2.get_credentials_token() except APIError as e: output('[red]令牌获取出错:[/red]') output_json(e.result) sys.exit(1) self._write_token_config(project, token_result) console.rule('[green]令牌内容如下[/green]') content = [] for key in ['access_token', 'token_type', 'expires_in', 'scope']: content.append('{} = {}'.format(key, token_result[key])) output_syntax('\n'.join(content), lexer='INI')
def _write_index(self, docs_dir): index_path = join(docs_dir, 'index.html') if exists(index_path) is False: output('[red]docs 目录下无 index.html[/red]') sys.exit(1) lines = [] with open(index_path, 'r') as f: for line in f.readlines(): lines.append(line) if line == ' window.$docsify = {\n': lines.append(' loadSidebar: true,\n') elif line == ' </script>\n': lines.append( ' <script src="//cdn.jsdelivr.net/npm/prismjs/' 'components/prism-json.min.js"></script>\n') lines.append( ' <script src="//cdn.jsdelivr.net/npm/docsify-kate' 'x@latest/dist/docsify-katex.js"></script>\n') lines.append( ' <link rel="stylesheet" href="//cdn.jsdelivr.net/' 'npm/katex@latest/dist/katex.min.css"/>\n') with open(index_path, 'w') as f: f.writelines(lines)
def rollback_model(self, args): version = args.version home = get_home() project = get_active_project() config_path = self.get_model_config_path() config = get_config_parser(config_path) section_name = DEFAULT_MODEL_SECTION model_id = config_get(config.get, section_name, 'model_id') oauth2_section = DEFAULT_OAUTH2_SECTION token_section = DEFAULT_TOKEN_SECTION config = read_config(project) access_token = config_get(config.get, token_section, 'access_token') endpoint = config_get(config.get, oauth2_section, 'endpoint') api = API(access_token, endpoint=endpoint, timeout=DEFAULT_MODEL_TIMEOUT) try: result = api.rollback_model(model_id, version) except APIError as e: output('[red]模型回滚失败:[/red]') output_json(e.result) sys.exit(1) task_id = result.task_id output('模型回滚任务:{}'.format(task_id)) task_path = join(home, '.bge', 'task_id') with open(task_path, 'w') as f: f.write(task_id) progress = self._wait_model_task(api, task_id, task_path) if 'SUCCESS' == progress: output('[green]模型 {} 灰度版已成功回滚至版本 {}。[/green]'.format( model_id, version)) elif 'FAILURE' == progress: output('[red]模型 {} 灰度回滚至版本 {} 失败。任务结果:{}[/red]'.format( model_id, version, result)) elif 'REVOKED' == progress: output('[white]模型 {} 灰度回滚至版本 {} 任务已被撤销失败。[/white]'.format( model_id, version))
def handler(self, args): access_token = args.access_token params = vars(args) for field in NOT_PARAM_FIELDS: params.pop(field, None) project = get_active_project() oauth2_section = DEFAULT_OAUTH2_SECTION token_section = DEFAULT_TOKEN_SECTION config = read_config(project) if not access_token: access_token = config_get(config.get, token_section, 'access_token') endpoint = config_get(config.get, oauth2_section, 'endpoint') api = API(access_token, endpoint=endpoint, timeout=18.) try: result = api.get_samples(**params) except APIError as e: output('[red]请求失败:[/red]') output_json(e.result) sys.exit(1) output('[green]请求成功:[/green]') output_json( result, cls=ModelEncoder )
def publish_model(self, args): """发布模型到稳定版""" message = args.message project = get_active_project() config_path = self.get_model_config_path() config = get_config_parser(config_path) section_name = DEFAULT_MODEL_SECTION model_id = config_get(config.get, section_name, 'model_id') oauth2_section = DEFAULT_OAUTH2_SECTION token_section = DEFAULT_TOKEN_SECTION config = read_config(project) access_token = config_get(config.get, token_section, 'access_token') endpoint = config_get(config.get, oauth2_section, 'endpoint') api = API(access_token, endpoint=endpoint, timeout=DEFAULT_MODEL_TIMEOUT) try: result = api.publish_model(model_id, message) except APIError as e: output('[red]模型发布失败:[/red]') output_json(e.result) sys.exit(1) output('[green]模型 {} 稳定版已成功发布。\n版本号:{}。[/green]'.format( model_id, result['version']))
def run_model(self, args): params = {} if args.file: filepath = args.file if not exists(filepath): output('文件不存在:{}'.format(filepath)) sys.exit(1) if isdir(filepath): output('存在文件夹:{}'.format(filepath)) sys.exit(1) with open(filepath, 'r') as fp: params = json.load(fp) elif args.args: params = dict(args.args) draft = args.draft test = args.test project = get_active_project() config_path = self.get_model_config_path() config = get_config_parser(config_path) section_name = DEFAULT_MODEL_SECTION model_id = config_get(config.get, section_name, 'model_id') oauth2_section = DEFAULT_OAUTH2_SECTION token_section = DEFAULT_TOKEN_SECTION config = read_config(project) access_token = config_get(config.get, token_section, 'access_token') endpoint = config_get(config.get, oauth2_section, 'endpoint') api = API(access_token, endpoint=endpoint, timeout=DEFAULT_MODEL_TIMEOUT) try: if draft is True: result = api.invoke_draft_model(model_id, **params) elif test is True: port = args.port api = API(access_token, endpoint='{}:{}'.format(TEST_SERVER_ENDPOINT, port), timeout=DEFAULT_MODEL_TIMEOUT) result = api.invoke_model(model_id, **params) else: result = api.invoke_model(model_id, **params) except APIError as e: output('[red]模型运行失败:[/red]') output_json(e.result) sys.exit(1) output('[green]模型返回值:[/green]') output_json(result.json())