def handler(self, args):
access_token = args.access_token
filepath = args.filepath
project = get_active_project()
oauth2_section = DEFAULT_OAUTH2_SECTION
token_section = DEFAULT_TOKEN_SECTION
config = read_config(project)
if not access_token:
access_token = config_get(config.get, token_section,
'access_token')
endpoint = config_get(config.get, oauth2_section, 'endpoint')
api = API(access_token, endpoint=endpoint, timeout=18.)
if not posixpath.isfile(filepath):
output('[red]只能上传文件[/red]')
sys.exit(1)
filename = args.filename
cmk_id = args.cmk_id
if not filename:
filename = posixpath.split(filepath)[1]
message = '正在上传文件 {}'.format(filepath)
with console.status(message, spinner='earth'):
try:
with open(filepath, 'rb') as fp:
object_name = api.upload(filename, fp, cmk_id=cmk_id)
except APIError as e:
output('[red]请求失败：[/red]')
output_json(e.result)
sys.exit(1)
output('[green]文件上传成功，存储位置：[/green]{}'.format(object_name))
def handler(self, args):
access_token = args.access_token
namespace = args.namespace
biosample_id = args.biosample_id
project = get_active_project()
oauth2_section = DEFAULT_OAUTH2_SECTION
token_section = DEFAULT_TOKEN_SECTION
config = read_config(project)
if not access_token:
access_token = config_get(config.get, token_section,
'access_token')
endpoint = config_get(config.get, oauth2_section, 'endpoint')
api = API(access_token, endpoint=endpoint, timeout=18.)
try:
result = api.get_data_items(namespace,
biosample_id,
collection_id=args.collection_id,
data_element_ids=args.data_element_ids,
next_page=args.next_page)
except APIError as e:
output('[red]请求失败：[/red]')
output_json(e.result)
sys.exit(1)
output('[green]请求成功：[/green]')
output_json(result, cls=ModelEncoder)
def upload_model_expfs(self, args):
home = get_home()
project = get_active_project()
config_path = self.get_model_config_path()
config = get_config_parser(config_path)
section_name = DEFAULT_MODEL_SECTION
model_id = config_get(config.get, section_name, 'model_id')
oauth2_section = DEFAULT_OAUTH2_SECTION
token_section = DEFAULT_TOKEN_SECTION
config = read_config(project)
access_token = config_get(config.get, token_section, 'access_token')
endpoint = config_get(config.get, oauth2_section, 'endpoint')
api = API(access_token,
endpoint=endpoint,
timeout=DEFAULT_MODEL_TIMEOUT)
try:
result = api.upload_model_expfs(model_id, 'uploadtest.zip')
except APIError as e:
output('[red]上传模型扩展集失败：[/red]')
output_json(e.result)
sys.exit(1)
task_id = result.task_id
output('上传模型扩展集任务：{}'.format(task_id))
task_path = join(home, '.bge', 'expfs.task_id')
with open(task_path, 'w') as f:
f.write(task_id)
output('上传模型扩展集任务返回结果：')
progress = self._wait_model_task(api, task_id, task_path)
if 'SUCCESS' == progress:
output('[green]模型 {} 上传模型扩展集成功。'.format(model_id))
elif 'FAILURE' == progress:
output('[red]模型 {} 上传模型扩展集失败。'.format(model_id))
elif 'REVOKED' == progress:
output('[white]模型 {} 上传模型扩展集任务已被撤销。'.format(model_id))
def _get_config_value(self, config_parser, key, type_):
oauth2_section = constants.DEFAULT_OAUTH2_SECTION
if 'str' == type_:
saved_value = config_get(
config_parser.get, oauth2_section, key)
elif 'int' == type_:
saved_value = config_get(
config_parser.getint, oauth2_section, key)
elif 'bool' == type_:
saved_value = config_get(
config_parser.getboolean, oauth2_section, key)
else:
raise ValueError('invalid type: {}'.format(type_))
return saved_value
def run_model(self, args):
params = {}
if args.file:
filepath = args.file
if not exists(filepath):
output('文件不存在：{}'.format(filepath))
sys.exit(1)
if isdir(filepath):
output('存在文件夹：{}'.format(filepath))
sys.exit(1)
with open(filepath, 'r') as fp:
params = json.load(fp)
elif args.args:
params = dict(args.args)
draft = args.draft
test = args.test
project = get_active_project()
config_path = self.get_model_config_path()
config = get_config_parser(config_path)
section_name = DEFAULT_MODEL_SECTION
model_id = config_get(config.get, section_name, 'model_id')
oauth2_section = DEFAULT_OAUTH2_SECTION
token_section = DEFAULT_TOKEN_SECTION
config = read_config(project)
access_token = config_get(config.get, token_section, 'access_token')
endpoint = config_get(config.get, oauth2_section, 'endpoint')
api = API(access_token,
endpoint=endpoint,
timeout=DEFAULT_MODEL_TIMEOUT)
try:
if draft is True:
result = api.invoke_draft_model(model_id, **params)
elif test is True:
port = args.port
api = API(access_token,
endpoint='{}:{}'.format(TEST_SERVER_ENDPOINT, port),
timeout=DEFAULT_MODEL_TIMEOUT)
result = api.invoke_model(model_id, **params)
else:
result = api.invoke_model(model_id, **params)
except APIError as e:
output('[red]模型运行失败：[/red]')
output_json(e.result)
sys.exit(1)
output('[green]模型返回值：[/green]')
output_json(result.json())
def _config_model(self, config=None):
output('开始配置解读模型...')
default_model_id = ''
default_runtime = 'python3'
default_memory_size = 128
default_timeout = 900
if config is not None:
section_name = DEFAULT_MODEL_SECTION
default_model_id = config_get(config.get, section_name, 'model_id')
default_runtime = config_get(config.get, section_name, 'runtime')
default_memory_size = config_get(config.getint, section_name,
'memory_size')
default_timeout = config_get(config.getint, section_name,
'timeout')
model_id = self._input_model_id(default=default_model_id)
try:
index = RUNTIME_CHOICES.index(default_runtime)
except IndexError:
index = 0
prompt = '请选择运行环境 [{}]'.format(default_runtime)
input_prompt = "请选择"
runtime = qprompt.enum_menu(RUNTIME_CHOICES).show(header=prompt,
dft=index + 1,
returns="desc",
msg=input_prompt)
try:
index = MEMORY_SIZE_CHOICES.index(default_memory_size)
except IndexError:
index = 0
prompt = '请选择内存占用（MB） [{}]'.format(default_memory_size)
input_prompt = "请选择"
memory_size = qprompt.enum_menu(MEMORY_SIZE_CHOICES).show(
header=prompt, dft=index + 1, returns="desc", msg=input_prompt)
prompt = '请输入模型运行超时时间，类型为整数'
while True:
timeout = qprompt.ask_int(msg=prompt, dft=default_timeout)
if (MIN_TIMEOUT and timeout < MIN_TIMEOUT) \
or (MAX_TIMEOUT and timeout > MAX_TIMEOUT):
output('超出范围 [{},{}]，请重新输入'.format(MIN_TIMEOUT or '',
MAX_TIMEOUT or ''))
continue
break
if not timeout:
timeout = default_timeout
return model_id, runtime, memory_size, timeout
def _install_sdk(self):
with console.status("正在安装 bge-python-sdk", spinner="earth"):
config_path = self.get_model_config_path()
config = get_config_parser(config_path)
section_name = DEFAULT_MODEL_SECTION
model_id = config_get(config.get, section_name, 'model_id')
runtime = config_get(config.get, section_name, 'runtime')
client = self._get_docker_client()
command = ('pip install --cache-dir /tmp/.cache/ --no-deps '
'bge-python-sdk pimento requests_toolbelt -t /code/lib')
image_name = RUNTIMES[runtime]
self._get_or_pull_image(client, image_name)
output('开始安装模型依赖包...')
command = ('sh -c "{}"').format(command)
container_name = generate_container_name(model_id)
self._force_remove_container(client, container_name)
self._run_by_container(client, image_name, command, container_name)
output('[green]安装完成[/green]')
def start_model(self, args):
port = args.port
home = get_home()
config_path = self.get_model_config_path()
config = get_config_parser(config_path)
section_name = DEFAULT_MODEL_SECTION
model_id = config_get(config.get, section_name, 'model_id')
runtime = config_get(config.get, section_name, 'runtime')
client = self._get_docker_client()
image_name = RUNTIMES[runtime]
self._get_or_pull_image(client, image_name)
command = 'python -u /server/app.py'
user = get_sys_user()
container_name = generate_container_name(model_id)
self._force_remove_container(client, container_name)
container = client.containers.run(
image_name,
command=command,
name=container_name,
volumes={home: {
'bind': WORKDIR,
'mode': 'rw'
}},
stop_signal='SIGINT',
ports={TEST_SERVER_PORT: port},
user=user,
detach=True,
stream=True,
auto_remove=True)
output('Model debug server is starting at {}...'.format(port))
output('Model {} was registered'.format(model_id))
output('\n\tURL: {}:{}/model/{}'.format(TEST_SERVER_ENDPOINT, port,
model_id))
output('\tMethod: GET\n')
try:
logs = container.logs(stream=True, follow=True)
for log in logs:
output(log.strip().decode('utf-8'))
finally:
if container.status != 'exited':
try:
container.remove(force=True)
except Exception:
pass
def model_versions(self, args):
limit = args.limit
next_page = args.next_page
project = get_active_project()
config_path = self.get_model_config_path()
config = get_config_parser(config_path)
section_name = DEFAULT_MODEL_SECTION
model_id = config_get(config.get, section_name, 'model_id')
oauth2_section = DEFAULT_OAUTH2_SECTION
token_section = DEFAULT_TOKEN_SECTION
config = read_config(project)
access_token = config_get(config.get, token_section, 'access_token')
endpoint = config_get(config.get, oauth2_section, 'endpoint')
api = API(access_token,
endpoint=endpoint,
timeout=DEFAULT_MODEL_TIMEOUT)
try:
result = api.model_versions(model_id,
limit=limit,
next_page=next_page)
except APIError as e:
output('[red]获取模型版本列表失败：[/red]')
output_json(e.result)
sys.exit(1)
output('第 {} 页，模型 {} 已发布版本：'.format(next_page or 1, model_id))
output('...')
length = None
items = result['result']
for item in items:
version = item['version']
message = item['message']
create_time = datetime.fromtimestamp(item['create_time'])
if length is None:
length = len(str(version))
output('发布时间：{}，版本号：{}，发布记录：{}'.format(create_time,
str(version).rjust(length),
message))
output('...')
size = len(items)
if size == 0 or size < limit or next_page == result['next_page']:
output('下一页：无')
else:
output('下一页：{}'.format(result['next_page']))
def handler(self, args):
access_token = args.access_token
project = get_active_project()
oauth2_section = DEFAULT_OAUTH2_SECTION
token_section = DEFAULT_TOKEN_SECTION
config = read_config(project)
if not access_token:
access_token = config_get(config.get, token_section,
'access_token')
endpoint = config_get(config.get, oauth2_section, 'endpoint')
api = API(access_token, endpoint=endpoint, timeout=18.)
try:
result = api.get_user()
except APIError as e:
output('[red]请求失败：[/red]')
output_json(e.result)
sys.exit(1)
output('[green]请求成功：[/green]')
output_json(result, cls=ModelEncoder)
def handler(self, args):
project = get_active_project()
config_path = get_config_path(project)
config_parser = get_config_parser(config_path)
oauth2_section = constants.DEFAULT_OAUTH2_SECTION
client_id = config_get(config_parser.get, oauth2_section, 'client_id')
client_secret = config_get(config_parser.get, oauth2_section,
'client_secret')
endpoint = config_get(config_parser.get, oauth2_section, 'endpoint')
redirect_uri = args.redirect_uri
oauth2 = OAuth2(client_id,
client_secret,
endpoint=endpoint,
timeout=60.)
authorization_url = oauth2.get_authorization_url(redirect_uri,
state=args.state,
scopes=args.scopes)
output('授权页面地址为：{}'.format(authorization_url))
output('浏览器启动中...')
webbrowser.open(authorization_url, new=1, autoraise=False)
output('请在浏览器登录并完成授权，复制跳转后页面链接的 code 参数并关闭浏览器。\n')
while True:
code = Prompt.ask('请输入浏览器回调地址携带的参数 [cyan]code[/cyan]')
code = code.strip()
if code:
break
output('[red]参数 code 不能为空[/red]')
try:
token_result = oauth2.exchange_authorization_code(
code, redirect_uri=redirect_uri)
except APIError as e:
output('[red]令牌获取出错：[/red]')
output_json(e.result)
sys.exit(1)
self._write_token_config(project, token_result)
output('[green]令牌内容如下[/green]')
content = []
for key in [
'access_token', 'token_type', 'expires_in', 'scope',
'refresh_token'
]:
content.append('{} = {}'.format(key, token_result[key]))
output_syntax('\n'.join(content), lexer='INI')
def handler(self, args):
access_token = args.access_token
object_name = args.object_name
mode = args.mode
project = get_active_project()
oauth2_section = DEFAULT_OAUTH2_SECTION
token_section = DEFAULT_TOKEN_SECTION
config = read_config(project)
if not access_token:
access_token = config_get(config.get, token_section,
'access_token')
endpoint = config_get(config.get, oauth2_section, 'endpoint')
api = API(access_token, endpoint=endpoint, timeout=18.)
filename = posixpath.split(object_name)[1]
if filename == '':
output('[red]下载失败，object_name 不是文件：{}[/red]'.format(
repr(object_name)))
sys.exit(1)
if posixpath.exists(filename):
basename, suffix = posixpath.splitext(filename)
i = 1
while True:
new_basename = '{} ({})'.format(basename, i)
filename = ''.join((new_basename, suffix))
if not posixpath.exists(filename):
break
i += 1
message = '正在下载 {}'.format(object_name)
with console.status(message, spinner='earth'):
try:
with open(filename, mode) as fp:
api.download(object_name,
fp,
region=args.region,
expiration_time=args.expiration_time,
chunk_size=args.chunk_size)
except APIError as e:
output('[red]请求失败：[/red]')
output_json(e.result)
sys.exit(1)
output('[green]文件下载成功：[/green]{}'.format(filename))
def handler(self, args):
access_token = args.access_token
project = get_active_project()
oauth2_section = DEFAULT_OAUTH2_SECTION
token_section = DEFAULT_TOKEN_SECTION
config = read_config(project)
if not access_token:
access_token = config_get(config.get, token_section,
'access_token')
endpoint = config_get(config.get, oauth2_section, 'endpoint')
api = API(access_token, endpoint=endpoint, timeout=18.)
dirpath = args.dirpath
cmk_id = args.cmk_id
files = []
for filename in os.listdir(dirpath):
filepath = join(dirpath, filename)
if isfile(filepath):
files.append(filepath)
if not files:
output('[red]文件夹中没有可上传的文件[/red]')
sys.exit(1)
message = '正在上传目录 {}'.format(dirpath)
with console.status(message, spinner='earth'):
try:
object_names = api.upload_dir(dirpath, cmk_id=cmk_id)
except APIError as e:
output('[red]请求失败：[/red]')
output_json(e.result)
sys.exit(1)
output('[green]文件上传成功：[/green]')
table = Table(title="文件列表",
expand=True,
show_header=True,
header_style="magenta")
table.add_column("序号", style="cyan", no_wrap=True)
table.add_column("存储位置", style="magenta", overflow='fold')
for i, object_name in enumerate(object_names, 1):
table.add_row(str(i), object_name)
console.print(table)
def handler(self, args):
project = get_active_project()
config_path = get_config_path(project)
config_parser = get_config_parser(config_path)
oauth2_section = constants.DEFAULT_OAUTH2_SECTION
client_id = config_get(config_parser.get, oauth2_section, 'client_id')
client_secret = config_get(
config_parser.get, oauth2_section, 'client_secret')
endpoint = config_get(config_parser.get, oauth2_section, 'endpoint')
oauth2 = OAuth2(
client_id, client_secret, endpoint=endpoint, timeout=60.)
try:
token_result = oauth2.get_credentials_token()
except APIError as e:
output('[red]令牌获取出错：[/red]')
output_json(e.result)
sys.exit(1)
self._write_token_config(project, token_result)
console.rule('[green]令牌内容如下[/green]')
content = []
for key in ['access_token', 'token_type', 'expires_in', 'scope']:
content.append('{} = {}'.format(key, token_result[key]))
output_syntax('\n'.join(content), lexer='INI')
def handler(self, args):
access_token = args.access_token
biosample_id = args.biosample_id
project = get_active_project()
oauth2_section = DEFAULT_OAUTH2_SECTION
token_section = DEFAULT_TOKEN_SECTION
config = read_config(project)
if not access_token:
access_token = config_get(config.get, token_section,
'access_token')
endpoint = config_get(config.get, oauth2_section, 'endpoint')
api = API(access_token, endpoint=endpoint, timeout=18.)
try:
result = api.get_taxon_abundance(biosample_id,
taxon_ids=args.taxon_ids,
next_page=args.next_page,
limit=args.limit)
except APIError as e:
output('[red]请求失败：[/red]')
output_json(e.result)
sys.exit(1)
output('[green]请求成功：[/green]')
output_json(result, cls=ModelEncoder)
def rollback_model(self, args):
version = args.version
home = get_home()
project = get_active_project()
config_path = self.get_model_config_path()
config = get_config_parser(config_path)
section_name = DEFAULT_MODEL_SECTION
model_id = config_get(config.get, section_name, 'model_id')
oauth2_section = DEFAULT_OAUTH2_SECTION
token_section = DEFAULT_TOKEN_SECTION
config = read_config(project)
access_token = config_get(config.get, token_section, 'access_token')
endpoint = config_get(config.get, oauth2_section, 'endpoint')
api = API(access_token,
endpoint=endpoint,
timeout=DEFAULT_MODEL_TIMEOUT)
try:
result = api.rollback_model(model_id, version)
except APIError as e:
output('[red]模型回滚失败：[/red]')
output_json(e.result)
sys.exit(1)
task_id = result.task_id
output('模型回滚任务：{}'.format(task_id))
task_path = join(home, '.bge', 'task_id')
with open(task_path, 'w') as f:
f.write(task_id)
progress = self._wait_model_task(api, task_id, task_path)
if 'SUCCESS' == progress:
output('[green]模型 {} 灰度版已成功回滚至版本 {}。[/green]'.format(
model_id, version))
elif 'FAILURE' == progress:
output('[red]模型 {} 灰度回滚至版本 {} 失败。任务结果：{}[/red]'.format(
model_id, version, result))
elif 'REVOKED' == progress:
output('[white]模型 {} 灰度回滚至版本 {} 任务已被撤销失败。[/white]'.format(
model_id, version))
def publish_model(self, args):
"""发布模型到稳定版"""
message = args.message
project = get_active_project()
config_path = self.get_model_config_path()
config = get_config_parser(config_path)
section_name = DEFAULT_MODEL_SECTION
model_id = config_get(config.get, section_name, 'model_id')
oauth2_section = DEFAULT_OAUTH2_SECTION
token_section = DEFAULT_TOKEN_SECTION
config = read_config(project)
access_token = config_get(config.get, token_section, 'access_token')
endpoint = config_get(config.get, oauth2_section, 'endpoint')
api = API(access_token,
endpoint=endpoint,
timeout=DEFAULT_MODEL_TIMEOUT)
try:
result = api.publish_model(model_id, message)
except APIError as e:
output('[red]模型发布失败：[/red]')
output_json(e.result)
sys.exit(1)
output('[green]模型 {} 稳定版已成功发布。\n版本号：{}。[/green]'.format(
model_id, result['version']))
def handler(self, args):
access_token = args.access_token
params = vars(args)
for field in NOT_PARAM_FIELDS:
params.pop(field, None)
project = get_active_project()
oauth2_section = DEFAULT_OAUTH2_SECTION
token_section = DEFAULT_TOKEN_SECTION
config = read_config(project)
if not access_token:
access_token = config_get(config.get, token_section, 'access_token')
endpoint = config_get(config.get, oauth2_section, 'endpoint')
api = API(access_token, endpoint=endpoint, timeout=18.)
try:
result = api.get_samples(**params)
except APIError as e:
output('[red]请求失败：[/red]')
output_json(e.result)
sys.exit(1)
output('[green]请求成功：[/green]')
output_json(
result,
cls=ModelEncoder
)
def deploy_model(self, args):
"""部署模型"""
ignore_source = args.ignore_source
home = get_home()
project = get_active_project()
config_path = self.get_model_config_path()
config = get_config_parser(config_path)
section_name = DEFAULT_MODEL_SECTION
model_id = config_get(config.get, section_name, 'model_id')
timestr = datetime.now().strftime('%Y%m%d%H%M%S%f')
randstr = uuid4().hex
zip_filename = '{}.{}.{}.zip'.format(model_id, timestr, randstr)
params = {}
params['runtime'] = config_get(config.get, section_name, 'runtime')
params['memory_size'] = config_get(config.getint, section_name,
'memory_size')
params['timeout'] = config_get(config.getint, section_name, 'timeout')
oauth2_section = DEFAULT_OAUTH2_SECTION
token_section = DEFAULT_TOKEN_SECTION
config = read_config(project)
access_token = config_get(config.get, token_section, 'access_token')
endpoint = config_get(config.get, oauth2_section, 'endpoint')
api = API(access_token,
endpoint=endpoint,
timeout=DEFAULT_MODEL_TIMEOUT)
object_name = None
if not ignore_source:
ignore_path = join(home, BGE_IGNORE_FILE)
if not exists(ignore_path):
output('未发现 .bgeignore 文件，初始化 {} ...'.format(ignore_path))
open(ignore_path, 'w').write(BGEIGNORE_TEMPLATE)
minify_path = join(home, BGE_MINIFY_FILE)
if not exists(minify_path):
output('未发现 .bgeminify 文件，初始化 {} ...'.format(minify_path))
open(minify_path, 'w').write(BGEMINIFY_TEMPLATE)
output('开始打包模型源码...')
zip_tmpdir = join(home, '.bge', 'tmp')
if not exists(zip_tmpdir):
os.makedirs(zip_tmpdir)
with tempfile.NamedTemporaryFile(suffix='.zip',
prefix='model-',
dir=zip_tmpdir,
delete=False) as tmp:
with zipfile.ZipFile(tmp.name, 'w', ZIP_COMPRESSION) as zf:
self._zip_codedir(home, zf)
tmp.flush()
tmp.seek(0, 2)
size = tmp.tell()
tmp.seek(0)
human_size = human_byte(size)
if size > 100 * 1024 * 1024:
output('打包后 zip 文件大小为 {}，最大限制 100MB'.format(human_size))
exit(1)
output('打包成功：{}'.format(tmp.name))
output('文件大小：{}'.format(human_size))
output('开始上传模型源码...')
try:
object_name = api.upload(zip_filename, tmp)
except APIError as e:
output('[red]上传模型源码失败：[/red]')
output_json(e.result)
sys.exit(1)
output('[green]上传成功[green]')
with console.status('模型部署中...', spinner='earth'):
try:
result = api.deploy_model(model_id,
object_name=object_name,
**params)
except APIError as e:
output('[red]部署模型失败：[/red]')
output_json(e.result)
sys.exit(1)
task_id = result.task_id
output('模型部署任务：{}'.format(task_id))
task_path = join(home, '.bge', 'task_id')
with open(task_path, 'w') as f:
f.write(task_id)
output('模型部署任务返回结果：')
progress = self._wait_model_task(api, task_id, task_path)
if 'SUCCESS' == progress:
output('[green]模型 {} 灰度部署成功。'.format(model_id))
elif 'FAILURE' == progress:
output('[red]模型 {} 灰度部署失败。任务结果：{}'.format(model_id, result))
elif 'REVOKED' == progress:
output('[white]模型 {} 灰度部署任务已被撤销。'.format(model_id))
