def forward_request(prev_request):
path = prev_request.path
if path.startswith('/visible/'):
# http://localhost:8080/visible-ability/visible/24f61c090e5a4c4f8fadacb9b0815b60/
path = path[9:]
i = path.find('/')
deployment_uuid = path[:i]
filename = path[i + 1:]
res = uvisible_client.find_and_call_ability(deployment_uuid)
if res['status'] != 'ok':
raise Exception('未找到部署实例')
deployment = res['value']
k8s_port = deployment.get('k8sPort')
if k8s_port is None:
raise Exception('k8s实例端口停止服务')
internal_ip = g.get_central_config(
)['kubernetes']['ability']['internalIP']
url = 'http://{}:{}/{}'.format(internal_ip, k8s_port, filename)
res = requests.get(url=url, headers=prev_request.headers,
stream=True) # 允许分块传输
return {
'response': res,
}
raise Exception('unsupported API name')
async def get_up_socket(prev_request):
path = prev_request.path
if path.startswith('/visible/'):
# http://localhost:8080/visible-ability/visible/9002/
path = path[9:]
i = path.find('/')
deployment_uuid = path[:i]
filename = path[i + 1:]
res = uvisible_client.find_and_call_ability(deployment_uuid)
if res['status'] != 'ok':
raise Exception('未找到部署实例')
deployment = res['value']
k8s_port = deployment.get('k8sPort')
if k8s_port is None:
raise Exception('k8s实例端口停止服务')
internal_ip = g.get_central_config(
)['kubernetes']['ability']['internalIP']
logging.critical("internal_ip: {}".format(internal_ip))
protocol = prev_request.protocol
if protocol == "http":
protocol = "ws"
else:
protocol = "wss"
url = "{}://{}:{}/{}".format(protocol, internal_ip, k8s_port, filename)
# 建立websocket连接
ws = await websockets.connect(url)
return ws
raise Exception('当前不支持3')
def delete_artifact(url):
config = g.get_central_config()
username = config['nexus']['maven']['username']
password = config['nexus']['maven']['password']
try:
res = requests.delete(url, auth=(username, password))
except Exception as e:
logging.error(str(e))
def send_mail(receiver, message):
config = g.get_central_config()
sender = config['spring']['mail']['username']
smtp_client = smtplib.SMTP(config['spring']['mail']['host'], config['spring']['mail']['port'])
smtp_client.login(sender, config['spring']['mail']['password'])
try:
message['From'] = Header(sender, 'utf-8')
smtp_client.sendmail(sender, [receiver], message.as_string())
return True
except Exception as e:
return False
def get_artifact(url):
config = g.get_central_config()
username = config['nexus']['maven']['username']
password = config['nexus']['maven']['password']
try:
res = requests.get(url, auth=(username, password))
except Exception as e:
logging.error(str(e))
return None
if res and res.status_code == 200:
return str(res.content, encoding='utf-8')
else:
return None
def upload_artifact(short_url, file_path):
config = g.get_central_config()
base_url = config['nexus']['maven']['url']
username = config['nexus']['maven']['username']
password = config['nexus']['maven']['password']
long_url = '{}/{}'.format(base_url, short_url)
with open(file_path, "rb") as file:
try:
res = requests.put(long_url, data=file, auth=(username, password))
except Exception as e:
logging.error(str(e))
return long_url if res.status_code == 201 else None
async def delete_artifact(url):
config = g.get_central_config()
username = config['nexus']['maven']['username']
password = config['nexus']['maven']['password']
request = HTTPRequest(url=url,
method='DELETE',
auth_username=username,
auth_password=password,
request_timeout=600)
http = AsyncHTTPClient()
try:
await http.fetch(request)
except Exception as e:
pass
def upload_artifact_data(short_url, data):
config = g.get_central_config()
base_url = config['nexus']['maven']['url']
username = config['nexus']['maven']['username']
password = config['nexus']['maven']['password']
if isinstance(data, str):
data = bytes(data, encoding='utf-8')
long_url = '{}/{}'.format(base_url, short_url)
try:
res = requests.put(long_url, data=data, auth=(username, password))
except Exception as e:
logging.error(str(e))
return None
return long_url if res.status_code == 201 else None
async def get_artifact(url):
config = g.get_central_config()
username = config['nexus']['maven']['username']
password = config['nexus']['maven']['password']
request = HTTPRequest(url=url,
method='GET',
auth_username=username,
auth_password=password,
request_timeout=600)
http = AsyncHTTPClient()
try:
res = await http.fetch(request)
except Exception as e:
return None
if res and res.code == 200:
return str(res.body, encoding='utf-8')
else:
return None
async def upload_artifact(short_url, file_body):
config = g.get_central_config()
base_url = config['nexus']['maven']['url']
username = config['nexus']['maven']['username']
password = config['nexus']['maven']['password']
long_url = '{}/{}'.format(base_url, short_url)
request = HTTPRequest(url=long_url,
method='PUT',
body=file_body,
auth_username=username,
auth_password=password,
request_timeout=600)
http = AsyncHTTPClient()
try:
res = await http.fetch(request)
except Exception as e:
return None
return long_url if res.code == 201 else None
def delete_docker_image(url):
image_server, image_id = url.split('/')
image_name, image_tag = image_id.split(':')
config = g.get_central_config()
username = config['nexus']['docker']['registryUsername']
password = config['nexus']['docker']['registryPassword']
headers = {
'Authorization':
'Basic {}'.format(
str(base64.b64encode('{}:{}'.format(username, password).encode()),
encoding='utf-8')),
'Accept':
'application/vnd.docker.distribution.manifest.v2+json',
}
get_url = 'https://{}/v2/{}/manifests/{}'.format(image_server, image_name,
image_tag)
res = requests.get(url=get_url, headers=headers)
digest = res.headers.get('Docker-Content-Digest')
delete_url = 'https://{}/v2/{}/manifests/{}'.format(
image_server, image_name, digest)
requests.delete(url=delete_url, headers=headers)
def generate_microservice_remote(task, solution, base_path):
config = g.get_central_config()
docker_host = config['nexus']['docker']['host']
docker_port = config['nexus']['docker']['port']
docker_server = config['nexus']['docker']['imagetagPrefix']
username = config['nexus']['docker']['registryUsername']
password = config['nexus']['docker']['registryPassword']
image_id_local = '{}:{}'.format(solution.uuid, solution.version)
image_id_remote = '{}/{}'.format(docker_server, image_id_local)
save_task_step_progress(task.uuid, '创建微服务', '执行', 10, '开始创建微服务...')
save_task_step_progress(task.uuid, '创建微服务', '执行', 15, '从模型文件中提取元数据...')
try:
with open(os.path.join(base_path, 'application.yml'),
'r',
encoding='utf-8') as file:
yml = yaml.load(file, Loader=yaml.SafeLoader)
except:
return False
try:
python_version = yml['python']['version']
except:
save_task_step_progress(task.uuid, '创建微服务', '失败', 100,
'元数据文件中无Python版本号。')
return False
if python_version.startswith('3.5'):
dockerfile_text = dockerfiles.python35
elif python_version.startswith('3.6'):
dockerfile_text = dockerfiles.python36
elif python_version.startswith('3.7'):
dockerfile_text = dockerfiles.python37
else:
save_task_step_progress(task.uuid, '创建微服务', '失败', 100,
'元数据文件中Python版本号无效。')
return False
dockerfile_text = dockerfile_text.replace(r'{DOCKER-SERVER}',
docker_server)
dockerfile_path = os.path.join(base_path, 'Dockerfile')
with open(dockerfile_path, 'w', encoding='utf-8') as file:
file.write(dockerfile_text)
save_task_step_progress(task.uuid, '创建微服务', '执行', 30, '准备微服务运行环境...')
save_task_step_progress(task.uuid, '创建微服务', '执行', 40, '生成微服务docker镜像...')
docker_cilent = docker.DockerClient('{}:{}'.format(docker_host,
docker_port))
docker_images = docker_cilent.images
try:
docker_images.build(tag=image_id_local, path=base_path, rm=True)
except Exception as e:
logging.error(str(e))
save_task_step_progress(task.uuid, '创建微服务', '失败', 100,
'生成微服务docker镜像失败。')
return False
image_obj = docker_images.get(image_id_local)
image_size = image_obj.attrs.get('Size')
save_task_step_progress(task.uuid, '创建微服务', '执行', 70, '为docker镜像打tag...')
try:
image_obj.tag(image_id_remote)
except Exception as e:
logging.error(str(e))
save_task_step_progress(task.uuid, '创建微服务', '失败', 100,
'生成微服务docker镜像失败。')
try:
docker_images.remove(image_id_local)
except:
pass
return False
save_task_step_progress(task.uuid, '创建微服务', '执行', 80, '推送docker至镜像仓库...')
try:
docker_images.push(image_id_remote,
auth_config={
'username': username,
'password': password
})
except Exception as e:
logging.error(str(e))
save_task_step_progress(task.uuid, '创建微服务', '失败', 100,
'生成微服务docker镜像失败。')
try:
docker_images.remove(image_id_remote)
docker_images.remove(image_id_local)
except:
pass
return False
save_task_step_progress(task.uuid, '创建微服务', '执行', 85, '删除本地docker镜像...')
try:
docker_images.remove(image_id_remote)
docker_images.remove(image_id_local)
except:
pass
save_task_step_progress(task.uuid, '创建微服务', '执行', 90,
'创建docker镜像的Artifact对象并写入数据库...')
artifact = Artifact()
artifact.solutionUuid = solution.uuid
artifact.name = image_id_local
artifact.type = 'DOCKER镜像'
artifact.url = image_id_remote
artifact.fileSize = image_size
try:
umm_client.create_artifact(artifact, jwt=g.oauth_client.get_jwt())
except:
save_task_step_progress(task.uuid, '创建微服务', '失败', 100,
'创建docker镜像的Artifact对象并写入数据库失败。')
return False
save_task_step_progress(task.uuid, '创建微服务', '执行', 95, '成功创建微服务docker镜像。')
return True
def generate_microservice_local(task, solution, base_path):
config = g.get_central_config()
docker_server = config['nexus']['docker']['imagetagPrefix']
username = config['nexus']['docker']['registryUsername']
password = config['nexus']['docker']['registryPassword']
image_id_local = '{}:{}'.format(solution.uuid, solution.version)
image_id_remote = '{}/{}'.format(docker_server, image_id_local)
save_task_step_progress(task.uuid, '创建微服务', '执行', 10, '开始创建微服务...')
save_task_step_progress(task.uuid, '创建微服务', '执行', 15, '从模型文件中提取元数据...')
try:
with open(os.path.join(base_path, 'application.yml'),
'r',
encoding='utf-8') as file:
yml = yaml.load(file, Loader=yaml.SafeLoader)
except:
return False
try:
python_version = str(yml['python']['version'])
except:
save_task_step_progress(task.uuid, '创建微服务', '失败', 100,
'元数据文件中无Python版本号。')
return False
if python_version.startswith('3.5'):
dockerfile_text = dockerfiles.python35
elif python_version.startswith('3.6'):
dockerfile_text = dockerfiles.python36
elif python_version.startswith('3.7'):
dockerfile_text = dockerfiles.python37
else:
save_task_step_progress(task.uuid, '创建微服务', '失败', 100,
'元数据文件中Python版本号无效。')
return False
dockerfile_text = dockerfile_text.replace(r'{DOCKER-SERVER}',
docker_server)
dockerfile_path = os.path.join(base_path, 'Dockerfile')
with open(dockerfile_path, 'w', encoding='utf-8') as file:
file.write(dockerfile_text)
save_task_step_progress(task.uuid, '创建微服务', '执行', 30, '准备微服务运行环境...')
save_task_step_progress(task.uuid, '创建微服务', '执行', 40, '生成微服务docker镜像...')
res = os.system('docker build -t {} {}'.format(image_id_local, base_path))
if res != 0:
save_task_step_progress(task.uuid, '创建微服务', '失败', 100,
'生成微服务docker镜像失败。')
return False
output = subprocess.getoutput(
'docker image inspect {}'.format(image_id_local))
image_size = json.loads(output)[0].get('Size')
save_task_step_progress(task.uuid, '创建微服务', '执行', 70, '为docker镜像打tag...')
res = os.system('docker tag {} {}'.format(image_id_local, image_id_remote))
if res != 0:
save_task_step_progress(task.uuid, '创建微服务', '失败', 100,
'为docker镜像打tag失败。')
os.system('docker image rm {}'.format(image_id_local))
return False
save_task_step_progress(task.uuid, '创建微服务', '执行', 75, '登录镜像仓库...')
res = os.system('docker login -u {} -p {} {}'.format(
username, password, docker_server))
if res != 0:
save_task_step_progress(task.uuid, '创建微服务', '失败', 100, '登录镜像仓库失败。')
os.system('docker image rm {}'.format(image_id_remote))
os.system('docker image rm {}'.format(image_id_local))
return False
save_task_step_progress(task.uuid, '创建微服务', '执行', 80, '推送docker至镜像仓库...')
res = os.system('docker push {}'.format(image_id_remote))
if res != 0:
save_task_step_progress(task.uuid, '创建微服务', '失败', 100,
'推送docker至镜像仓库失败。')
os.system('docker image rm {}'.format(image_id_remote))
os.system('docker image rm {}'.format(image_id_local))
return False
save_task_step_progress(task.uuid, '创建微服务', '执行', 85, '删除本地docker镜像...')
os.system('docker image rm {}'.format(image_id_remote))
os.system('docker image rm {}'.format(image_id_local))
save_task_step_progress(task.uuid, '创建微服务', '执行', 90,
'创建docker镜像的Artifact对象并写入数据库...')
artifact = Artifact()
artifact.solutionUuid = solution.uuid
artifact.name = image_id_local
artifact.type = 'DOCKER镜像'
artifact.url = image_id_remote
artifact.fileSize = image_size
try:
umm_client.create_artifact(artifact, jwt=g.oauth_client.get_jwt())
except:
save_task_step_progress(task.uuid, '创建微服务', '失败', 100,
'创建docker镜像的Artifact对象并写入数据库失败。')
return False
save_task_step_progress(task.uuid, '创建微服务', '执行', 95, '成功创建微服务docker镜像。')
return True
def forward_request(prev_request):
path = prev_request.path
if path.startswith('/model/'):
deployment_uuid = path[7:]
res = umm_client.find_ability(deployment_uuid)
if res['status'] != 'ok':
raise Exception('未找到部署实例')
deployment = res['value']
k8s_port = deployment.get('k8sPort')
if k8s_port is None:
raise Exception('k8s实例端口停止服务')
internal_ip = g.get_central_config(
)['kubernetes']['ability']['internalIP']
url = 'http://{}:{}/api/model'.format(internal_ip, k8s_port)
res = requests.post(url=url,
data=prev_request.body,
headers=prev_request.headers)
return {
'response': res,
}
if path.startswith('/file/'):
deployment_uuid, method = path[6:].split('/')
res = umm_client.find_ability(deployment_uuid)
if res['status'] != 'ok':
raise Exception('未找到部署实例')
deployment = res['value']
k8s_port = deployment.get('k8sPort')
if k8s_port is None:
raise Exception('k8s实例端口停止服务')
internal_ip = g.get_central_config(
)['kubernetes']['ability']['internalIP']
url = 'http://{}:{}/api/file/{}'.format(internal_ip, k8s_port, method)
res = requests.post(url=url,
data=prev_request.body,
headers=prev_request.headers)
return {
'response': res,
}
if path.startswith('/stream/'):
deployment_uuid, method = path[8:].split('/')
res = umm_client.find_ability(deployment_uuid)
if res['status'] != 'ok':
raise Exception('未找到部署实例')
deployment = res['value']
k8s_port = deployment.get('k8sPort')
if k8s_port is None:
raise Exception('k8s实例端口停止服务')
internal_ip = g.get_central_config(
)['kubernetes']['ability']['internalIP']
url = 'http://{}:{}/api/stream/{}'.format(internal_ip, k8s_port,
method)
res = requests.post(url=url,
data=prev_request.body,
headers=prev_request.headers)
return {
'response': res,
}
if path.startswith('/web/'):
path = path[5:]
i = path.find('/')
deployment_uuid = path[:i]
filename = path[i + 1:]
res = umm_client.find_ability(deployment_uuid)
if res['status'] != 'ok':
raise Exception('未找到部署实例')
deployment = res['value']
k8s_port = deployment.get('k8sPort')
if k8s_port is None:
raise Exception('k8s实例端口停止服务')
internal_ip = g.get_central_config(
)['kubernetes']['ability']['internalIP']
url = 'http://{}:{}/web/{}'.format(internal_ip, k8s_port, filename)
res = requests.get(url=url, headers=prev_request.headers)
return {
'response': res,
}
raise Exception('unsupported API name')