def get_token(self):
panacube_ip = "http://" + GetModel.panacube_ip()
url = '{}/v1/management/login/'.format(panacube_ip)
username, password = GetModel.account()
current_app.logger.info('panacube登录username={}, password={}'.format(
username, password))
if username and password:
try:
resp = requests.post(url,
data={
"username": username,
"password": password
},
verify=False)
except Exception as e:
raise e
current_app.logger.info(resp.json())
return resp.json().get('data').get('token')
current_app.logger.error('参数有误--->username={}, password={}'.format(
username, password))
return {
'code':
1,
'message':
'参数有误--->username={}, password={}'.format(username, password)
}
def get_security_group_id(self):
"""新建安全组"""
CREATE_SECURITY_GROUPS = "http://{}/awstack-resource/v1/security_groups".format(
GetModel.awcloud_ip())
security_group = {"name": "panacube3.0", "description": "3.0项目部署"}
q.put({"type": "云平台", "speed_p": "stop", "percentage": "69"})
resp = requests.post(CREATE_SECURITY_GROUPS,
data=json.dumps(security_group),
headers=self.append_headers(),
verify=False)
# ['id]
group_obj = resp.json()
if 'code' in group_obj:
if group_obj['code'] == '02310100':
_url = "http://{}/awstack-resource/v1/security_groups".format(
GetModel.awcloud_ip())
data = requests.get(_url,
headers=self.append_headers(),
verify=False)
js_data = data.json()['data']['data']
_id = None
for i in js_data:
if i['name'] == 'default':
self.set_rule(i['id'], OPEN_PORT)
_id = i['id']
return {'id': _id}
if group_obj['data'] is not None:
self.set_rule(group_obj['data']['data']['id'], OPEN_PORT)
return group_obj
def get_node_info():
"""
获取所有的node节点信息
:return:
"""
if GetModel.deploy_type() == 'kvm':
headers = GetModel.get_auth_header()
return json.dumps({"code": 0, "data": ClearAwCloud(headers).get_node_info()})
else:
return json.dumps({"code": 0, "data": Panacube().get_node()})
def __init__(self, key=None, code=None):
self._session = requests.Session()
self._session.keep_alive = False
self._headers = {}
self.request_data = {
"enterpriseLoginName": "awcloud",
"userName": GetModel.account()[0],
"password": GetModel.account()[1],
"verificationCode": code,
"verificationKey": key
}
def push_bigdata_db():
# get_nodes()
time.sleep(4)
panaocs = GetModel.get_panaocs()
panaocs['netmask'] = exchange_mask(panaocs['netmask'])
# del panaocs['panacube_ip']
data = [
{
"param_name": "bigdata",
"param_value": panaocs,
"description": None,
"level": 1,
"update_time": "0000-00-00 00:00:00.000000",
"parent_id": None
},
{
"param_name": "panaocs",
"param_value": panaocs,
"description": None,
"level": 1,
"update_time": "0000-00-00 00:00:00.000000",
"parent_id": None
},
{
"param_name": "node",
"param_value": GetModel.get_node(),
"description": None,
"level": 1,
"update_time": "0000-00-00 00:00:00.000000",
"parent_id": None
}
]
if GetModel.deploy_type() == 'docker':
if os.path.exists('/usr/local/udsafe/parent_bigdata_info'):
node_list = os.popen("consul members | awk '{ print $1 }'").read().split('\n')[1:-1]
for node in node_list:
ssh_popen_1(node, del_param_name)
os.system(del_param_name)
for _list in data:
os.system(write_db.format(_list['param_name'], json.dumps(_list['param_value'])))
else:
from app_projects.models.m_insert import insert_sqls, delete_sql
if os.path.exists('/usr/local/udsafe/parent_bigdata_info'):
delete_sql()
insert_sqls(data)
current_app.logger.info('写入数据配置开始')
current_app.logger.info(data)
current_app.logger.info('写入数据配置结束')
q.put({"type": "大数据", "speed_p": "stop", "percentage": "100"})
def get_nodes():
db_update_panaocs()
insert_node = {}
node = GetModel.get_node()
count_cpu = GetModel.cpu_count()
for item in node:
# 将node节点数据写入数据库
insert_node[item['node-name']] = {
"cpu_start": int(count_cpu) - int(item['cpu']) + 1,
"mem": item['mem'],
"cpu_stop": count_cpu
}
current_app.logger.info('更新写入数据库的node数据 >>> {}'.format(insert_node))
model.update_or_create('node', insert_node)
def get_system_info(ip):
"""获取物理机的详情"""
# 字节转换GB单位
mem = int(
ssh_popen_1(ip,
"cat /proc/meminfo | grep MemTotal").split(':')[1].replace(
' ', '').split('kB')[0]) / 1024**2
_cpu_count = ssh_popen_1(
ip, "cat /proc/cpuinfo | grep 'processor' | wc -l") # 可用的总数
available_disk = ssh_popen_1(ip, "ceph osd df | grep TOTAL")
if available_disk:
_a_disk = available_disk.split(' ')[1:]
disk_data = []
for item in _a_disk:
if item:
if 'G' in item:
disk_data.append(int(item.split('G')[0]))
elif 'M' in item:
disk_data.append(int(item.split('M')[0]) / 1024.0)
data = {
"cpu": "4/{}".format(_cpu_count),
"mem": "8/{}".format(mem),
"disk": "{}/{}".format(disk_data[1], int(disk_data[0]))
}
model.update_or_create('count_cpu', int(_cpu_count) - 1)
model.update_or_create('count_mem', mem)
model.update_or_create('count_disk', int(disk_data[0]))
current_app.logger.info('-----model panaocs-----{}'.format(
GetModel.cpu_count()))
return data
return []
def db_update_panaocs():
"""
Write the assembled panaocs data to SQLite and return data
:return:
"""
node = GetModel.get_node()
# File last data before overwriting data
with open('/usr/local/udsafe/parent_bigdata_info', 'w') as f:
f.write(json.dumps(node))
try:
max_node_ip = max(int(i["network_ip"].split('.')[-1]) for i in node)
except Exception:
max_node_ip = max(int(i["network_ip"].split('.')[-1]) for i in node['node'])
panaocs_data = {
"zfs_pool": "bigdata",
"repo_ip": str(max_node_ip + 1),
"netmask": node[0]['netmask'],
"start_ip": str(max_node_ip + 1 + 5),
"link": "br0",
"panacube_ip": str(max_node_ip + 2),
"panacube_nic_name": "eth2",
"network_segment": '.'.join(node[0]['network_ip'].split('.')[:-1]) + '.*'
}
current_app.logger.info(panaocs_data)
model.update_or_create('panaocs', panaocs_data)
current_app.logger.info('大数据配置写入文件')
return panaocs_data
def update_image_info(self, image_id):
current_app.logger.info("storage image id: {}".format(image_id))
url = 'http://{}/awstack-resource/v1/image/{}'.format(
GetModel.awcloud_ip(), image_id)
request_data = {
"name": "智能存储镜像",
"disk_format": "qcow2",
"architecture": "x86_64",
"os_version": None,
"os_type": "Linux",
"os_distro": "CentOS X64",
"is_public": True,
"is_protected": False,
"imageUid": "a5eb23fb-0b3b-4cb2-b4ce-5886b191994c",
"vol_size": "10",
"minimum_disk": "10",
"hw_disk_bus": "scsi",
"hw_video_model": "vga",
"hw_vif_model": "virtio",
"hw_qemu_guest_agent": "true",
"hw_boot_menu": "false",
"hw_vif_multiqueue_enabled": "false"
}
headers = self.headers
headers['Content-Type'] = 'application/json;charset=UTF-8'
resp = requests.put(url,
data=json.dumps(request_data),
headers=headers,
verify=False)
current_app.logger.info("update storage image info: {}".format(
resp.json()))
return resp.json()
def get_or_set_system_info():
if request.method == 'GET':
"""
字符串拼接ip=192.168.102.91
返回对应服务器的数据
"""
ip = request.args.get("ip")
data =get_system_info(ip)
print 'cpu总数: ', GetModel.cpu_count()
return json.dumps({"code": 0, "data": data})
elif request.method == 'POST':
"""
设置节点信息
# _ = [{
# bigdata: 2
# cpu: 2
# ip: "192.168.66.11"
# mem: 2
# netmask: "255.255.255.0"
# network_ip: "192.168.56.66"
# network_name: "eno3"
# node-name: "node-1"
# }]
"""
r_data = request.json
current_app.logger.info("server accept node data: {}".format(r_data))
node_obj = r_data['node']
print '接受node节点数据', node_obj
for item in node_obj:
if isinstance(item, list):
return json.dumps({"code": 1, 'message': '数据有误'})
current_app.logger.info(r_data)
model.update_or_create('node', json.dumps(node_obj))
return json.dumps({"code": 0, 'message': 'Successful'})
def deploy_separate_bigdata():
if request.method == "GET":
"""
get获取所有的物理磁盘、排除当前移动硬盘
"""
headers = GetModel.get_auth_header()
return json.dumps({"data": ClearAwCloud(headers).get_sds_partitions(), "code": 0})
def block():
if str(GetModel.deploy_type()) == 'kvm':
if os.path.exists('/usr/local/udsafe/automatedkvm.txt'):
with open('/usr/local/udsafe/automatedkvm.txt', 'r') as f:
kvm_ip = f.read()
return json.dumps({'ip': kvm_ip})
else:
w_kvm = {
'ip': GetModel.kvm_ip()
}
os.system('mkdir -p /usr/local/udsafe/')
with open('/usr/local/udsafe/automatedkvm.txt', 'wb') as f:
f.write(GetModel.kvm_ip())
return json.dumps(w_kvm)
else:
return json.dumps({'ip': GetModel.panacube_ip()})
def set_rule(self, id, port):
"""
设置安全组的开放端口
:param id:
:param port:
:return:
"""
ADD_GROUP_RULE = "http://{}/awstack-resource/v1/security_groups/rule".format(
GetModel.awcloud_ip())
if isinstance(port, list):
for i in port:
rule_body = {
"cidr": "0.0.0.0/0",
"direction": "ingress",
"enabled": "1",
"ethertype": "IPV4",
"fromPort": i,
"groupId": id,
"ipProtocol": "tcp",
"toPort": i
}
if i == "-1":
rule_body['ipProtocol'] = "icmp"
requests.post(ADD_GROUP_RULE,
data=json.dumps(rule_body),
headers=self.append_headers(),
verify=False)
def clear_code(token):
"""
清除验证码
:param token:
:return:
"""
url = 'http://{}:80/awstack-user/v1/params'.format(GetModel.awcloud_ip())
headers = {
"Content-Type": "application/json",
"X-Auth-Token": token,
}
body = {
"paramId": 917,
"enterpriseUid": "0",
"regionUid": "0",
"regionKey": "0",
"parentId": 1,
"paramName": "LOGIN_CAPTCHA_ISOPEN",
"path": "1/917/",
"paramValue": "-1"
}
resp = requests.put(url, headers=headers, data=json.dumps(body))
if resp.status_code == 200:
return True
return False
def set_system_info(ip, cpu, mem):
"""设置nova配置文件 cpu和内存"""
vcpu_pin_set = '4-{}'.format(GetModel.cpu_count() - cpu)
mem_set = (mem + 8) * 1024
out_cpu = ssh_popen_1(ip, "cat /etc/nova/nova.conf | grep vcpu_pin_set")
out_mem = ssh_popen_1(
ip, "cat /etc/nova/nova.conf | grep reserved_host_memory_mb")
if out_cpu:
ssh_popen_1(
ip, "sed -i 's/{}/vcpu_pin_set = {}/' {file}".format(
out_cpu, vcpu_pin_set, file="/etc/nova/nova.conf"))
else:
ssh_popen_1(
ip, "echo vcpu_pin_set = {} >> {file}".format(
vcpu_pin_set, file="/etc/nova/nova.conf"))
# 设置内存 单位为M
if out_mem:
ssh_popen_1(
ip, "sed -i 's/{}/reserved_host_memory_mb = {}/' {file}".format(
out_mem, mem_set, file="/etc/nova/nova.conf"))
else:
ssh_popen_1(
ip, "echo reserved_host_memory_mb = {} >> {file}".format(
mem_set, file="/etc/nova/nova.conf"))
current_app.logger.info('设置nova配置完成')
def get_node_info(self):
"""
获取node物理节点
:return:
"""
GET_NODE = "http://{}/awstack-user/v1/enterprises/66666666666666666666666666666666/regions/8b4f22a536f845769820b5777d549238/nodes/list".format(
GetModel.awcloud_ip())
resp = requests.get(GET_NODE, verify=False)
return resp.json()
def get_cloud_status(self, cloud_id):
url = 'http://{}/awstack-resource/v1/server/{}'.format(
GetModel.awcloud_ip(), cloud_id)
resp = requests.get(url, headers=self.append_headers(), verify=False)
current_app.logger.info("get cloud ID {} status: {}".format(
cloud_id,
resp.json().get('data').get('data').get('status')))
if resp.json()['data']['data'].get('status') == 'ACTIVE':
return True
return False
def login_params(self):
authUrl = 'http://{}'.format(GetModel.awcloud_ip())
return {
"url": authUrl + "/awstack-user/v1/login",
"json": self.request_data,
"headers": {
"Content-Type": "application/json"
}
}
def verify_ip():
"""
验证ip是否可用
:return:
"""
ip = request.args.get("ip")
network_id = request.args.get("network_id")
model.update_or_create('network_id', network_id)
model.update_or_create('kvm_ip', ip)
data = ClearAwCloud(GetModel.get_auth_header()).get_network_card(ip, network_id)
return json.dumps({"code": 0, "data": data})
def get_sds_partitions(self):
import copy
url = 'http://{}/awstack-user/v1/region/FFFFF/ceph'.format(
GetModel.awcloud_ip())
headers = copy.deepcopy(self.headers)
headers['X-Register-Code'] = 'FFFFF'
resp = requests.get(url, headers=headers, verify=False)
resp_data = resp.json()
if resp_data['code'] == '02120701':
raise ValueError('获取osd tree信息失败')
return resp_data
def get_volume_type(self):
"""
获取云硬盘ID
:return:
"""
GET_VOLUMES = "http://{}/awstack-user/v1/storage/list".format(
GetModel.awcloud_ip())
resp = requests.get(GET_VOLUMES,
headers=self.append_headers(),
verify=False)
return resp.json()
def push_message():
"""
消息推送
:return:
"""
URL = 'http://{}/awstack-user/v1/params'.format(GetModel.awcloud_ip())
current_app.logger.info('message push API: {}'.format(URL))
manager = AWCloudManage('', '')
is_login = True
while is_login:
try:
manager._login()
is_login = False
except ConnectionError as e:
print "docker 未启动连接失败"
time.sleep(3)
headers = manager.get_auth_header()
body = {
"enterpriseUid":
"66666666666666666666666666666666",
"paramValue":
"http://{}/v1/awcloud-message/".format(GetModel.panacube_ip()),
"paramName":
"resource_pass",
"parentId":
939,
"regionUid":
"8b4f22a536f845769820b5777d549238",
"paramLevel":
2,
"paramDesc":
"消息推送",
"regionKey":
0
}
resp = requests.post(URL, data=json.dumps(body), headers=headers)
current_app.logger.info("message return info: {}".format(resp.json()))
if resp.json().get('code') == '01170101':
print 'push message: 参数名称已存在'
current_app.logger.info('push message: 添加消息推送成功')
def create_flavor(self):
"""创建规格"""
CREATE_FLAVOR = "http://{}/awstack-resource/v1/flavor".format(
GetModel.awcloud_ip())
q.put({"type": "云平台", "speed_p": "stop", "percentage": "49"})
resp = requests.post(CREATE_FLAVOR,
data=json.dumps(flavor_info),
headers=self.append_headers(),
verify=False)
js_data = resp.json()
current_app.logger.info(
'create flavor return data: {}'.format(js_data))
if js_data['code'] == '02013201':
d = requests.get("http://{}/awstack-resource/v1/flavors".format(
GetModel.awcloud_ip()),
headers=self.headers,
verify=False)
for i in d.json()['data']['data']:
if i['name'] == 'panacube3.0':
return {"id": i['id']}
return js_data
def insert_sqls(data):
print 'KVM>>>IP>>> {}'.format(GetModel.kvm_ip())
db = re_conn()
curr = db.cursor()
if isinstance(data, list):
for item in data:
if isinstance(item, dict):
insert_sql(item)
else:
logger.info('写入数据库参数类型有误---dict类型')
return False
db.close()
else:
logger.info('写入数据库参数类型有误----list类型')
return False
def get_external_network(self):
"""
获取外部网络
:return:
"""
# ip = model.get_params('awcloud_ip')
ip = GetModel.awcloud_ip()
current_app.logger.info('海云IP地址为--->{}'.format(ip))
GET_EXTERNAL_NETWORK = "http://{}/awstack-resource/v1/getExternalNetworks".format(
ip)
resp = requests.get(GET_EXTERNAL_NETWORK,
headers=self.headers,
verify=False)
return resp.json()
def get_network_card(self, ip, network_id):
"""
查询该ip是否可用
:param ip: ip是需要设置panacube3.0的IP network_id是外部网络ｉｄ
:return:
"""
_url = 'http://{}/awstack-resource/v1/networks/{}'.format(
GetModel.awcloud_ip(), network_id)
resp = requests.get(_url, headers=self.headers, verify=False)
for i in resp.json()['data']['data']:
if ip in i['ip']:
current_app.logger.info('创建云主机Ip已存在---{}'.format(ip))
return "null"
current_app.logger.info('验证KVM ＩＰ: {}'.format(ip))
return ip
def upload_image(self, path, _type="bigdata"):
"""
上传镜像
:param path: 镜像path
:param name: 镜像名字
:param _t: 镜像类型
:param size: 镜像大小
:return:
"""
UPLOAD_IMAGE = "http://{}/awstack-resource/v1/uploadimagez".format(
GetModel.awcloud_ip())
self.create_project()
image_data = {
'file': ('demo.iso', open(path, 'rb')),
"videoQueue": "false",
"qemu": "false",
"diskCtrl": "false",
"disk_format": "qcow2",
"architecture": "x86_64",
"os_distro": "CentOS X64",
"os_type": "Linux",
}
if _type == "bigdata":
image_data.update(name="智能存储镜像",
vol_size="3",
is_public="true",
minimum_disk="3")
else:
image_data.update(name="panacube3.0",
vol_size="2",
is_public="false",
minimum_disk="2")
m = MultipartEncoder(
fields=image_data,
boundary='---------------------------7de1ae242c06ca')
_headers = self.append_headers()
m = MultipartEncoderMonitor(m)
_headers["Content-Type"] = m.content_type
resp = requests.post(UPLOAD_IMAGE,
data=m,
headers=_headers,
verify=False)
return resp.json()
def get_node(self):
panacube_ip = "http://" + GetModel.panacube_ip()
ip_data = []
url = '{}/v1/monitor/undefined/TubePhyHost/'.format(panacube_ip)
get_token = self.get_token()
if 'code' in get_token and get_token['code'] == 1:
return get_token
resp = requests.get(
url,
headers={"Authorization": 'TOKEN ' + self.get_token()},
verify=False)
current_app.logger.info("get nodes info: {}".format(resp.json()))
for i in resp.json()['data']:
ip_data.append({
"node-name": i['hostName'],
"ip": i['hostInfo']['ips']['br_mgmt']
})
return ip_data
def accept_info():
"""
获取获取2.0或者3.0的账号密码 -> GET获取外部网络信息
# {
# "awcloud_ip": "",
# "username": "******",
# "password": "******",
# "panacube_ip": 3
# }
:return:
"""
if request.method == "GET":
r_data = ClearAwCloud(GetModel.get_auth_header()).get_external_network()
if 'code' in r_data:
return json.dumps({'code': 1})
return json.dumps({'data': r_data, 'code': 0})
r_data = request.json
model.update_or_create('panacube', r_data)
return json.dumps({"code": 0, 'message': 'Successful'})
def kvm_image_upload():
"""
上传panacube镜像 需要2.0的账号密码
:return:
"""
id2 = ClearAwCloud(GetModel.get_auth_header()).upload_image(
PathDir.image_path(None), 'panacube3.0')
q.put({
"type": "镜像上传",
"size": "上传大小",
"speed": "conducting",
"percentage": "{}".format(generate_random(4, 15))
})
current_app.logger.info('upload kvm image: {}'.format(id2))
model.update_or_create('panacube_img_id', id2['id'])
q.put({
"type": "镜像上传",
"size": "上传大小",
"speed": "conducting",
"percentage": "{}".format(generate_random(16, 32))
})