def _configure(request):
'''
展现configure页面
'''
context = {}
id = request.GET.get("template_id",None)
try:
try:
tem = services_template.objects.get(id=int(id))
except:
raise NameError,u"模板不存在"
context["template"] = tem
try:
rl = role.objects.filter(service=tem)
except:
rl = None
context["role"] = rl
except:
logger.error("_configure " + traceback.format_exc())
#这里需要处理
return render_to_response('configure.html',context,RequestContext(request))
def _role(request):
'''
展现role.html页面
'''
_template_id = request.GET.get("template_id",None)
_role = None
context = {}
try:
try:
_template = services_template.objects.get(id=int(_template_id))
context["template"] = _template
except:
logger.error("_configure_add_reg " + traceback.format_exc())
raise IndexError,"template id is invalid"
role_conf = []
_role = role.objects.filter(service=_template)
for item in _role:
_result = template_configure.objects.filter(id__in=[ x.configure_item.template_configure_id.id for x in role_conf_item.objects.filter(role_id=item) ])
role_conf.append([item,_result])
except:
logger.error("_role " + traceback.format_exc())
context["role"] = role_conf
return render_to_response('role.html',context,RequestContext(request))
def fetch_commit(request):
_need_add = request.POST.get("_need_add","").split("&_&")
_need_delete = request.POST.get("_need_delete","").split("&_&")
_exist_index = request.POST.get("_exist_index","").split("&_&")
ser_id = request.POST.get("ser_id",None)
context = {"result":"","status":"ok"}
try:
try:
ser = services.objects.get(id=ser_id)
except:
logger.error("fetch_commit " + traceback.format_exc())
traceback.print_exc()
raise IndexError,"无法找到对应的service"
_save(_need_add,1,ser)
_save(_need_delete,0,ser)
_save(_exist_index,0,ser)
except:
logger.error("fetch_commit " + traceback.format_exc())
traceback.print_exc()
context["status"] = "fail"
context["result"] = str(e)
return HttpResponse(json.dumps(context))
def _configure_content(request):
'''
当用户左击某个配置文件后，返回这个配置文件的所有配置项,每个配置项关联的role
'''
_id = request.POST.get("id",None)
result = {"status": "ok","result":[]}
try:
id = _id.split("_")[1]
_tc = template_configure.objects.get(id=int(id))
con = configure.objects.filter(template_configure_id=_tc).exclude(key__in=["#relative_role","node.master","node.data","http.enabled"])
for item in con:
role_name = role_conf_item.objects.filter(configure_item=item)
role_array = [[x.id,x.role_id.name] for x in role_name]
value = item.value.replace("\n","</br>").replace(" "," ")
result["result"].append({"key":item.key,"value":value,"id":item.id,"role":copy.deepcopy(role_array)})
except:
traceback.print_exc()
logger.error("_configure_content " + traceback.format_exc())
result["status"] = "fail"
return HttpResponse(json.dumps(result))
def _add(request):
'''
创建template
'''
name = request.POST.get("name","default").strip()
desc = request.POST.get("desc",'').strip()
icon = request.POST.get("img",None)
type = request.POST.get("type",name)
mock_flag = request.POST.get("mock_flag",False)
try:
st = services_template()
st.name = name
st.description = desc
st.type = type
st.icon = icon
st.save()
template_name = name
try:
web_request("master","pillar_module.pillar_operation",[template_name,"temp"],type,"add_template")
except:
logger.error("modify " + traceback.format_exc())
result = {"status": "ok"}
except Exception,e:
logger.error("_add " + traceback.format_exc())
result = {"status": "fail","result":str(e)}
def conf_file(request):
'''
展现conf_file 页面
'''
context = {}
id = request.GET.get("template_id",None)
try:
try:
tem = services_template.objects.get(id=int(id))
except:
raise NameError,u"模板不存在"
context["template"] = tem
try:
context["len"] = 0
conf = template_configure.objects.filter(services_template_id=tem)
for item in conf:
context["len"] = 1
except:
traceback.print_exc()
conf = None
context["conf"] = conf
except:
logger.error("_configure " + traceback.format_exc())
#这里需要处理
return render_to_response('conf_file.html',context,RequestContext(request))
def _add_role(request):
'''
增加role
'''
id = request.POST.get("id",None)
name = request.POST.get("name",str(uuid.uuid4())).strip().replace(".","_")
try:
new_item = role()
name = name.replace(".","_").strip()
new_item.name = name
new_item.service = services_template.objects.get(id=int(id))
result = {"status": "ok", "filename": name}
new_item.save()
type = new_item.service.type
name = new_item.service.name
try:
web_request("master","pillar_module.pillar_operation",[name,new_item.name],type,"add_role")
except:
logger.error("modify " + traceback.format_exc())
except:
logger.error("_add_role " + traceback.format_exc())
result = {"status": "fail", "filename": name}
return HttpResponse(json.dumps(result))
def _update_cluster(self,cluster_file,template,name,version):
result = {"result":"fail"}
_name = "clustername: " + name
try:
cl = services.objects.get(name=name)
except:
logger.error("add_cluster " + traceback.format_exc())
result["reason"] = "cluster invalid"
return result
try:
# es系统每台机器的 id是 grains["id"]来表示的，grains["id"] 就是target
ma_list = machine.objects.filter(id__in=[x["machine_id"] for x in instance_machine.objects.filter(ser_id=cl.id).values("machine_id")] ).values("target")
ma_list = ",".join([x["target"] for x in ma_list])
except:
logger.error("add_cluster " + traceback.format_exc())
result["reason"] = "machine invalid"
return result
cluster_ser = "es-zenPingUnicastHosts: '[" + ma_list + "]'"
es_version = "es-version: '" + version + "'"
#现在采取粗暴的重写方式("w")来修改文件，应该温柔点,现在只是将 集群的 name和 集群的机器列表当成 cluster的信息.
#默认所有集群的其他配置一致，这个需要再考虑
with salt.utils.fopen(cluster_file, 'w') as fp_:
print >> fp_,_name
print >> fp_,cluster_ser
print >> fp_,es_version
def _configure_pair(request):
'''
用户增加某个配置文件的 单个配置项
'''
id = request.POST.get("id",None)
name = request.POST.get("name",None)
value = request.POST.get("value",None)
_id = request.POST.get("_id",None)
try:
try:
id = _id.split("_")[1]
_tc = template_configure.objects.get(id=int(id))
except:
raise IndexError,"configure id is wrong"
co = configure()
co.template_configure_id = _tc
co.key = name
co.value = value
co.save()
__refresh_conf([_tc])
except:
logger.error("_add_role " + traceback.format_exc())
result = {"status": "ok"}
return HttpResponse(json.dumps(result))
def _configure_modify(request):
'''
修改某个配置的 value
'''
id = request.POST.get("id",None)
context = request.POST.get("context",None)
result = {"status": "ok"}
conf_file = []
try:
id = id.split("&")
context = context.split("&sperate&")
element = zip(id,context)
for item in element:
try:
conf = configure.objects.get(id=int(item[0])) #得到配置项
_tc = conf.template_configure_id #得到包含配置项的配置文件
except:
traceback.print_exc()
continue
conf.value = item[1].replace("\\n","\n")
conf.save()
conf_file.append(conf.template_configure_id)
__refresh_conf(conf_file)
except:
result["status"] = "fail"
logger.error("_configure_modify " + traceback.format_exc())
return HttpResponse(json.dumps(result))
def __rmdir(self,name):
try:
if os.path.isdir(name):
from shutil import rmtree
rmtree(name)
except:
logger.error("__rmdir " + traceback.format_exc())
def modify(request):
ser_id = request.POST.get("ser_id",None)
role_list = request.POST.get("role_list","").split("+")
context = {"status":"ok","result":""}
try:
if ser_id == None or not ser_id.isdigit():
raise NameError,"ser invalid"
#先删除所有ser_id关联的记录
instance_machine.objects.filter(ser_id=ser_id).delete()
#往instance_machine中存ser_id role_id machine_id的映射关系,其中ser_id和machine_id是必须的
for item in role_list:
id_list = item.split("_")
role_id = id_list[0] if len(id_list) > 0 else None
machine_id_list = id_list[1].split("&") if len(id_list) > 1 else []
for machine_id_item in machine_id_list:
if not machine_id_item.isdigit():
continue
try:
ser_item = services.objects.get(id=ser_id)
except:
continue
im = instance_machine()
im.ser_id = int(ser_id)
im.role_id = role_id
im.machine_id = machine_id_item
im.save()
'''
还要更新cluster信息，因为cluster的这个变量es-zenPingUnicastHosts
在有新机器加入时，需要修改,role_machine关联变化时es-zenPingUnicastHosts也要修改，
还未写修改变量的接口,
但是 可以 直接 重写cluster
'''
type = ser_item.belong_template.type
name = ser_item.belong_template.name
try:
web_request("master","pillar_module.pillar_operation",[name,ser_item.name],type,"add_cluster")
except:
logger.error("modify " + traceback.format_exc())
#更新所有target的pillar信息
web_request("master","pillar_module.pillar_operation",[],None,"_refresh")
except Exception,e:
context["status"] = "fail"
context["result"] = e
logger.error("modify " + traceback.format_exc())
traceback.print_exc()
def __mkdir(self,name):
try:
if not os.path.isdir(name):
mask = os.umask(002)
os.mkdir(name)
os.umask(mask)
except:
logger.error("__mkdir " + traceback.format_exc())
def _configure_add_reg(request):
'''
录入文件中的所有配置
'''
_name = request.POST.get("name","").strip()
_reg = request.POST.get("reg",None)
_id = request.POST.get("id",None)
_checked = request.POST.get("checked",None)
_mock_flag = request.POST.get("mock_flag",None)
try:
try:
id = _id.split("_")[1]
_tc = template_configure.objects.get(id=int(id))
except:
raise IndexError,"configure id is wrong"
conf_ele = {}
#获取关联这份配置文件的所有role
with open(os.path.join(files_path, _name), 'r') as destination:
_content = destination.read()
m = re.findall(_reg,_content)
for item in m:
if not len(item) == 2:
continue
if len(re.findall(r'[<>]',item[0])):
continue
if _checked == "checked" and item[0].strip() in conf_ele:
continue
if _checked == "checked":
conf_ele[item[0].strip()] = 0
co = configure()
co.template_configure_id = _tc
co.key = item[0].strip()
co.value = item[1].rstrip()
co.save()
result = {"status": "ok"}
__refresh_conf([_tc])
except:
logger.error("_configure_add_reg " + traceback.format_exc())
result = {"status": "fail"}
if _mock_flag:
return
else:
return HttpResponse(json.dumps(result))
def _del(request):
'''
删除整个template
'''
id = request.POST.get("id",None)
try:
item = services_template.objects.get(id=int(id))
result = {"status": "ok"}
type = item.type
name = item.name
try:
web_request("master","pillar_module.pillar_operation",[name,"temp"],type,"sub_template")
except:
logger.error("modify " + traceback.format_exc())
#还需要删除所有该cluster的 role，ser，instance,configure_file,configure
sv = services.objects.filter(belong_template=item)
for sv_item in sv:
instance_machine.objects.filter(ser_id=sv_item.id).delete()
indices.objects.filter(ser=sv_item).delete()
sv.delete()
rl = role.objects.filter(service=item)
for rl_item in rl:
rl_conf = role_conf_item.objects.filter(role_id=rl_item).delete()
rl.delete()
cf = template_configure.objects.filter(services_template_id=item)
for cf_item in cf:
conf = configure.objects.filter(template_configure_id=cf_item)
conf.delete()
cf.delete()
item.delete()
#更新所有target的pillar信息
web_request("master","pillar_module.pillar_operation",[],None,"_refresh")
#还要删除source配置
soure_root = settings.SOURCE_ROOT
template_path = soure_root + "/" + '/'.join([item.type,item.name])
from shutil import rmtree
rmtree(template_path)
except:
logger.error("_del " + traceback.format_exc())
result = {"status": "fail"}
return HttpResponse(json.dumps(result))
def cluster_com(request):
context = {"status":"ok","data":""}
com = request.POST.get("com",None)
id = request.POST.get("id",None)
new_name = request.POST.get("new_name",None)
try:
if com == "rename":
try:
cl = cluster.objects.get(id=int(id))
except:
raise NameError,"cluster name is invalid"
cl.name = new_name.strip()
cl.save()
else:
#依次删除service,instance_machine,role,service
__fetch_sql = '''
select instance_machine.id,
instance_machine.ser_id,
instance_machine.machine_id
from services
INNER JOIN instance_machine
ON services.id = instance_machine.ser_id
INNER JOIN machine
ON machine.id = instance_machine.machine_id
where machine.cluster = {0};
'''.format(id)
cursor = connection.cursor()
cursor.execute(__fetch_sql)
raw = cursor.fetchall()
in_ma = []
_ser = []
for item in raw:
in_ma.append(item[0])
_ser.append(item[1])
services.objects.filter(id__in=list(set(_ser))).delete()
for item in machine.objects.filter(cluster=int(id)):
item.cluster = None
item.save()
instance_machine.objects.filter(id__in=list(set(in_ma))).delete()
cluster.objects.filter(id=int(id)).delete()
except Exception,e:
logger.error("ser_command " + traceback.format_exc())
context = {"status":"fail","data":str(e)}
def highstate_commit(idsarry, job_id):
try:
web_request(",".join(idsarry), "saltutil.refresh_pillar", [], None, None)
cmd = 'salt -L "' + ",".join(idsarry) + '" state.highstate '
print "=========cmd=========", cmd
res = cmd
time.sleep(20)
result = web_request(",".join(idsarry), "state.highstate", [], None, None)
res = res + "\n" + str(result)
except:
logger.error("highstate_commit " + traceback.format_exc())
return res
def logstash_load(name):
ret = {"status":"ok"}
try:
st = services_template.objects.get(name=name)
ret = {"status":"fail","result":"已存在同名模板，不允许再创建哦"}
except:
try:
request = {"name":name,"desc":"logstash","img":"logstash.png","type":"logstash","mock_flag":True}
a = mock_http(request)
_add(a)
except Exception,e:
traceback.print_exc()
ret["status"] = "fail"
ret["result"] = e
logger.error("fetch_role " + traceback.format_exc())
def highstate_commit(request):
idarray = request.POST.get("id",None)
context = {"status":"ok"}
try:
web_request("master","pillar_module.pillar_operation",[],None,"_refresh")
idarray = idarray.split("&")
id = str(uuid.uuid1())
celery_highstate.delay(idarray,id)
context["job_id"] = id
except:
logger.error("highstate_commit " + traceback.format_exc())
context = {"status":"failed"}
return HttpResponse(json.dumps(context))
def _role_configure(request):
'''
获取某个role对应的某个配置的 值
'''
configure_id = request.GET.get("configure_id",None)
role_id = request.GET.get("role_id",None)
result = {"status": "ok"}
try:
rc = role_conf_item.objects.get(id=role_id)
result["result"] = rc.value
except:
traceback.print_exc()
result["status"] = "fail"
logger.error("_configure_modify " + traceback.format_exc())
return HttpResponse(json.dumps(result))
def highstate(request):
try:
all_machine = machine.objects.all()
context = {}
machine_list = []
for item in all_machine:
machine_list.append([item.id,item.target])
context["machine"] = machine_list
context["status"] = "ok"
except:
logger.error("highstate " + traceback.format_exc())
context = {"status":"failed"}
return HttpResponse(json.dumps(context))
def fetch_role(request):
id = request.POST.get("id",None)
ret = {"status":"ok"}
result = []
try:
st = services_template.objects.get(id=id)
all_role = role.objects.filter(service=st)
result = [[x.id,x.name] for x in all_role]
ret["result"] = result
except Exception,e:
traceback.print_exc()
ret["status"] = "fail"
ret["result"] = e
logger.error("fetch_role " + traceback.format_exc())
def _add_file(request):
'''
增加template时上传图标
'''
try:
f = request.FILES.get("upload_file",None)
name = str(uuid.uuid4())
with open(os.path.join(files_path, name), 'wb') as destination:
for chunk in f.chunks():
destination.write(chunk)
result = {"status": "ok", "filename": name}
except:
logger.error("_add_file " + traceback.format_exc())
result = {"status": "fail", "filename": name}
return HttpResponse(json.dumps(result))
def _get_role_conf(request):
'''
获取role list中的每个role 对应的 所有配置文件
'''
_id_list = request.POST.get("id","").split("&")
result = []
for item in _id_list:
try:
ret = {}
_role_item = role.objects.get(id=item)
ret["name"] = _role_item.name
ret["id"] = _role_item.id
ret["conf"] = []
_template = _role_item.service
_template_name = _template.name
_rc = role_configure.objects.filter(role_id=_role_item.id)
_conf = template_configure.objects.filter(services_template_id=_template)
_conf_ele = {}
for _item in _rc:
_conf_ele[_item.conf] = 0
for _item in _conf:
tag = 0
if _item.id in _conf_ele:
tag = 1
ret["conf"].append([tag,_item.id,_item.name])
result.append(ret)
except:
traceback.print_exc()
logger.error("_configure_add_reg " + traceback.format_exc())
continue
ret = {"status":"ok","result":result,"tn":_template_name}
return HttpResponse(json.dumps(ret))
def __add_ele(self,template,_type,name):
try:
name = str(name)
template_path = os.path.join(self._root_path,template)
self.__mkdir(template_path)
ele_path = os.path.join(template_path,_type)
self.__mkdir(ele_path)
os.system("rm -rf " + os.path.join(ele_path,name + ".sls"))
mask = os.umask(002)
os.system("touch " + os.path.join(ele_path,name + ".sls"))
os.umask(mask)
return os.path.join(ele_path,name + ".sls")
except:
logger.error("__add_ele " + traceback.format_exc())
return None
def role_conf(self,role_conf,template):
ret = []
try:
st = services_template.objects.get(name=template)
all_role = role.objects.filter(name__in=role_conf,service=st)
all_conf_id = role_conf_item.objects.filter(role_id__in=all_role)
all_conf_id = [x.configure_item.template_configure_id.id for x in all_conf_id]
result = template_configure.objects.filter(id__in=all_conf_id).values("name","deploy_path")
ret = []
for item in result:
if item["deploy_path"] == None or item["deploy_path"].strip() == "":
item.pop("deploy_path")
ret.append(item)
except:
logger.error("role_conf " + traceback.format_exc())
return ret
def update_elasticsearch_version_pillar(new_version, service_tmp, service_name):
# elasticsearch_sls=r"/srv/pillar/elasticsearch/template/elasticsearch.sls"
# elasticsearch_sls_bak=r"/srv/pillar/elasticsearch/template/elasticsearch.sls."+datetime.datetime.now().strftime('%y-%m-%d-%H:%M:%S');
# open(elasticsearch_sls_bak, "wb").write(open(elasticsearch_sls, "rb").read())
# f = open (elasticsearch_sls_bak, "r+")
# fr = open(elasticsearch_sls, 'w')
# fr.write(re.sub(r'es-version: "\d+\.\d+\.\d+"',r'es-version: "'+new_version+'"', f.read()))
# f.close()
# fr.close()
try:
result = web_request(
"master",
"pillar_module.pillar_operation",
[service_tmp, service_name, new_version],
"elasticsearch",
"update_cluster_version",
)
except:
logger.error("modify " + traceback.format_exc())
def read_elasticsearch_version_pillar(service_tmp, service_name):
# elasticsearch_sls=r"/srv/pillar/elasticsearch/template/elasticsearch.sls"
# fr = open(elasticsearch_sls, 'r')
# for line in fr.readlines():
# for i in re.findall(r'es-version: "(\d+\.\d+\.\d+)"',line):
# return i
# fr.close()
try:
result = web_request(
"master",
"pillar_module.pillar_operation",
[service_tmp, service_name],
"elasticsearch",
"read_cluster_version",
)
except:
logger.error("modify " + traceback.format_exc())
result = ""
return " ".join(result[0].values())
def machine_operations(id, action):
print id, action
context = {}
ret = ""
try:
if id == None:
raise NameError, "请选择instance"
_id_list = id.split("_")
_item = instance_machine.objects.filter(machine_id__in=_id_list)
print _item
_operation = {}
es__operation = set()
for _ele in _item:
try:
_target = machine.objects.get(id=int(_ele.machine_id)).target
except:
logger.error("_target fail " + traceback.format_exc())
context["result"].append([_ele.machine_id, u"instance记录不存在"])
continue
try:
_role = role.objects.get(id=int(_ele.role_id))
_template = _role.service.type
except:
logger.error("_role fail " + traceback.format_exc())
context["result"].append([_ele.machine_id, u"无法找到对应模板"])
continue
if _template == "elasticsearch":
es__operation.add(_target)
elif _template not in _operation:
_operation[_template] = set()
if not _template == "elasticsearch" and not _target in _operation[_template]:
_operation[_template].add(_target)
if not es__operation == set():
result = web_request(",".join(list(es__operation)), "elasticsearch." + action, [], None, None)
ret = ret + str(result)
for _tem, _targ in _operation.items():
result = web_request(",".join(list(_targ)), _tem + "." + action, [], None, None)
ret = ret + str(result)
context["result"] = ret
return ret
except:
logger.error("hosts_operations " + traceback.format_exc())
context = {"status": "failed"}
def _add_role(self,role_file,template,name):
try:
st = services_template.objects.get(name=template)
role_item = role.objects.get(name=name,service=st)
all_object = role_conf_item.objects.filter(role_id=role_item)
all_configure = []
for x in all_object:
yaml_regular = []
value_array = x.value.split("\n")
for index,item in enumerate(value_array):
#第一行和 最后一行必须是空2格
if index == 0 or index == len(value_array) - 1:
item = item.lstrip()
yaml_regular.append(" " + item)
all_configure.append(str(x.configure_item.id) + ": |\n" + "\n".join(yaml_regular) )
except:
logger.error("_add_role " + traceback.format_exc())
return
#现在采取粗暴的重写方式("w")来修改文件，应该温柔点
with salt.utils.fopen(role_file, 'w') as fp_:
print >> fp_,"\n".join(all_configure)
fields={"raw": Keyword()},
),
"country":
Nested(properties={
"name":
Text(
analyzer=html_strip,
fields={
"raw": Keyword(),
},
)
}),
})
class Index:
name = ADDRESS_ADDRESS_DOCUMENT_NAME
settings = {
"number_of_shards": 1,
"number_of_replicas": 1,
"blocks": {
"read_only_allow_delete": None
},
}
try:
# Create the mappings in Elasticsearch
Address.init()
except Exception as err:
logger.error(err)