def _configure(request): ''' 展现configure页面 ''' context = {} id = request.GET.get("template_id",None) try: try: tem = services_template.objects.get(id=int(id)) except: raise NameError,u"模板不存在" context["template"] = tem try: rl = role.objects.filter(service=tem) except: rl = None context["role"] = rl except: logger.error("_configure " + traceback.format_exc()) #这里需要处理 return render_to_response('configure.html',context,RequestContext(request))
def _role(request): ''' 展现role.html页面 ''' _template_id = request.GET.get("template_id",None) _role = None context = {} try: try: _template = services_template.objects.get(id=int(_template_id)) context["template"] = _template except: logger.error("_configure_add_reg " + traceback.format_exc()) raise IndexError,"template id is invalid" role_conf = [] _role = role.objects.filter(service=_template) for item in _role: _result = template_configure.objects.filter(id__in=[ x.configure_item.template_configure_id.id for x in role_conf_item.objects.filter(role_id=item) ]) role_conf.append([item,_result]) except: logger.error("_role " + traceback.format_exc()) context["role"] = role_conf return render_to_response('role.html',context,RequestContext(request))
def fetch_commit(request): _need_add = request.POST.get("_need_add","").split("&_&") _need_delete = request.POST.get("_need_delete","").split("&_&") _exist_index = request.POST.get("_exist_index","").split("&_&") ser_id = request.POST.get("ser_id",None) context = {"result":"","status":"ok"} try: try: ser = services.objects.get(id=ser_id) except: logger.error("fetch_commit " + traceback.format_exc()) traceback.print_exc() raise IndexError,"无法找到对应的service" _save(_need_add,1,ser) _save(_need_delete,0,ser) _save(_exist_index,0,ser) except: logger.error("fetch_commit " + traceback.format_exc()) traceback.print_exc() context["status"] = "fail" context["result"] = str(e) return HttpResponse(json.dumps(context))
def _configure_content(request): ''' 当用户左击某个配置文件后,返回这个配置文件的所有配置项,每个配置项关联的role ''' _id = request.POST.get("id",None) result = {"status": "ok","result":[]} try: id = _id.split("_")[1] _tc = template_configure.objects.get(id=int(id)) con = configure.objects.filter(template_configure_id=_tc).exclude(key__in=["#relative_role","node.master","node.data","http.enabled"]) for item in con: role_name = role_conf_item.objects.filter(configure_item=item) role_array = [[x.id,x.role_id.name] for x in role_name] value = item.value.replace("\n","</br>").replace(" "," ") result["result"].append({"key":item.key,"value":value,"id":item.id,"role":copy.deepcopy(role_array)}) except: traceback.print_exc() logger.error("_configure_content " + traceback.format_exc()) result["status"] = "fail" return HttpResponse(json.dumps(result))
def _add(request): ''' 创建template ''' name = request.POST.get("name","default").strip() desc = request.POST.get("desc",'').strip() icon = request.POST.get("img",None) type = request.POST.get("type",name) mock_flag = request.POST.get("mock_flag",False) try: st = services_template() st.name = name st.description = desc st.type = type st.icon = icon st.save() template_name = name try: web_request("master","pillar_module.pillar_operation",[template_name,"temp"],type,"add_template") except: logger.error("modify " + traceback.format_exc()) result = {"status": "ok"} except Exception,e: logger.error("_add " + traceback.format_exc()) result = {"status": "fail","result":str(e)}
def conf_file(request): ''' 展现conf_file 页面 ''' context = {} id = request.GET.get("template_id",None) try: try: tem = services_template.objects.get(id=int(id)) except: raise NameError,u"模板不存在" context["template"] = tem try: context["len"] = 0 conf = template_configure.objects.filter(services_template_id=tem) for item in conf: context["len"] = 1 except: traceback.print_exc() conf = None context["conf"] = conf except: logger.error("_configure " + traceback.format_exc()) #这里需要处理 return render_to_response('conf_file.html',context,RequestContext(request))
def _add_role(request): ''' 增加role ''' id = request.POST.get("id",None) name = request.POST.get("name",str(uuid.uuid4())).strip().replace(".","_") try: new_item = role() name = name.replace(".","_").strip() new_item.name = name new_item.service = services_template.objects.get(id=int(id)) result = {"status": "ok", "filename": name} new_item.save() type = new_item.service.type name = new_item.service.name try: web_request("master","pillar_module.pillar_operation",[name,new_item.name],type,"add_role") except: logger.error("modify " + traceback.format_exc()) except: logger.error("_add_role " + traceback.format_exc()) result = {"status": "fail", "filename": name} return HttpResponse(json.dumps(result))
def _update_cluster(self,cluster_file,template,name,version): result = {"result":"fail"} _name = "clustername: " + name try: cl = services.objects.get(name=name) except: logger.error("add_cluster " + traceback.format_exc()) result["reason"] = "cluster invalid" return result try: # es系统每台机器的 id是 grains["id"]来表示的,grains["id"] 就是target ma_list = machine.objects.filter(id__in=[x["machine_id"] for x in instance_machine.objects.filter(ser_id=cl.id).values("machine_id")] ).values("target") ma_list = ",".join([x["target"] for x in ma_list]) except: logger.error("add_cluster " + traceback.format_exc()) result["reason"] = "machine invalid" return result cluster_ser = "es-zenPingUnicastHosts: '[" + ma_list + "]'" es_version = "es-version: '" + version + "'" #现在采取粗暴的重写方式("w")来修改文件,应该温柔点,现在只是将 集群的 name和 集群的机器列表当成 cluster的信息. #默认所有集群的其他配置一致,这个需要再考虑 with salt.utils.fopen(cluster_file, 'w') as fp_: print >> fp_,_name print >> fp_,cluster_ser print >> fp_,es_version
def _configure_pair(request): ''' 用户增加某个配置文件的 单个配置项 ''' id = request.POST.get("id",None) name = request.POST.get("name",None) value = request.POST.get("value",None) _id = request.POST.get("_id",None) try: try: id = _id.split("_")[1] _tc = template_configure.objects.get(id=int(id)) except: raise IndexError,"configure id is wrong" co = configure() co.template_configure_id = _tc co.key = name co.value = value co.save() __refresh_conf([_tc]) except: logger.error("_add_role " + traceback.format_exc()) result = {"status": "ok"} return HttpResponse(json.dumps(result))
def _configure_modify(request): ''' 修改某个配置的 value ''' id = request.POST.get("id",None) context = request.POST.get("context",None) result = {"status": "ok"} conf_file = [] try: id = id.split("&") context = context.split("&sperate&") element = zip(id,context) for item in element: try: conf = configure.objects.get(id=int(item[0])) #得到配置项 _tc = conf.template_configure_id #得到包含配置项的配置文件 except: traceback.print_exc() continue conf.value = item[1].replace("\\n","\n") conf.save() conf_file.append(conf.template_configure_id) __refresh_conf(conf_file) except: result["status"] = "fail" logger.error("_configure_modify " + traceback.format_exc()) return HttpResponse(json.dumps(result))
def __rmdir(self,name): try: if os.path.isdir(name): from shutil import rmtree rmtree(name) except: logger.error("__rmdir " + traceback.format_exc())
def modify(request): ser_id = request.POST.get("ser_id",None) role_list = request.POST.get("role_list","").split("+") context = {"status":"ok","result":""} try: if ser_id == None or not ser_id.isdigit(): raise NameError,"ser invalid" #先删除所有ser_id关联的记录 instance_machine.objects.filter(ser_id=ser_id).delete() #往instance_machine中存ser_id role_id machine_id的映射关系,其中ser_id和machine_id是必须的 for item in role_list: id_list = item.split("_") role_id = id_list[0] if len(id_list) > 0 else None machine_id_list = id_list[1].split("&") if len(id_list) > 1 else [] for machine_id_item in machine_id_list: if not machine_id_item.isdigit(): continue try: ser_item = services.objects.get(id=ser_id) except: continue im = instance_machine() im.ser_id = int(ser_id) im.role_id = role_id im.machine_id = machine_id_item im.save() ''' 还要更新cluster信息,因为cluster的这个变量es-zenPingUnicastHosts 在有新机器加入时,需要修改,role_machine关联变化时es-zenPingUnicastHosts也要修改, 还未写修改变量的接口, 但是 可以 直接 重写cluster ''' type = ser_item.belong_template.type name = ser_item.belong_template.name try: web_request("master","pillar_module.pillar_operation",[name,ser_item.name],type,"add_cluster") except: logger.error("modify " + traceback.format_exc()) #更新所有target的pillar信息 web_request("master","pillar_module.pillar_operation",[],None,"_refresh") except Exception,e: context["status"] = "fail" context["result"] = e logger.error("modify " + traceback.format_exc()) traceback.print_exc()
def __mkdir(self,name): try: if not os.path.isdir(name): mask = os.umask(002) os.mkdir(name) os.umask(mask) except: logger.error("__mkdir " + traceback.format_exc())
def _configure_add_reg(request): ''' 录入文件中的所有配置 ''' _name = request.POST.get("name","").strip() _reg = request.POST.get("reg",None) _id = request.POST.get("id",None) _checked = request.POST.get("checked",None) _mock_flag = request.POST.get("mock_flag",None) try: try: id = _id.split("_")[1] _tc = template_configure.objects.get(id=int(id)) except: raise IndexError,"configure id is wrong" conf_ele = {} #获取关联这份配置文件的所有role with open(os.path.join(files_path, _name), 'r') as destination: _content = destination.read() m = re.findall(_reg,_content) for item in m: if not len(item) == 2: continue if len(re.findall(r'[<>]',item[0])): continue if _checked == "checked" and item[0].strip() in conf_ele: continue if _checked == "checked": conf_ele[item[0].strip()] = 0 co = configure() co.template_configure_id = _tc co.key = item[0].strip() co.value = item[1].rstrip() co.save() result = {"status": "ok"} __refresh_conf([_tc]) except: logger.error("_configure_add_reg " + traceback.format_exc()) result = {"status": "fail"} if _mock_flag: return else: return HttpResponse(json.dumps(result))
def _del(request): ''' 删除整个template ''' id = request.POST.get("id",None) try: item = services_template.objects.get(id=int(id)) result = {"status": "ok"} type = item.type name = item.name try: web_request("master","pillar_module.pillar_operation",[name,"temp"],type,"sub_template") except: logger.error("modify " + traceback.format_exc()) #还需要删除所有该cluster的 role,ser,instance,configure_file,configure sv = services.objects.filter(belong_template=item) for sv_item in sv: instance_machine.objects.filter(ser_id=sv_item.id).delete() indices.objects.filter(ser=sv_item).delete() sv.delete() rl = role.objects.filter(service=item) for rl_item in rl: rl_conf = role_conf_item.objects.filter(role_id=rl_item).delete() rl.delete() cf = template_configure.objects.filter(services_template_id=item) for cf_item in cf: conf = configure.objects.filter(template_configure_id=cf_item) conf.delete() cf.delete() item.delete() #更新所有target的pillar信息 web_request("master","pillar_module.pillar_operation",[],None,"_refresh") #还要删除source配置 soure_root = settings.SOURCE_ROOT template_path = soure_root + "/" + '/'.join([item.type,item.name]) from shutil import rmtree rmtree(template_path) except: logger.error("_del " + traceback.format_exc()) result = {"status": "fail"} return HttpResponse(json.dumps(result))
def cluster_com(request): context = {"status":"ok","data":""} com = request.POST.get("com",None) id = request.POST.get("id",None) new_name = request.POST.get("new_name",None) try: if com == "rename": try: cl = cluster.objects.get(id=int(id)) except: raise NameError,"cluster name is invalid" cl.name = new_name.strip() cl.save() else: #依次删除service,instance_machine,role,service __fetch_sql = ''' select instance_machine.id, instance_machine.ser_id, instance_machine.machine_id from services INNER JOIN instance_machine ON services.id = instance_machine.ser_id INNER JOIN machine ON machine.id = instance_machine.machine_id where machine.cluster = {0}; '''.format(id) cursor = connection.cursor() cursor.execute(__fetch_sql) raw = cursor.fetchall() in_ma = [] _ser = [] for item in raw: in_ma.append(item[0]) _ser.append(item[1]) services.objects.filter(id__in=list(set(_ser))).delete() for item in machine.objects.filter(cluster=int(id)): item.cluster = None item.save() instance_machine.objects.filter(id__in=list(set(in_ma))).delete() cluster.objects.filter(id=int(id)).delete() except Exception,e: logger.error("ser_command " + traceback.format_exc()) context = {"status":"fail","data":str(e)}
def highstate_commit(idsarry, job_id): try: web_request(",".join(idsarry), "saltutil.refresh_pillar", [], None, None) cmd = 'salt -L "' + ",".join(idsarry) + '" state.highstate ' print "=========cmd=========", cmd res = cmd time.sleep(20) result = web_request(",".join(idsarry), "state.highstate", [], None, None) res = res + "\n" + str(result) except: logger.error("highstate_commit " + traceback.format_exc()) return res
def logstash_load(name): ret = {"status":"ok"} try: st = services_template.objects.get(name=name) ret = {"status":"fail","result":"已存在同名模板,不允许再创建哦"} except: try: request = {"name":name,"desc":"logstash","img":"logstash.png","type":"logstash","mock_flag":True} a = mock_http(request) _add(a) except Exception,e: traceback.print_exc() ret["status"] = "fail" ret["result"] = e logger.error("fetch_role " + traceback.format_exc())
def highstate_commit(request): idarray = request.POST.get("id",None) context = {"status":"ok"} try: web_request("master","pillar_module.pillar_operation",[],None,"_refresh") idarray = idarray.split("&") id = str(uuid.uuid1()) celery_highstate.delay(idarray,id) context["job_id"] = id except: logger.error("highstate_commit " + traceback.format_exc()) context = {"status":"failed"} return HttpResponse(json.dumps(context))
def _role_configure(request): ''' 获取某个role对应的某个配置的 值 ''' configure_id = request.GET.get("configure_id",None) role_id = request.GET.get("role_id",None) result = {"status": "ok"} try: rc = role_conf_item.objects.get(id=role_id) result["result"] = rc.value except: traceback.print_exc() result["status"] = "fail" logger.error("_configure_modify " + traceback.format_exc()) return HttpResponse(json.dumps(result))
def highstate(request): try: all_machine = machine.objects.all() context = {} machine_list = [] for item in all_machine: machine_list.append([item.id,item.target]) context["machine"] = machine_list context["status"] = "ok" except: logger.error("highstate " + traceback.format_exc()) context = {"status":"failed"} return HttpResponse(json.dumps(context))
def fetch_role(request): id = request.POST.get("id",None) ret = {"status":"ok"} result = [] try: st = services_template.objects.get(id=id) all_role = role.objects.filter(service=st) result = [[x.id,x.name] for x in all_role] ret["result"] = result except Exception,e: traceback.print_exc() ret["status"] = "fail" ret["result"] = e logger.error("fetch_role " + traceback.format_exc())
def _add_file(request): ''' 增加template时上传图标 ''' try: f = request.FILES.get("upload_file",None) name = str(uuid.uuid4()) with open(os.path.join(files_path, name), 'wb') as destination: for chunk in f.chunks(): destination.write(chunk) result = {"status": "ok", "filename": name} except: logger.error("_add_file " + traceback.format_exc()) result = {"status": "fail", "filename": name} return HttpResponse(json.dumps(result))
def _get_role_conf(request): ''' 获取role list中的每个role 对应的 所有配置文件 ''' _id_list = request.POST.get("id","").split("&") result = [] for item in _id_list: try: ret = {} _role_item = role.objects.get(id=item) ret["name"] = _role_item.name ret["id"] = _role_item.id ret["conf"] = [] _template = _role_item.service _template_name = _template.name _rc = role_configure.objects.filter(role_id=_role_item.id) _conf = template_configure.objects.filter(services_template_id=_template) _conf_ele = {} for _item in _rc: _conf_ele[_item.conf] = 0 for _item in _conf: tag = 0 if _item.id in _conf_ele: tag = 1 ret["conf"].append([tag,_item.id,_item.name]) result.append(ret) except: traceback.print_exc() logger.error("_configure_add_reg " + traceback.format_exc()) continue ret = {"status":"ok","result":result,"tn":_template_name} return HttpResponse(json.dumps(ret))
def __add_ele(self,template,_type,name): try: name = str(name) template_path = os.path.join(self._root_path,template) self.__mkdir(template_path) ele_path = os.path.join(template_path,_type) self.__mkdir(ele_path) os.system("rm -rf " + os.path.join(ele_path,name + ".sls")) mask = os.umask(002) os.system("touch " + os.path.join(ele_path,name + ".sls")) os.umask(mask) return os.path.join(ele_path,name + ".sls") except: logger.error("__add_ele " + traceback.format_exc()) return None
def role_conf(self,role_conf,template): ret = [] try: st = services_template.objects.get(name=template) all_role = role.objects.filter(name__in=role_conf,service=st) all_conf_id = role_conf_item.objects.filter(role_id__in=all_role) all_conf_id = [x.configure_item.template_configure_id.id for x in all_conf_id] result = template_configure.objects.filter(id__in=all_conf_id).values("name","deploy_path") ret = [] for item in result: if item["deploy_path"] == None or item["deploy_path"].strip() == "": item.pop("deploy_path") ret.append(item) except: logger.error("role_conf " + traceback.format_exc()) return ret
def update_elasticsearch_version_pillar(new_version, service_tmp, service_name): # elasticsearch_sls=r"/srv/pillar/elasticsearch/template/elasticsearch.sls" # elasticsearch_sls_bak=r"/srv/pillar/elasticsearch/template/elasticsearch.sls."+datetime.datetime.now().strftime('%y-%m-%d-%H:%M:%S'); # open(elasticsearch_sls_bak, "wb").write(open(elasticsearch_sls, "rb").read()) # f = open (elasticsearch_sls_bak, "r+") # fr = open(elasticsearch_sls, 'w') # fr.write(re.sub(r'es-version: "\d+\.\d+\.\d+"',r'es-version: "'+new_version+'"', f.read())) # f.close() # fr.close() try: result = web_request( "master", "pillar_module.pillar_operation", [service_tmp, service_name, new_version], "elasticsearch", "update_cluster_version", ) except: logger.error("modify " + traceback.format_exc())
def read_elasticsearch_version_pillar(service_tmp, service_name): # elasticsearch_sls=r"/srv/pillar/elasticsearch/template/elasticsearch.sls" # fr = open(elasticsearch_sls, 'r') # for line in fr.readlines(): # for i in re.findall(r'es-version: "(\d+\.\d+\.\d+)"',line): # return i # fr.close() try: result = web_request( "master", "pillar_module.pillar_operation", [service_tmp, service_name], "elasticsearch", "read_cluster_version", ) except: logger.error("modify " + traceback.format_exc()) result = "" return " ".join(result[0].values())
def machine_operations(id, action): print id, action context = {} ret = "" try: if id == None: raise NameError, "请选择instance" _id_list = id.split("_") _item = instance_machine.objects.filter(machine_id__in=_id_list) print _item _operation = {} es__operation = set() for _ele in _item: try: _target = machine.objects.get(id=int(_ele.machine_id)).target except: logger.error("_target fail " + traceback.format_exc()) context["result"].append([_ele.machine_id, u"instance记录不存在"]) continue try: _role = role.objects.get(id=int(_ele.role_id)) _template = _role.service.type except: logger.error("_role fail " + traceback.format_exc()) context["result"].append([_ele.machine_id, u"无法找到对应模板"]) continue if _template == "elasticsearch": es__operation.add(_target) elif _template not in _operation: _operation[_template] = set() if not _template == "elasticsearch" and not _target in _operation[_template]: _operation[_template].add(_target) if not es__operation == set(): result = web_request(",".join(list(es__operation)), "elasticsearch." + action, [], None, None) ret = ret + str(result) for _tem, _targ in _operation.items(): result = web_request(",".join(list(_targ)), _tem + "." + action, [], None, None) ret = ret + str(result) context["result"] = ret return ret except: logger.error("hosts_operations " + traceback.format_exc()) context = {"status": "failed"}
def _add_role(self,role_file,template,name): try: st = services_template.objects.get(name=template) role_item = role.objects.get(name=name,service=st) all_object = role_conf_item.objects.filter(role_id=role_item) all_configure = [] for x in all_object: yaml_regular = [] value_array = x.value.split("\n") for index,item in enumerate(value_array): #第一行和 最后一行必须是空2格 if index == 0 or index == len(value_array) - 1: item = item.lstrip() yaml_regular.append(" " + item) all_configure.append(str(x.configure_item.id) + ": |\n" + "\n".join(yaml_regular) ) except: logger.error("_add_role " + traceback.format_exc()) return #现在采取粗暴的重写方式("w")来修改文件,应该温柔点 with salt.utils.fopen(role_file, 'w') as fp_: print >> fp_,"\n".join(all_configure)
fields={"raw": Keyword()}, ), "country": Nested(properties={ "name": Text( analyzer=html_strip, fields={ "raw": Keyword(), }, ) }), }) class Index: name = ADDRESS_ADDRESS_DOCUMENT_NAME settings = { "number_of_shards": 1, "number_of_replicas": 1, "blocks": { "read_only_allow_delete": None }, } try: # Create the mappings in Elasticsearch Address.init() except Exception as err: logger.error(err)