def module_update(request): """ update (mobile/class/prog..etc) module """ if request.method == 'POST': action = request.get_full_path().split('=')[1] if action == 'deploy': tgt = request.POST.get('tgt') arg = request.POST.getlist('module') if tgt: if arg: if len(arg) < 2: sapi = SaltAPI(url=settings.SALT_API['url'],username=settings.SALT_API['user'],password=settings.SALT_API['password']) jid = sapi.async_deploy(tgt,arg) db = db_operate() sql = 'select `return` from salt_returns where jid=%s' ret = db.select_table(settings.RETURNS_MYSQL,sql,str(i)) #通过jid获取执行结果 else: ret['亲,由于我比较菜,暂不支持同时部署多个模块!'] = '亲,由于我比较菜,暂不支持同时部署多个模块!' else: ret['请选择将要更新的模块!'] = '请选择将要更新的模块!' else: ret['亲,没有指定目标主机,请重新输入!'] = '亲,没有指定目标主机,请重新输入!' return render_to_response('salt_module_update.html', {'ret': ret},context_instance=RequestContext(request))
def host_list_manage(request, id=None): """ Manage Host List """ #user = request.user if id: host_list = get_object_or_404(HostList, pk=id) action = 'edit' page_name = '编辑主机' db = db_operate() sql = 'select ip from hostlist_hostlist where id = %s' % (id) ret = db.mysql_command(settings.DATABASES, sql) else: host_list = HostList() action = 'add' page_name = '新增主机' if request.method == 'GET': delete = request.GET.get('delete') id = request.GET.get('id') if delete: host_list = get_object_or_404(HostList, pk=id) host_list.delete() return HttpResponseRedirect(reverse('host_list')) if request.method == 'POST': form = HostsListForm(request.POST, instance=host_list) operate = request.POST.get('operate') if form.is_valid(): if action == 'add': form.save() return HttpResponseRedirect(reverse('host_list')) if operate: if operate == 'update': form.save() return HttpResponseRedirect(reverse('host_list')) else: pass else: form = HostsListForm(instance=host_list) return render_to_response('host_manage.html', { "form": form, "page_name": page_name, "action": action, }, context_instance=RequestContext(request))
def host_list_manage(request,id=None): """ Manage Host List """ #user = request.user if id: host_list = get_object_or_404(HostList, pk=id) action = 'edit' page_name = '编辑主机' db = db_operate() sql = 'select ip from hostlist_hostlist where id = %s' % (id) ret = db.mysql_command(settings.DATABASES,sql) else: host_list = HostList() action = 'add' page_name = '新增主机' if request.method == 'GET': delete = request.GET.get('delete') id = request.GET.get('id') if delete: host_list = get_object_or_404(HostList, pk=id) host_list.delete() return HttpResponseRedirect(reverse('host_list')) if request.method == 'POST': form = HostsListForm(request.POST,instance=host_list) operate = request.POST.get('operate') if form.is_valid(): if action == 'add': form.save() return HttpResponseRedirect(reverse('host_list')) if operate: if operate == 'update': form.save() return HttpResponseRedirect(reverse('host_list')) else: pass else: form = HostsListForm(instance=host_list) return render_to_response('host_manage.html', {"form": form, "page_name": page_name, "action": action, },context_instance=RequestContext(request))
def recordDetail(request): user = request.user hostsft = {} if 'jid' in request.GET: job_id = request.GET.get('jid') jid_record = OperateRecord.objects.get(jid=job_id) try: jidStatus = ReturnRecord.objects.get(jid=job_id) hostsft['sum'] = jidStatus.tgt_total hostsft['rsum'] = jidStatus.tgt_ret hostsft['unre'] = jidStatus.tgt_unret hostsft['unrestr'] = jidStatus.tgt_unret_list hostsft['fa'] = jidStatus.tgt_fail hostsft['tr'] = jidStatus.tgt_succ except: hostsft['sum'] = 'Null' hostsft['rsum'] = 'Null' hostsft['unre'] = 'Null' hostsft['unrestr'] = 'Null' hostsft['fa'] = 'Null' hostsft['tr'] = 'Null' db = db_operate() sql = 'select id,`return` from salt_returns where jid=%s' jid_result = db.select_table(settings.RETURNS_MYSQL,sql,str(job_id)) ret, hostfa, hosttr = outFormat(jid_result) else: jid_record = '' ret = {} return render_to_response( 'common_record_detail.html', {'jid_record': jid_record, 'hostsft': hostsft, 'ret': ret }, context_instance=RequestContext(request) )
def module_deploy(request): """ deploy (mobile/manager/info..) module out ret:{'host1':{'cont':'format result','status': colour },...} , hostsft:{'sum':'','rsum':'','unre':'','unrestr':'','fa':'','tr':''} """ ret = {} unret = {} valcon = {} hostsft = {} jid = [] #liv = [] objlist = [] hostfa = 0 hosttr = 0 hostsum = 0 hostrsum = 0 if request.method == 'POST': action = request.get_full_path().split('=')[1] if action == 'deploy': tgt = request.POST.get('tgt') arg = request.POST.getlist('module') tgtTest,tgtminis,tgtminisnox = tgtCheck() tgtlist = tgt.split(',') if tgt: if tgt in tgtTest: if arg: if len(arg) < 2: sapi = SaltAPI(url=settings.SALT_API['url'],username=settings.SALT_API['user'],password=settings.SALT_API['password']) for i in arg: obj = sapi.async_deploy(tgt,i) jid.append(obj) db = db_operate() for i in jid: time.sleep(30) sql = 'select id,`return` from salt_returns where jid=%s' result = db.select_table(settings.RETURNS_MYSQL,sql,str(i)) #result = {'zhaogb-201':{'file_|-info_so_1_|-/usr/lib64/libodbc.so.1_|-managed': {'comment': 'zhaogb-201', 'name': '/usr/lib64/libodbc.so.1', 'start_time': 'zhaogb-201', 'result': True, 'duration': 'zhaogb-201', '__run_num__': 2, 'changes': {}}},'zhaogb-202':{'file_|-info_so_1_|-/usr/lib64/libodbc.so.1_|-managed': {'comment': 'zhaogb-202', 'name': 'zhaogb-202', 'start_time': 'zhaogb-202', 'result': True, 'duration': 'zhaogb-202', '__run_num__': 2, 'changes': {'diff': 'New file', 'mode': '0644'}}}} hostrsum = len(result) returnset = set(result.keys()) for ka,va in result.iteritems(): # result {'zhaogb-201':{},'zhaobg-202':{}} # ka zhaogb-201,zhaogb-202 # va {'mo_watch':{'comment':'','result':'',...}} valcon = {} longstrva = '' falseStatus = 0 trueStatus = 0 liv = [] for kva in va.keys(): # kva mo_watch,... liva = kva.split('_|-') liv.append(va[kva]['result']) if va[kva]['changes']: changesStr = '' if liva[0] == 'file': if 'diff' in va[kva]['changes'].keys() and va[kva]['changes']['diff'] != '': changesStr += '\n\t对比 : \n\t\t{0}'.format(va[kva]['changes']['diff']) if 'mode' in va[kva]['changes'].keys() and va[kva]['changes']['mode'] != '': changesStr += '\n\t权限 : {0}'.format(va[kva]['changes']['mode']) #changesStr = '\n\t对比 : \n\t\t{0}\n\t权限 : {1}'.format(va[kva]['changes']['diff'],va[kva]['changes']['mode']) elif liva[0] == 'cmd': if 'pid' in va[kva]['changes'].keys() and va[kva]['changes']['pid'] != '': changesStr += '\n\tPID : {0}'.format(va[kva]['changes']['pid']) if 'retcode' in va[kva]['changes'].keys() and va[kva]['changes']['retcode'] != '': changesStr += '\n\t返回代码 : {0}'.format(va[kva]['changes']['retcode']) if 'stderr' in va[kva]['changes'].keys() and va[kva]['changes']['stderr'] != '': changesStr += '\n\t错误 : {0}'.format(va[kva]['changes']['stderr']) if 'stdout' in va[kva]['changes'].keys() and va[kva]['changes']['stdout'] != '': changesStr += '\n\t输出 : {0}'.format(va[kva]['changes']['stdout']) else: pass va[kva]['changes'] = changesStr else: pass strva = '结果 : {0}\n标签 : {1}\n操作 : {2}\n开始 : {3}\n耗时 : {4} ms\n变动 : {5}\n{6}\n'.format(va[kva]['result'],liva[1],va[kva]['comment'],va[kva]['start_time'],va[kva]['duration'],va[kva]['changes'],'------------------------------------------------------------') longstrva += strva if False in liv: colour = 'False' hostfa += 1 elif True in liv: colour = 'True' hosttr += 1 else: pass # error write to logging totalStatus = len(liv) for livStatus in liv: if livStatus == False: falseStatus += 1 elif livStatus == True: trueStatus += 1 else: pass # error write to logging longstrva += '失败 : {0}\n成功 : {1}\n总计 : {2}'.format(falseStatus, trueStatus, totalStatus) valcon['status'] = colour valcon['cont'] = longstrva unret[ka] = valcon if tgt == '*': hostsum = len(tgtminis) sumset = set(tgtminis) diffset = sumset - returnset hostunre = len(diffset) hostunrestr = ','.join(list(diffset)) elif '*' in tgt: tgtnox = re.sub(r'\*','',tgt) for linox in tgtminisnox: if tgtnox == linox: hostsum += 1 tgtmat = tgtnox + '[0-9]+' for mini in tgtminis: matchObj = re.search(tgtmat,mini, re.M|re.I) if matchObj: objlist.append(matchObj.group()) sumset = set(objlist) diffset = sumset - returnset hostunre = len(diffset) hostunrestr = ','.join(list(diffset)) else: hostsum = len(tgtlist) sumset = set(tgtlist) diffset = sumset - returnset hostunre = len(diffset) hostunrestr = ','.join(list(diffset)) hostsft['sum'] = hostsum hostsft['rsum'] = hostrsum hostsft['unre'] = hostunre hostsft['unrestr'] = hostunrestr hostsft['fa'] = hostfa hostsft['tr'] = hosttr else: valcon['status'] = 'False' valcon['cont'] = '虽然我很菜,但这个不是我的问题,Salt Stack不支持同时执行多个模板。不过我正在尝试通过异步处理来支持!' unret['亲,Salt Stack不支持同时执行多个模板'] = valcon else: valcon['status'] = 'False' valcon['cont'] = '亲,为何不选择一个模块试试呢!' unret['请选择将要部署的模块'] = valcon else: valcon['status'] = 'False' valcon['cont'] = '目标主机只能形如zhaogb-201、zhaogb-*、* 这三种!' unret['目标主机不合法'] = valcon else: valcon['status'] = 'False' valcon['cont'] = '需要指定目标主机,才能执行相应模板!' unret['亲,木有指定目标主机'] = valcon if unret: ret = unret #ret = {'zhaogb-201':{'cont':'zhaogb-201','status': 'True'},'zhaogb-202':{'cont':'zhaogb-202','status': 'Fasle'},'zhaogb-203':{'cont':'zhaogb-203','status': 'Fasle'},'zhaogb-205':{'cont':'zhaogb-205','status': 'True'}} else: valcon['status'] = 'False' valcon['cont'] = '骚年,你不相信我,就算点开看,也还是没有返回结果!' ret['没有返回任何结果'] = valcon return render_to_response('salt_module_deploy.html', {'ret': ret, 'hostsft': hostsft},context_instance=RequestContext(request))
def module_deploy(request): """ deploy (mobile/manager/info..) module out ret:{'host1':{'cont':'format result','status': colour },...} , hostsft:{'sum':'','rsum':'','unre':'','unrestr':'','fa':'','tr':''} """ ret = {} unret = {} valcon = {} hostsft = {} jid = [] #liv = [] objlist = [] hostfa = 0 hosttr = 0 hostsum = 0 hostrsum = 0 if request.method == 'POST': action = request.get_full_path().split('=')[1] if action == 'deploy': tgt = request.POST.get('tgt') arg = request.POST.getlist('module') tgtTest, tgtminis, tgtminisnox = tgtCheck() tgtlist = tgt.split(',') if tgt: if tgt in tgtTest: if arg: if len(arg) < 2: sapi = SaltAPI( url=settings.SALT_API['url'], username=settings.SALT_API['user'], password=settings.SALT_API['password']) for i in arg: obj = sapi.async_deploy(tgt, i) jid.append(obj) db = db_operate() for i in jid: time.sleep(30) sql = 'select id,`return` from salt_returns where jid=%s' result = db.select_table( settings.RETURNS_MYSQL, sql, str(i)) #result = {'zhaogb-201':{'file_|-info_so_1_|-/usr/lib64/libodbc.so.1_|-managed': {'comment': 'zhaogb-201', 'name': '/usr/lib64/libodbc.so.1', 'start_time': 'zhaogb-201', 'result': True, 'duration': 'zhaogb-201', '__run_num__': 2, 'changes': {}}},'zhaogb-202':{'file_|-info_so_1_|-/usr/lib64/libodbc.so.1_|-managed': {'comment': 'zhaogb-202', 'name': 'zhaogb-202', 'start_time': 'zhaogb-202', 'result': True, 'duration': 'zhaogb-202', '__run_num__': 2, 'changes': {'diff': 'New file', 'mode': '0644'}}}} hostrsum = len(result) returnset = set(result.keys()) for ka, va in result.iteritems(): # result {'zhaogb-201':{},'zhaobg-202':{}} # ka zhaogb-201,zhaogb-202 # va {'mo_watch':{'comment':'','result':'',...}} valcon = {} longstrva = '' falseStatus = 0 trueStatus = 0 liv = [] for kva in va.keys(): # kva mo_watch,... liva = kva.split('_|-') liv.append(va[kva]['result']) if va[kva]['changes']: changesStr = '' if liva[0] == 'file': if 'diff' in va[kva][ 'changes'].keys( ) and va[kva]['changes'][ 'diff'] != '': changesStr += '\n\t对比 : \n\t\t{0}'.format( va[kva]['changes']['diff']) if 'mode' in va[kva][ 'changes'].keys( ) and va[kva]['changes'][ 'mode'] != '': changesStr += '\n\t权限 : {0}'.format( va[kva]['changes']['mode']) #changesStr = '\n\t对比 : \n\t\t{0}\n\t权限 : {1}'.format(va[kva]['changes']['diff'],va[kva]['changes']['mode']) elif liva[0] == 'cmd': if 'pid' in va[kva]['changes'].keys( ) and va[kva]['changes'][ 'pid'] != '': changesStr += '\n\tPID : {0}'.format( va[kva]['changes']['pid']) if 'retcode' in va[kva][ 'changes'].keys( ) and va[kva]['changes'][ 'retcode'] != '': changesStr += '\n\t返回代码 : {0}'.format( va[kva]['changes'] ['retcode']) if 'stderr' in va[kva][ 'changes'].keys( ) and va[kva]['changes'][ 'stderr'] != '': changesStr += '\n\t错误 : {0}'.format( va[kva]['changes'] ['stderr']) if 'stdout' in va[kva][ 'changes'].keys( ) and va[kva]['changes'][ 'stdout'] != '': changesStr += '\n\t输出 : {0}'.format( va[kva]['changes'] ['stdout']) else: pass va[kva]['changes'] = changesStr else: pass strva = '结果 : {0}\n标签 : {1}\n操作 : {2}\n开始 : {3}\n耗时 : {4} ms\n变动 : {5}\n{6}\n'.format( va[kva]['result'], liva[1], va[kva]['comment'], va[kva]['start_time'], va[kva]['duration'], va[kva]['changes'], '------------------------------------------------------------' ) longstrva += strva if False in liv: colour = 'False' hostfa += 1 elif True in liv: colour = 'True' hosttr += 1 else: pass # error write to logging totalStatus = len(liv) for livStatus in liv: if livStatus == False: falseStatus += 1 elif livStatus == True: trueStatus += 1 else: pass # error write to logging longstrva += '失败 : {0}\n成功 : {1}\n总计 : {2}'.format( falseStatus, trueStatus, totalStatus) valcon['status'] = colour valcon['cont'] = longstrva unret[ka] = valcon if tgt == '*': hostsum = len(tgtminis) sumset = set(tgtminis) diffset = sumset - returnset hostunre = len(diffset) hostunrestr = ','.join(list(diffset)) elif '*' in tgt: tgtnox = re.sub(r'\*', '', tgt) for linox in tgtminisnox: if tgtnox == linox: hostsum += 1 tgtmat = tgtnox + '[0-9]+' for mini in tgtminis: matchObj = re.search( tgtmat, mini, re.M | re.I) if matchObj: objlist.append(matchObj.group()) sumset = set(objlist) diffset = sumset - returnset hostunre = len(diffset) hostunrestr = ','.join(list(diffset)) else: hostsum = len(tgtlist) sumset = set(tgtlist) diffset = sumset - returnset hostunre = len(diffset) hostunrestr = ','.join(list(diffset)) hostsft['sum'] = hostsum hostsft['rsum'] = hostrsum hostsft['unre'] = hostunre hostsft['unrestr'] = hostunrestr hostsft['fa'] = hostfa hostsft['tr'] = hosttr else: valcon['status'] = 'False' valcon[ 'cont'] = '虽然我很菜,但这个不是我的问题,Salt Stack不支持同时执行多个模板。不过我正在尝试通过异步处理来支持!' unret['亲,Salt Stack不支持同时执行多个模板'] = valcon else: valcon['status'] = 'False' valcon['cont'] = '亲,为何不选择一个模块试试呢!' unret['请选择将要部署的模块'] = valcon else: valcon['status'] = 'False' valcon['cont'] = '目标主机只能形如zhaogb-201、zhaogb-*、* 这三种!' unret['目标主机不合法'] = valcon else: valcon['status'] = 'False' valcon['cont'] = '需要指定目标主机,才能执行相应模板!' unret['亲,木有指定目标主机'] = valcon if unret: ret = unret #ret = {'zhaogb-201':{'cont':'zhaogb-201','status': 'True'},'zhaogb-202':{'cont':'zhaogb-202','status': 'Fasle'},'zhaogb-203':{'cont':'zhaogb-203','status': 'Fasle'},'zhaogb-205':{'cont':'zhaogb-205','status': 'True'}} else: valcon['status'] = 'False' valcon['cont'] = '骚年,你不相信我,就算点开看,也还是没有返回结果!' ret['没有返回任何结果'] = valcon return render_to_response('salt_module_deploy.html', { 'ret': ret, 'hostsft': hostsft }, context_instance=RequestContext(request))
def routine_maintenance(request): """ routine maintenance out {'host1':{'cont':'format result','status': colour },...} """ user = request.user result = '' ret = {} hostsft = {} dcen_list = [] data_centers = {} get_errors = [] errors = [] sls_list = [] sls_mod_dict = {} result_dc = DataCenter.objects.all() for dc in result_dc: dcen_list.append(dc.dcen) data_centers[dc.dcen] = dc.dccn dcen_list.sort() result_sls = CommonOperate.objects.all() for row_data in result_sls: sls_mod_dict[row_data.slsfile] = row_data.module sls_list.append(row_data.slsfile) if request.method == 'POST': check_tgt = request.POST.get('tgt', '') check_dc_list = request.POST.get('datacenter', '') module_detection = moduleDetection('state.sls', user) if module_detection: get_errors.append(module_detection) if not (check_tgt or check_dc_list): get_errors.append(u'需要指定目标主机或目标机房!') if not request.POST.get('module', ''): get_errors.append(u'请选择将要安装的模块!') if get_errors: for error in get_errors: errors.append(error.encode('utf-8')) else: tgt = request.POST.get('tgt') dc_list = request.POST.getlist('datacenter') arg = request.POST.get('module') module_lock = moduleLock('state.sls', user) if tgt: minion_id_from_tgt_set = targetToMinionID(tgt) else: minion_id_from_tgt_set = set([]) if dc_list: minion_id_from_dc_set = datacenterToMinionID(dc_list) else: minion_id_from_dc_set = set([]) all_minion_id_set = minion_id_from_tgt_set.union(minion_id_from_dc_set) if all_minion_id_set: sapi = SaltAPI( url=settings.SALT_API['url'], username=settings.SALT_API['user'], password=settings.SALT_API['password']) tgt_list = tgt.split(',') if ('*' in tgt_list) or ('*' in dc_list): jid = sapi.async_deploy_all(arg) else: tgt_list_to_str = ','.join(all_minion_id_set) jid = sapi.async_deploy(tgt_list_to_str,arg) if dc_list: operate_tgt = dc_list[0] elif tgt: operate_tgt = tgt_list[0] else: operate_tgt = 'unknown' op_time = time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(time.time())) op_user = arg op_tgt = '%s...' % operate_tgt p1 = OperateRecord.objects.create( nowtime=op_time, username=user, user_operate=op_user, simple_tgt=op_tgt, jid=jid) find_job = findJob(all_minion_id_set,jid) db = db_operate() time.sleep(30) sql = 'select id,`return` from salt_returns where jid=%s' result = db.select_table(settings.RETURNS_MYSQL,sql,str(jid)) hostsum = len(all_minion_id_set) sumset = all_minion_id_set hostrsum = len(result) returnset = set(result.keys()) ret, hostfa, hosttr = outFormat(result) diffset = sumset.difference(returnset) hostunre = len(diffset) hostunrestr = ','.join(list(diffset)) hostsft['sum'] = hostsum hostsft['rsum'] = hostrsum hostsft['unre'] = hostunre hostsft['unrestr'] = hostunrestr hostsft['fa'] = hostfa hostsft['tr'] = hosttr saveRecord = ReturnRecord.objects.create( jid=jid, tgt_total=hostsum, tgt_ret=hostrsum, tgt_succ=hosttr, tgt_fail=hostfa, tgt_unret=hostunre, tgt_unret_list=hostunrestr ) module_unlock = moduleUnlock('state.sls', user) else: tips = u'请输入正确的服务器IP或MinionID' errors.append(tips.encode('utf-8')) return render_to_response( 'salt_routine_maintenance.html', {'ret': ret, 'hostsft': hostsft, 'dcen_list': dcen_list, 'data_centers': data_centers, 'sls_list': sls_list, 'sls_mod_dict': sls_mod_dict, 'errors': errors }, context_instance=RequestContext(request) )
def dataReplaceHistory(request): ''' Repalce data of all market history; :param request: :return: dc_list ['dctest1','dctest2','dctest3','dctest4'] data_centers {'dctest1':'测试机房1','dctest2':'测试机房2',...} stock_exchanges ['SH','SZ','B$',....] ''' user = request.user data_path = '/srv/salt/dzh_store/mobileserver/DATA/' dc_list = [] data_centers = {} stock_exchanges = [] ret = {} valcon = {} hostsft = {} judgerrors = [] errors = [] #hostsum = 0 hostrsum = 0 result_dc = DataCenter.objects.all() for dc in result_dc: dc_list.append(dc.dcen) data_centers[dc.dcen] = dc.dccn dc_list.sort() result_stk = StockExchage.objects.all() for stkcode in result_stk: stock_exchanges.append(stkcode.stkexchen) if request.method == 'POST': if not request.POST.get('datacenter', ''): judgerrors.append(u'需要指定目标机房,才能允许后续操作!') if not request.POST.get('stockexchange', ''): judgerrors.append(u'亲,需要指定本次将要补数据的市场!') if not request.POST.get('statesls', ''): judgerrors.append(u'行情程序需要重启吗?总不能让我随机吧!') if judgerrors: for error in judgerrors: errors.append(error.encode('utf8')) elif not judgerrors: getdclist = request.POST.getlist('datacenter') getexchlist = request.POST.getlist('stockexchange') getstatesls = request.POST.get('statesls') if getdclist: result_host_set = datacenterToMinionID(getdclist) else: result_host_set = set([]) stkexch_set = set(os.listdir(data_path)) clear_dir = stkexch_set.difference(set(getexchlist)) for exchcode in clear_dir: day_path = os.path.join(data_path, exchcode, 'history/day') day_files = os.listdir(day_path) if day_files: for dyfile in day_files: dyfile_path = os.path.join(day_path, dyfile) os.remove(dyfile_path) log.info('Delete Other Market Success') sapi = SaltAPI(url=settings.SALT_API['url'], username=settings.SALT_API['user'], password=settings.SALT_API['password']) if '*' in getdclist: hostsum = len(result_host_set) sumset = set(result_host_set) jid = sapi.async_deploy_all(getstatesls) else: hostsum = len(result_host_set) sumset = set(result_host_set) tgt_list_to_str = ','.join(list(result_host_set)) jid = sapi.async_deploy(tgt_list_to_str, getstatesls) if getexchlist: operate_tgt = getexchlist[0] else: operate_tgt = 'unknown' op_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time())) op_user = getstatesls op_tgt = '%s...' % operate_tgt p1 = OperateRecord.objects.create(nowtime=op_time, username=user, user_operate=op_user, simple_tgt=op_tgt, jid=jid) log.info('Enter function findJob') find_job = findJob(result_host_set, jid) log.info('Quit function findJob') db = db_operate() time.sleep(30) sql = 'select id,`return` from salt_returns where jid=%s' result = db.select_table(settings.RETURNS_MYSQL, sql, str(jid)) hostrsum = len(result) returnset = set(result.keys()) ret, hostfa, hosttr = outFormat(result) diffset = sumset.difference(returnset) hostunre = len(diffset) hostunrestr = ','.join(list(diffset)) hostsft['sum'] = hostsum hostsft['rsum'] = hostrsum hostsft['unre'] = hostunre hostsft['unrestr'] = hostunrestr hostsft['fa'] = hostfa hostsft['tr'] = hosttr saveRecord = ReturnRecord.objects.create( jid=jid, tgt_total=hostsum, tgt_ret=hostrsum, tgt_succ=hosttr, tgt_fail=hostfa, tgt_unret=hostunre, tgt_unret_list=hostunrestr) return render_to_response('data_replace_history.html', { 'ret': ret, 'errors': errors, 'hostsft': hostsft, 'dc_list': dc_list, 'data_centers': data_centers, 'stock_exchanges': stock_exchanges }, context_instance=RequestContext(request))
def dataReplaceAll(request): ''' :param request: :return: ''' user = request.user pattern_coarse = re.compile(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})') pattern = re.compile( r'(\b(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b)' ) arg_tardata = 'dzh_sls.mobileserver.data_compress' arg_gettar = '/home/MobileServer/data.tar.gz' arg_dist = 'dzh_sls.mobileserver.data_all' var_cache_path = '/var/cache/salt/master/minions/' source_path = '/srv/salt/dzh_store/mobileserver/tar/data.tar.gz' dc_list = [] data_centers = {} ts_list_new = [] stkindex_dict = {} ret = {} valcon = {} hostsft = {} jid = [] judgerrors = [] errors = [] #hostsum = 0 hostrsum = 0 result_dc = DataCenter.objects.all() for dc in result_dc: dc_list.append(dc.dcen) data_centers[dc.dcen] = dc.dccn dc_list.sort() result_stkindex = StockIndex.objects.all() for row_index in result_stkindex: stkindex_dict[row_index.stkindex] = row_index.exchange if request.method == 'POST': form = RedataAllForms(request.POST) if form.is_valid(): form_data_dict = form.cleaned_data ds = form_data_dict['data_source'] dp = form_data_dict['data_path'] ts = form_data_dict['target_server'] try: ds_hostname = HostList.objects.get(ip=ds) ds_hn = ds_hostname.hostname except: judgerrors.append(u'数据源服务器: %s 不在主机列表中!' % ds) if ts: ts_list = ts.split(',') for ts_i in ts_list: if '*' not in ts_i.split(): match_coarse = pattern_coarse.search(ts_i) if match_coarse: match = pattern.search(ts_i) if match: try: result_ts = HostList.objects.get( ip=match.group()) ts_list_new.append(result_ts.hostname) except DoesNotExist: judgerrors.append(u'目标主机 %s 不在主机列表中!' % ts_i) else: judgerrors.append(u'目标主机地址 %s 不合法!' % ts_i) else: try: result_ts = HostList.objects.get(hostname=ts_i) ts_list_new.append(ts_i) except DoesNotExist: judgerrors.append(u'目标主机 %s 不在主机列表中!' % ts_i) else: ts_list_new.append(ts_i) if not request.POST.get('datacenter', '') and not ts_list_new: judgerrors.append(u'需要指定目标机房或目标机房,才能允许后续操作!') else: get_dc_list = request.POST.getlist('datacenter') if not judgerrors: if ts: tgt_set = manageTgt(ts) else: tgt_set = set([]) if get_dc_list: if '*' in get_dc_list: result_host_set = set(['*']) else: result_host_set = hostSet(get_dc_list) else: result_host_set = set([]) tgt_list = list(tgt_set.union(result_host_set)) tgt_judge, tgtminis = judgeTarget(tgt_list) if not tgt_judge: judgerrors.append( u'目标主机只能形如zhaogb-201、zhaogb-*、* 这三种(可多个用英文逗号隔开)!') else: sapi = SaltAPI(url=settings.SALT_API['url'], username=settings.SALT_API['user'], password=settings.SALT_API['password']) tar_data_fun = 'state.sls' tar_data = sapi.masterToMinion(ds_hn, tar_data_fun, arg_tardata) if tar_data: tk = tar_data['return'][0][ds_hn].keys() result_tar_data = tar_data['return'][0][ds_hn][ tk[0]]['result'] if result_tar_data: get_tar_data_fun = 'cp.push' get_tar_data = sapi.masterToMinion( ds_hn, get_tar_data_fun, arg_gettar) if get_tar_data: get_tar_status = get_tar_data['return'][0][ ds_hn] if get_tar_status: data_path = os.path.join( var_cache_path, ds_hn, 'files', arg_gettar[1:]) if os.path.exists(data_path): os.rename(data_path, source_path) if os.path.exists(source_path): if '*' in tgt_list: hostsum = len(result_host_set) sumset = set(result_host_set) obj = sapi.async_deploy_all( arg_dist) else: hostsum = len(result_host_set) sumset = set(result_host_set) tgt_list_to_str = ','.join( result_host_set) obj = sapi.async_deploy( tgt_list_to_str, arg_dist) jid.append(obj) if get_dc_list: operate_tgt = get_dc_list[0] else: operate_tgt = 'unknown' op_time = time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time())) op_user = arg_dist op_tgt = '%s...' % operate_tgt p1 = OperateRecord.objects.create( nowtime=op_time, username=user, user_operate=op_user, simple_tgt=op_tgt, jid=jid[0]) db = db_operate() for i in jid: time.sleep(30) sql = 'select id,`return` from salt_returns where jid=%s' result = db.select_table( settings.RETURNS_MYSQL, sql, str(i)) hostrsum = len(result) returnset = set(result.keys()) ret, hostfa, hosttr = outFormat( result) diffset = sumset.difference( returnset) hostunre = len(diffset) hostunrestr = ','.join( list(diffset)) hostsft['sum'] = hostsum hostsft['rsum'] = hostrsum hostsft['unre'] = hostunre hostsft['unrestr'] = hostunrestr hostsft['fa'] = hostfa hostsft['tr'] = hosttr os.remove(source_path) saveRecord = ReturnRecord.objects.create( jid=jid[0], tgt_total=hostsum, tgt_ret=hostrsum, tgt_succ=hosttr, tgt_fail=hostfa, tgt_unret=hostunre, tgt_unret_list=hostunrestr) else: judgerrors.append( u'Master cache目录下无data.tar.gz数据包!') else: judgerrors.append( u'拉取数据失败(拉取数据操作返回结果为False)!') else: judgerrors.append( u'拉取数据命令执行失败(Salt-api返回结果为空)!') else: judgerrors.append(u'打包数据失败(打包数据操作返回结果为False)!') else: judgerrors.append(u'打包数据命令执行失败(Salt-api返回结果为空)!') else: form = RedataAllForms( initial={'data_path': '/home/MobileServer/DATA/'}) if judgerrors: for error in judgerrors: errors.append(error.encode('utf8')) return render_to_response('data_replace_all.html', { 'form': form, 'dc_list': dc_list, 'data_centers': data_centers, 'ret': ret, 'errors': errors, 'hostsft': hostsft, 'stkindex_dict': stkindex_dict }, context_instance=RequestContext(request))
def dataReplaceHistory(request): ''' Repalce data of all market history; :param request: :return: dc_list ['dctest1','dctest2','dctest3','dctest4'] data_centers {'dctest1':'测试机房1','dctest2':'测试机房2',...} stock_exchanges ['SH','SZ','B$',....] ''' user = request.user data_path = '/srv/salt/dzh_store/mobileserver/DATA/' dc_list = [] data_centers = {} stock_exchanges = [] ret = {} valcon = {} hostsft = {} judgerrors = [] errors = [] #hostsum = 0 hostrsum = 0 result_dc = DataCenter.objects.all() for dc in result_dc: dc_list.append(dc.dcen) data_centers[dc.dcen] = dc.dccn dc_list.sort() result_stk = StockExchage.objects.all() for stkcode in result_stk: stock_exchanges.append(stkcode.stkexchen) if request.method == 'POST': if not request.POST.get('datacenter', ''): judgerrors.append(u'需要指定目标机房,才能允许后续操作!') if not request.POST.get('stockexchange', ''): judgerrors.append(u'亲,需要指定本次将要补数据的市场!') if not request.POST.get('statesls', ''): judgerrors.append(u'行情程序需要重启吗?总不能让我随机吧!') if judgerrors: for error in judgerrors: errors.append(error.encode('utf8')) elif not judgerrors: getdclist = request.POST.getlist('datacenter') getexchlist = request.POST.getlist('stockexchange') getstatesls = request.POST.get('statesls') if getdclist: result_host_set = datacenterToMinionID(getdclist) else: result_host_set = set([]) stkexch_set = set(os.listdir(data_path)) clear_dir = stkexch_set.difference(set(getexchlist)) for exchcode in clear_dir: day_path = os.path.join(data_path, exchcode, 'history/day') day_files = os.listdir(day_path) if day_files: for dyfile in day_files: dyfile_path = os.path.join(day_path,dyfile) os.remove(dyfile_path) log.info('Delete Other Market Success') sapi = SaltAPI( url=settings.SALT_API['url'], username=settings.SALT_API['user'], password=settings.SALT_API['password']) if '*' in getdclist: hostsum = len(result_host_set) sumset = set(result_host_set) jid = sapi.async_deploy_all(getstatesls) else: hostsum = len(result_host_set) sumset = set(result_host_set) tgt_list_to_str = ','.join(list(result_host_set)) jid = sapi.async_deploy(tgt_list_to_str,getstatesls) if getexchlist: operate_tgt = getexchlist[0] else: operate_tgt = 'unknown' op_time = time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(time.time())) op_user = getstatesls op_tgt = '%s...' % operate_tgt p1 = OperateRecord.objects.create( nowtime=op_time, username=user, user_operate=op_user, simple_tgt=op_tgt, jid=jid) log.info('Enter function findJob') find_job = findJob(result_host_set,jid) log.info('Quit function findJob') db = db_operate() time.sleep(30) sql = 'select id,`return` from salt_returns where jid=%s' result = db.select_table(settings.RETURNS_MYSQL,sql,str(jid)) hostrsum = len(result) returnset = set(result.keys()) ret, hostfa, hosttr = outFormat(result) diffset = sumset.difference(returnset) hostunre = len(diffset) hostunrestr = ','.join(list(diffset)) hostsft['sum'] = hostsum hostsft['rsum'] = hostrsum hostsft['unre'] = hostunre hostsft['unrestr'] = hostunrestr hostsft['fa'] = hostfa hostsft['tr'] = hosttr saveRecord = ReturnRecord.objects.create( jid=jid, tgt_total=hostsum, tgt_ret=hostrsum, tgt_succ=hosttr, tgt_fail=hostfa, tgt_unret=hostunre, tgt_unret_list=hostunrestr ) return render_to_response( 'data_replace_history.html', {'ret': ret, 'errors': errors, 'hostsft': hostsft, 'dc_list': dc_list, 'data_centers': data_centers, 'stock_exchanges': stock_exchanges }, context_instance=RequestContext(request) )
def dataReplaceAll(request): ''' :param request: :return: ''' user = request.user pattern_coarse = re.compile(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})') pattern = re.compile( r'(\b(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b)') arg_tardata = 'dzh_sls.mobileserver.data_compress' arg_gettar = '/home/MobileServer/data.tar.gz' arg_dist = 'dzh_sls.mobileserver.data_all' var_cache_path = '/var/cache/salt/master/minions/' source_path = '/srv/salt/dzh_store/mobileserver/tar/data.tar.gz' dc_list = [] data_centers = {} ts_list_new = [] stkindex_dict = {} ret = {} valcon = {} hostsft = {} jid = [] judgerrors = [] errors = [] #hostsum = 0 hostrsum = 0 result_dc = DataCenter.objects.all() for dc in result_dc: dc_list.append(dc.dcen) data_centers[dc.dcen] = dc.dccn dc_list.sort() result_stkindex = StockIndex.objects.all() for row_index in result_stkindex: stkindex_dict[row_index.stkindex] = row_index.exchange if request.method == 'POST': form = RedataAllForms(request.POST) if form.is_valid(): form_data_dict = form.cleaned_data ds = form_data_dict['data_source'] dp = form_data_dict['data_path'] ts = form_data_dict['target_server'] try: ds_hostname = HostList.objects.get(ip=ds) ds_hn = ds_hostname.hostname except: judgerrors.append(u'数据源服务器: %s 不在主机列表中!' % ds) if ts: ts_list = ts.split(',') for ts_i in ts_list: if '*' not in ts_i.split(): match_coarse = pattern_coarse.search(ts_i) if match_coarse: match = pattern.search(ts_i) if match: try: result_ts = HostList.objects.get(ip=match.group()) ts_list_new.append(result_ts.hostname) except DoesNotExist: judgerrors.append(u'目标主机 %s 不在主机列表中!' % ts_i) else: judgerrors.append(u'目标主机地址 %s 不合法!' % ts_i ) else: try: result_ts = HostList.objects.get(hostname=ts_i) ts_list_new.append(ts_i) except DoesNotExist: judgerrors.append(u'目标主机 %s 不在主机列表中!' % ts_i) else: ts_list_new.append(ts_i) if not request.POST.get('datacenter', '') and not ts_list_new: judgerrors.append(u'需要指定目标机房或目标机房,才能允许后续操作!') else: get_dc_list = request.POST.getlist('datacenter') if not judgerrors: if ts: tgt_set = manageTgt(ts) else: tgt_set = set([]) if get_dc_list: if '*' in get_dc_list: result_host_set = set(['*']) else: result_host_set = hostSet(get_dc_list) else: result_host_set = set([]) tgt_list = list(tgt_set.union(result_host_set)) tgt_judge, tgtminis = judgeTarget(tgt_list) if not tgt_judge: judgerrors.append(u'目标主机只能形如zhaogb-201、zhaogb-*、* 这三种(可多个用英文逗号隔开)!') else: sapi = SaltAPI( url=settings.SALT_API['url'], username=settings.SALT_API['user'], password=settings.SALT_API['password']) tar_data_fun = 'state.sls' tar_data = sapi.masterToMinion(ds_hn,tar_data_fun,arg_tardata) if tar_data: tk = tar_data['return'][0][ds_hn].keys() result_tar_data = tar_data['return'][0][ds_hn][tk[0]]['result'] if result_tar_data: get_tar_data_fun = 'cp.push' get_tar_data = sapi.masterToMinion(ds_hn,get_tar_data_fun,arg_gettar) if get_tar_data: get_tar_status = get_tar_data['return'][0][ds_hn] if get_tar_status: data_path = os.path.join(var_cache_path, ds_hn, 'files', arg_gettar[1:]) if os.path.exists(data_path): os.rename(data_path, source_path) if os.path.exists(source_path): if '*' in tgt_list: hostsum = len(result_host_set) sumset = set(result_host_set) obj = sapi.async_deploy_all(arg_dist) else: hostsum = len(result_host_set) sumset = set(result_host_set) tgt_list_to_str = ','.join(result_host_set) obj = sapi.async_deploy(tgt_list_to_str,arg_dist) jid.append(obj) if get_dc_list: operate_tgt = get_dc_list[0] else: operate_tgt = 'unknown' op_time = time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(time.time())) op_user = arg_dist op_tgt = '%s...' % operate_tgt p1 = OperateRecord.objects.create( nowtime=op_time, username=user, user_operate=op_user, simple_tgt=op_tgt, jid=jid[0]) db = db_operate() for i in jid: time.sleep(30) sql = 'select id,`return` from salt_returns where jid=%s' result = db.select_table(settings.RETURNS_MYSQL,sql,str(i)) hostrsum = len(result) returnset = set(result.keys()) ret, hostfa, hosttr = outFormat(result) diffset = sumset.difference(returnset) hostunre = len(diffset) hostunrestr = ','.join(list(diffset)) hostsft['sum'] = hostsum hostsft['rsum'] = hostrsum hostsft['unre'] = hostunre hostsft['unrestr'] = hostunrestr hostsft['fa'] = hostfa hostsft['tr'] = hosttr os.remove(source_path) saveRecord = ReturnRecord.objects.create( jid=jid[0], tgt_total=hostsum, tgt_ret=hostrsum, tgt_succ=hosttr, tgt_fail=hostfa, tgt_unret=hostunre, tgt_unret_list=hostunrestr ) else: judgerrors.append(u'Master cache目录下无data.tar.gz数据包!') else: judgerrors.append(u'拉取数据失败(拉取数据操作返回结果为False)!') else: judgerrors.append(u'拉取数据命令执行失败(Salt-api返回结果为空)!') else: judgerrors.append(u'打包数据失败(打包数据操作返回结果为False)!') else: judgerrors.append(u'打包数据命令执行失败(Salt-api返回结果为空)!') else: form = RedataAllForms(initial={'data_path': '/home/MobileServer/DATA/'}) if judgerrors: for error in judgerrors: errors.append(error.encode('utf8')) return render_to_response( 'data_replace_all.html', {'form': form, 'dc_list': dc_list, 'data_centers': data_centers, 'ret': ret, 'errors': errors, 'hostsft': hostsft, 'stkindex_dict': stkindex_dict }, context_instance=RequestContext(request) )