def dataReplaceAll(request): ''' :param request: :return: ''' user = request.user pattern_coarse = re.compile(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})') pattern = re.compile( r'(\b(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b)' ) arg_tardata = 'dzh_sls.mobileserver.data_compress' arg_gettar = '/home/MobileServer/data.tar.gz' arg_dist = 'dzh_sls.mobileserver.data_all' var_cache_path = '/var/cache/salt/master/minions/' source_path = '/srv/salt/dzh_store/mobileserver/tar/data.tar.gz' dc_list = [] data_centers = {} ts_list_new = [] stkindex_dict = {} ret = {} valcon = {} hostsft = {} jid = [] judgerrors = [] errors = [] #hostsum = 0 hostrsum = 0 result_dc = DataCenter.objects.all() for dc in result_dc: dc_list.append(dc.dcen) data_centers[dc.dcen] = dc.dccn dc_list.sort() result_stkindex = StockIndex.objects.all() for row_index in result_stkindex: stkindex_dict[row_index.stkindex] = row_index.exchange if request.method == 'POST': form = RedataAllForms(request.POST) if form.is_valid(): form_data_dict = form.cleaned_data ds = form_data_dict['data_source'] dp = form_data_dict['data_path'] ts = form_data_dict['target_server'] try: ds_hostname = HostList.objects.get(ip=ds) ds_hn = ds_hostname.hostname except: judgerrors.append(u'数据源服务器: %s 不在主机列表中!' % ds) if ts: ts_list = ts.split(',') for ts_i in ts_list: if '*' not in ts_i.split(): match_coarse = pattern_coarse.search(ts_i) if match_coarse: match = pattern.search(ts_i) if match: try: result_ts = HostList.objects.get( ip=match.group()) ts_list_new.append(result_ts.hostname) except DoesNotExist: judgerrors.append(u'目标主机 %s 不在主机列表中!' % ts_i) else: judgerrors.append(u'目标主机地址 %s 不合法!' % ts_i) else: try: result_ts = HostList.objects.get(hostname=ts_i) ts_list_new.append(ts_i) except DoesNotExist: judgerrors.append(u'目标主机 %s 不在主机列表中!' % ts_i) else: ts_list_new.append(ts_i) if not request.POST.get('datacenter', '') and not ts_list_new: judgerrors.append(u'需要指定目标机房或目标机房,才能允许后续操作!') else: get_dc_list = request.POST.getlist('datacenter') if not judgerrors: if ts: tgt_set = manageTgt(ts) else: tgt_set = set([]) if get_dc_list: if '*' in get_dc_list: result_host_set = set(['*']) else: result_host_set = hostSet(get_dc_list) else: result_host_set = set([]) tgt_list = list(tgt_set.union(result_host_set)) tgt_judge, tgtminis = judgeTarget(tgt_list) if not tgt_judge: judgerrors.append( u'目标主机只能形如zhaogb-201、zhaogb-*、* 这三种(可多个用英文逗号隔开)!') else: sapi = SaltAPI(url=settings.SALT_API['url'], username=settings.SALT_API['user'], password=settings.SALT_API['password']) tar_data_fun = 'state.sls' tar_data = sapi.masterToMinion(ds_hn, tar_data_fun, arg_tardata) if tar_data: tk = tar_data['return'][0][ds_hn].keys() result_tar_data = tar_data['return'][0][ds_hn][ tk[0]]['result'] if result_tar_data: get_tar_data_fun = 'cp.push' get_tar_data = sapi.masterToMinion( ds_hn, get_tar_data_fun, arg_gettar) if get_tar_data: get_tar_status = get_tar_data['return'][0][ ds_hn] if get_tar_status: data_path = os.path.join( var_cache_path, ds_hn, 'files', arg_gettar[1:]) if os.path.exists(data_path): os.rename(data_path, source_path) if os.path.exists(source_path): if '*' in tgt_list: hostsum = len(result_host_set) sumset = set(result_host_set) obj = sapi.async_deploy_all( arg_dist) else: hostsum = len(result_host_set) sumset = set(result_host_set) tgt_list_to_str = ','.join( result_host_set) obj = sapi.async_deploy( tgt_list_to_str, arg_dist) jid.append(obj) if get_dc_list: operate_tgt = get_dc_list[0] else: operate_tgt = 'unknown' op_time = time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time())) op_user = arg_dist op_tgt = '%s...' % operate_tgt p1 = OperateRecord.objects.create( nowtime=op_time, username=user, user_operate=op_user, simple_tgt=op_tgt, jid=jid[0]) db = db_operate() for i in jid: time.sleep(30) sql = 'select id,`return` from salt_returns where jid=%s' result = db.select_table( settings.RETURNS_MYSQL, sql, str(i)) hostrsum = len(result) returnset = set(result.keys()) ret, hostfa, hosttr = outFormat( result) diffset = sumset.difference( returnset) hostunre = len(diffset) hostunrestr = ','.join( list(diffset)) hostsft['sum'] = hostsum hostsft['rsum'] = hostrsum hostsft['unre'] = hostunre hostsft['unrestr'] = hostunrestr hostsft['fa'] = hostfa hostsft['tr'] = hosttr os.remove(source_path) saveRecord = ReturnRecord.objects.create( jid=jid[0], tgt_total=hostsum, tgt_ret=hostrsum, tgt_succ=hosttr, tgt_fail=hostfa, tgt_unret=hostunre, tgt_unret_list=hostunrestr) else: judgerrors.append( u'Master cache目录下无data.tar.gz数据包!') else: judgerrors.append( u'拉取数据失败(拉取数据操作返回结果为False)!') else: judgerrors.append( u'拉取数据命令执行失败(Salt-api返回结果为空)!') else: judgerrors.append(u'打包数据失败(打包数据操作返回结果为False)!') else: judgerrors.append(u'打包数据命令执行失败(Salt-api返回结果为空)!') else: form = RedataAllForms( initial={'data_path': '/home/MobileServer/DATA/'}) if judgerrors: for error in judgerrors: errors.append(error.encode('utf8')) return render_to_response('data_replace_all.html', { 'form': form, 'dc_list': dc_list, 'data_centers': data_centers, 'ret': ret, 'errors': errors, 'hostsft': hostsft, 'stkindex_dict': stkindex_dict }, context_instance=RequestContext(request))
def dataCollection(request): ''' The Servers data collection; :param request: :return: ''' user = request.user fun = 'grains.item' arg_ip = 'ip4_interfaces' arg_id = 'id' arg_host = 'localhost' minion_info = {} errors = '' if request.method == 'GET': if not request.GET.get('datacenter',''): errors.append('xxx') if not errors: datacenter = request.GET.get('datacenter') tgt = '*_*_*_%s_*' % datacenter sapi = SaltAPI( url = settings.SALT_API['url'], username = settings.SALT_API['user'], password = settings.SALT_API['password'] ) minion_ip_return = sapi.masterToMinion(tgt,fun,arg_ip) minion_id_return = sapi.masterToMinion(tgt,fun,arg_id) minion_host_return = sapi.masterToMinion(tgt,fun,arg_host) minion_ip_dict = minion_ip_return['return'][0] minion_id_dict = minion_id_return['return'][0] minion_host_dict = minion_host_return['return'][0] minion_keys_list = minion_ip_dict.keys() minion_sum = len(minion_keys_list) minion_keys_list.sort() for mini_key in minion_keys_list: minion_info_list = [] minion_info_list.append(minion_ip_dict[mini_key]['ip4_interfaces']['eth0'][0]) minion_info_list.append(minion_host_dict[mini_key]['localhost']) minion_id = minion_id_dict[mini_key]['id'] minion_info_list.append(minion_id) minion_id_split = minion_id.split('_') # CNET_HQ_ZJ_WZ_61_164_153_56 minion_id_noen = minion_id_split[0] minion_id_catagoryen = minion_id_split[1] minion_id_paen = minion_id_split[2] minion_id_dcen = minion_id_split[3] minion_id_networkoperator = NetworkOperator.objects.get(noen=minion_id_noen) minion_id_nocn = minion_id_networkoperator.nocn minion_info_list.append(minion_id_nocn) minion_id_catagory = Catagory.objects.get(catagoryen=minion_id_catagoryen) minion_id_catagorycn = minion_id_catagory.catagorycn minion_info_list.append(minion_id_catagorycn) minion_id_provincearea = ProvinceArea.objects.get(paen=minion_id_paen) minion_id_pacn = minion_id_provincearea.pacn minion_info_list.append(minion_id_pacn) minion_id_datacenter = DataCenter.objects.get(dcen=minion_id_dcen) minion_id_dccn = minion_id_datacenter.dccn minion_info_list.append(minion_id_dccn) minion_info_tuple = tuple(minion_info_list) minion_info[mini_key] = minion_info_tuple for mininfo_key,mininfo_data in minion_info.iteritems(): h = HostList.objects.create( ip=mininfo_data[0], hostname=mininfo_data[1], minionid=mininfo_data[2], nocn=mininfo_data[3], catagorycn=mininfo_data[4], pacn=mininfo_data[5], dccn=mininfo_data[6] ) return render_to_response( 'common_data_collection.html', {'minion_sum' : minion_sum, 'minion_keys_list': minion_keys_list, 'minion_info_dict': minion_info, }, context_instance=RequestContext(request) )
def dataReplaceAll(request): ''' :param request: :return: ''' user = request.user pattern_coarse = re.compile(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})') pattern = re.compile( r'(\b(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b)') arg_tardata = 'dzh_sls.mobileserver.data_compress' arg_gettar = '/home/MobileServer/data.tar.gz' arg_dist = 'dzh_sls.mobileserver.data_all' var_cache_path = '/var/cache/salt/master/minions/' source_path = '/srv/salt/dzh_store/mobileserver/tar/data.tar.gz' dc_list = [] data_centers = {} ts_list_new = [] stkindex_dict = {} ret = {} valcon = {} hostsft = {} jid = [] judgerrors = [] errors = [] #hostsum = 0 hostrsum = 0 result_dc = DataCenter.objects.all() for dc in result_dc: dc_list.append(dc.dcen) data_centers[dc.dcen] = dc.dccn dc_list.sort() result_stkindex = StockIndex.objects.all() for row_index in result_stkindex: stkindex_dict[row_index.stkindex] = row_index.exchange if request.method == 'POST': form = RedataAllForms(request.POST) if form.is_valid(): form_data_dict = form.cleaned_data ds = form_data_dict['data_source'] dp = form_data_dict['data_path'] ts = form_data_dict['target_server'] try: ds_hostname = HostList.objects.get(ip=ds) ds_hn = ds_hostname.hostname except: judgerrors.append(u'数据源服务器: %s 不在主机列表中!' % ds) if ts: ts_list = ts.split(',') for ts_i in ts_list: if '*' not in ts_i.split(): match_coarse = pattern_coarse.search(ts_i) if match_coarse: match = pattern.search(ts_i) if match: try: result_ts = HostList.objects.get(ip=match.group()) ts_list_new.append(result_ts.hostname) except DoesNotExist: judgerrors.append(u'目标主机 %s 不在主机列表中!' % ts_i) else: judgerrors.append(u'目标主机地址 %s 不合法!' % ts_i ) else: try: result_ts = HostList.objects.get(hostname=ts_i) ts_list_new.append(ts_i) except DoesNotExist: judgerrors.append(u'目标主机 %s 不在主机列表中!' % ts_i) else: ts_list_new.append(ts_i) if not request.POST.get('datacenter', '') and not ts_list_new: judgerrors.append(u'需要指定目标机房或目标机房,才能允许后续操作!') else: get_dc_list = request.POST.getlist('datacenter') if not judgerrors: if ts: tgt_set = manageTgt(ts) else: tgt_set = set([]) if get_dc_list: if '*' in get_dc_list: result_host_set = set(['*']) else: result_host_set = hostSet(get_dc_list) else: result_host_set = set([]) tgt_list = list(tgt_set.union(result_host_set)) tgt_judge, tgtminis = judgeTarget(tgt_list) if not tgt_judge: judgerrors.append(u'目标主机只能形如zhaogb-201、zhaogb-*、* 这三种(可多个用英文逗号隔开)!') else: sapi = SaltAPI( url=settings.SALT_API['url'], username=settings.SALT_API['user'], password=settings.SALT_API['password']) tar_data_fun = 'state.sls' tar_data = sapi.masterToMinion(ds_hn,tar_data_fun,arg_tardata) if tar_data: tk = tar_data['return'][0][ds_hn].keys() result_tar_data = tar_data['return'][0][ds_hn][tk[0]]['result'] if result_tar_data: get_tar_data_fun = 'cp.push' get_tar_data = sapi.masterToMinion(ds_hn,get_tar_data_fun,arg_gettar) if get_tar_data: get_tar_status = get_tar_data['return'][0][ds_hn] if get_tar_status: data_path = os.path.join(var_cache_path, ds_hn, 'files', arg_gettar[1:]) if os.path.exists(data_path): os.rename(data_path, source_path) if os.path.exists(source_path): if '*' in tgt_list: hostsum = len(result_host_set) sumset = set(result_host_set) obj = sapi.async_deploy_all(arg_dist) else: hostsum = len(result_host_set) sumset = set(result_host_set) tgt_list_to_str = ','.join(result_host_set) obj = sapi.async_deploy(tgt_list_to_str,arg_dist) jid.append(obj) if get_dc_list: operate_tgt = get_dc_list[0] else: operate_tgt = 'unknown' op_time = time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(time.time())) op_user = arg_dist op_tgt = '%s...' % operate_tgt p1 = OperateRecord.objects.create( nowtime=op_time, username=user, user_operate=op_user, simple_tgt=op_tgt, jid=jid[0]) db = db_operate() for i in jid: time.sleep(30) sql = 'select id,`return` from salt_returns where jid=%s' result = db.select_table(settings.RETURNS_MYSQL,sql,str(i)) hostrsum = len(result) returnset = set(result.keys()) ret, hostfa, hosttr = outFormat(result) diffset = sumset.difference(returnset) hostunre = len(diffset) hostunrestr = ','.join(list(diffset)) hostsft['sum'] = hostsum hostsft['rsum'] = hostrsum hostsft['unre'] = hostunre hostsft['unrestr'] = hostunrestr hostsft['fa'] = hostfa hostsft['tr'] = hosttr os.remove(source_path) saveRecord = ReturnRecord.objects.create( jid=jid[0], tgt_total=hostsum, tgt_ret=hostrsum, tgt_succ=hosttr, tgt_fail=hostfa, tgt_unret=hostunre, tgt_unret_list=hostunrestr ) else: judgerrors.append(u'Master cache目录下无data.tar.gz数据包!') else: judgerrors.append(u'拉取数据失败(拉取数据操作返回结果为False)!') else: judgerrors.append(u'拉取数据命令执行失败(Salt-api返回结果为空)!') else: judgerrors.append(u'打包数据失败(打包数据操作返回结果为False)!') else: judgerrors.append(u'打包数据命令执行失败(Salt-api返回结果为空)!') else: form = RedataAllForms(initial={'data_path': '/home/MobileServer/DATA/'}) if judgerrors: for error in judgerrors: errors.append(error.encode('utf8')) return render_to_response( 'data_replace_all.html', {'form': form, 'dc_list': dc_list, 'data_centers': data_centers, 'ret': ret, 'errors': errors, 'hostsft': hostsft, 'stkindex_dict': stkindex_dict }, context_instance=RequestContext(request) )