def setupES(es_server_url='http://localhost:9200',deleteOld=1,doPrint=False,overrideTests=False, forceReplicas=-1): #ip_url=getURLwithIP(es_server_url) es = ElasticSearch(es_server_url,timeout=5) #get_template #es.send_request('GET', ['_template', name],query_params=query_params) #list_template #res = es.cluster_state(metric='metadata') templateList = es.send_request('GET', ['_template']) #templateList = res['metadata']['templates'] TEMPLATES = ["runappliance"] for template_name in TEMPLATES: if template_name not in templateList: printout(template_name+"template not present. It will be created. ",doPrint,False) create_template(es,template_name) else: norm_name = convert(templateList[template_name]) if deleteOld==0: printout(template_name+" already exists. Add 'replace' parameter to update if different, or forceupdate to always update.",doPrint,False) else: printout(template_name+" already exists.",doPrint,False) loaddoc = load_template(es,template_name) if forceReplicas>=0: loaddoc['settings']['index']['number_of_replicas']=forceReplicas if loaddoc!=None: mappingSame = norm_name['mappings']==loaddoc['mappings'] #settingSame = norm_name['settings']==loaddoc['settings'] settingsSame=True if int(norm_name['settings']['index.number_of_replicas'])!=int(loaddoc['settings']['index']['number_of_replicas']): settingsSame=False if int(norm_name['settings']['index.number_of_shards'])!=int(loaddoc['settings']['index']['number_of_shards']): settingsSame=False #currently analyzer settings are ot checked #if norm_name['settings']['index']['analysis']!=loaddoc['settings']['analysis']: # settingsSame=False if not (mappingSame and settingsSame) or deleteOld>1: #test is override if overrideTests==False: try: if norm_name['settings']['index,test']==True: printout("Template test setting found, skipping update...",doPrint,True) return except:pass #delete_template(es,template_name) printout("Updating "+template_name+" ES template",doPrint,True) create_template(es,template_name) else: printout('runappliance ES template is up to date',doPrint,True)
def setupES(es_server_url='http://localhost:9200',deleteOld=1,doPrint=False,overrideTests=False, forceReplicas=-1, forceShards=-1, create_index_name=None,subsystem="cdaq"): #ip_url=getURLwithIP(es_server_url) es = ElasticSearch(es_server_url,timeout=5) #list_template templateList = es.send_request('GET', ['_template']) TEMPLATES = ["runappliance_"+subsystem] loaddoc = None for template_name in TEMPLATES: template_label = template_name.split('_')[0] if template_name not in templateList: printout(template_name + " template not present. It will be created. ",doPrint,False) loaddoc = create_template(es,template_name,template_label,subsystem,forceReplicas,forceShards) else: loaddoc = create_template(es,None,template_label,subsystem,forceReplicas,forceShards,send=False) norm_name = convert(templateList[template_name]) if deleteOld==0: printout(template_name+" already exists. Add 'replace' parameter to update if different, or forceupdate to always update.",doPrint,False) else: printout(template_name+" already exists.",doPrint,False) if loaddoc!=None: mappingSame = norm_name['mappings']==loaddoc['mappings'] #settingSame = norm_name['settings']==loaddoc['settings'] settingsSame=True #convert to int before comparison if int(norm_name['settings']['index']['number_of_replicas'])!=int(loaddoc['settings']['index']['number_of_replicas']): settingsSame=False if int(norm_name['settings']['index']['number_of_shards'])!=int(loaddoc['settings']['index']['number_of_shards']): settingsSame=False #add more here if other settings need to be added if 'translog' not in norm_name['settings']['index'] or norm_name['settings']['index']['translog']!=loaddoc['settings']['index']['translog']: settingsSame=False #currently analyzer settings are not verified if not (mappingSame and settingsSame) or deleteOld>1: #test is override if overrideTests==False: try: if norm_name['settings']['test']==True: printout("Template test setting found, skipping update...",doPrint,True) break except:pass printout("Updating "+template_name+" ES template",doPrint,True) create_template(es,template_name,template_label,subsystem,forceReplicas,forceShards) else: printout('runappliance ES template is up to date',doPrint,True) #create index if create_index_name: if loaddoc: try: c_res = es.send_request('PUT', [create_index_name], body = loaddoc) if c_res!={'acknowledged':True}: printout("Result of index " + create_index_name + " create request: " + str(c_res),doPrint,True ) except ElasticHttpError, ex: if ex[1]['type']=='index_already_exists_exception': #this is for index pre-creator printout("Attempting to intialize already existing index "+create_index_name,doPrint,True) try: doc_resp = es.send_request('GET', ['_cat','indices',create_index_name],query_params={'h':'status'}) if doc_resp.strip('\n')=='close': printout("Index "+create_index_name+ " is already closed! Index will be reopened",doPrint,True) c_res = es.send_request('POST', [create_index_name,'_open']) except ElasticHttpError as ex: printout("setupES got ElasticHttpError getting index open/close state: "+str(ex),doPrint,True) except Exception as ex: printout("setupEs got Exception getting index open/closed state: "+str(ex),doPrint,True) except Exception as ex: #if type(ex)==RemoteTransportException: print "a",type(ex) printout("Index not created: "+str(ex),doPrint,True)