def upgradeMaster(islandID): clusterMgr = mc.getClusterManager() clusters = clusterMgr.enumClusters(islandID) for cluster in clusters: locs = clusterMgr.enumClusterLocations(cluster.getID()) hosts = [] for loc in locs: masterURL = URL(loc.getClusterLocationProperty(SolrClusterAdapter.SOLR_MASTER_HOST_URL_PROP)) masterHost = masterURL.getHost() hosts.append(masterHost) service = Service("ems-solr") service.setThreadCount(1) service.invoke("stop", hosts) failures = runJob(["/usr/local/bin/ender", "remote-action", "upgrade-support", "save-index"], hosts) if not failures.isEmpty(): print "WARN: Failed to backup the index on all hosts. Check logs.", failures failures = runJob(["/usr/local/bin/ender", "remote-action", "upgrade-support", "upgrade-index"], hosts) if not failures.isEmpty(): print "Failed to start the backup job running on all hosts. Check logs.", failures return 1 failures = runJob( ["/usr/local/bin/ender", "remote-action", "upgrade-support", "upgrade-status"], hosts, sleepTime=120, maxWaits=43200, interimResult=3, ) if not failures.isEmpty(): print "ERROR: upgrade job failed. Upgrade aborted", failures return 1 failures = runJob(["/usr/local/bin/ender", "remote-action", "upgrade-support", "remove-solr-service"], hosts) if not failures.isEmpty(): print "ERROR: unable to remove solr service from all hosts", failures return 1 print "Please check for any errors in the output before proceeding" print "If it appears the Solr index has been upgraded successfully. Please proceed to install the new version of Solr and then run the validate-master step" return 0
def fullStatus(islandID): islandMgr = mc.getIslandManager() island = islandMgr.getIsland(int(islandID)) print "INFO: island status for island", islandID print "INFO: search URL", islandID, island.getSearchURL() print "INFO: search Parmeters", islandID, island.getSearchParms() print "INFO: cluster and shard status for island", islandID clusterMgr = mc.getClusterManager() clusters = clusterMgr.enumClusters(islandID) masters = [] slaves = [] masterURLs = [] slaveURLs = [] for cluster in clusters: print "INFO: cluster", cluster.getID(), "F:", cluster.isFeedEnabled(), "U:", cluster.isUpdateEnabled(), "P:", cluster.isPurgeEnabled() locs = clusterMgr.enumClusterLocations(cluster.getID()) for loc in locs: print "INFO: location", loc.getId(), "F:", loc.isFeedEnabled(), "U:", loc.isUpdateEnabled(), "P:", loc.isPurgeEnabled(), "M:", loc.isManagementEnabled() slaveURL = URL(loc.getClusterLocationProperty(SolrClusterAdapter.SOLR_SLAVE_HOST_URL_PROP)) slaveHost = slaveURL.getHost() slaves.append(slaveHost) masterURL = URL(loc.getClusterLocationProperty(SolrClusterAdapter.SOLR_MASTER_HOST_URL_PROP)) masterHost = masterURL.getHost() masters.append(masterHost) masterURL = masterURL.toString() + "/admin/system" masterURLs.append(masterURL) slaveURL = slaveURL.toString() + "/admin/system" slaveURLs.append(slaveURL) print "INFO: Service Status for island", islandID service = Service("ems-solr") service.setThreadCount(1) responses = service.invoke("status", masters) for key in responses.keySet(): sr = responses.get(key) print "INFO: Master", key, sr.getStdout() responses = service.invoke("status", slaves) for key in responses.keySet(): sr = responses.get(key) print "INFO: Slave", key, sr.getStdout() print "INFO: Version Info for island", islandID for masterURL in masterURLs: masterF = None try: masterF = urllib.urlopen(masterURL) except: print "WARN: Solr not running on", masterURL continue try: masterData = masterF.read() look = '<str name="solr-spec-version">' idx = masterData.find(look) idx = idx + len(look) end = masterData.find("</str>", idx) print "INFO:", masterURL, masterData[idx:end] finally: masterF.close() for slaveURL in slaveURLs: slaveF = None try: slaveF = urllib.urlopen(slaveURL) except: print "WARN: Solr not running on", slaveURL continue try: slaveData = slaveF.read() look = '<str name="solr-spec-version">' idx = slaveData.find(look) idx = idx + len(look) end = slaveData.find("</str>", idx) print "INFO:", slaveURL, slaveData[idx:end] finally: slaveF.close() print "INFO: Upgrade activities" ulog = "/tmp/.solr_upgrade_log_" + str(islandID) if os.path.exists(ulog): for line in open(ulog): print "INFO:", line.strip() else: print "INFO: No activity log found:", ulog
def upgradeSlave(islandID): clusterMgr = mc.getClusterManager() clusters = clusterMgr.enumClusters(islandID) hosts = [] for cluster in clusters: locs = clusterMgr.enumClusterLocations(cluster.getID()) for loc in locs: slaveURL = URL(loc.getClusterLocationProperty(SolrClusterAdapter.SOLR_SLAVE_HOST_URL_PROP)) slaveHost = slaveURL.getHost() masterURL = URL(loc.getClusterLocationProperty(SolrClusterAdapter.SOLR_MASTER_HOST_URL_PROP)) masterURL = masterURL.toString() + "/m1Monitor?command=register" slaveURL = slaveURL.toString() slaveURL = slaveURL + "/m1Monitor?command=shutdown" try: slaveF = urllib.urlopen(slaveURL) except: print "WARN: slave down, expected up", slaveURL logOutput(slaveF) slaveF.close() try: masterF = urllib.urlopen(masterURL) except: print "ERROR: master down, must be up", masterURL return 1 logOutput(masterF) masterF.close() hosts.append(slaveHost) islandMgr = mc.getIslandManager() retries = 5 done = False while not done and retries > 0: done = True island = islandMgr.getIsland(islandID) parms = island.getSearchParms() for cluster in clusters: locs = clusterMgr.enumClusterLocations(cluster.getID()) for loc in locs: slaveURL = URL(loc.getClusterLocationProperty(SolrClusterAdapter.SOLR_SLAVE_HOST_URL_PROP)).toString() if slaveURL in parms: print slaveURL, "still in search params, waiting for update" done = False retries = retries - 1 if not done and retries == 0: print "Unable to start slave upgrade because search URL does not show masters handling all search requests" return 1 print "Stopping Solr on slaves" service = Service("ems-solr") service.setThreadCount(1) service.invoke("stop", hosts) print "Removing Solr service from slaves to prepare for upgrade" failures = runJob(["/usr/local/bin/ender", "remote-action", "upgrade-support", "remove-solr-service"], hosts) if failures.isEmpty(): print "INFO: Search is now successfully running on masters, please use Extropy to upgrade slaves" else: print "WARN: check logs for errors before upgrading slaves. Host(s) reporting falilure:", failures return 0