def upgradeMaster(islandID):
clusterMgr = mc.getClusterManager()
clusters = clusterMgr.enumClusters(islandID)
for cluster in clusters:
locs = clusterMgr.enumClusterLocations(cluster.getID())
hosts = []
for loc in locs:
masterURL = URL(loc.getClusterLocationProperty(SolrClusterAdapter.SOLR_MASTER_HOST_URL_PROP))
masterHost = masterURL.getHost()
hosts.append(masterHost)
service = Service("ems-solr")
service.setThreadCount(1)
service.invoke("stop", hosts)
failures = runJob(["/usr/local/bin/ender", "remote-action", "upgrade-support", "save-index"], hosts)
if not failures.isEmpty():
print "WARN: Failed to backup the index on all hosts. Check logs.", failures
failures = runJob(["/usr/local/bin/ender", "remote-action", "upgrade-support", "upgrade-index"], hosts)
if not failures.isEmpty():
print "Failed to start the backup job running on all hosts. Check logs.", failures
return 1
failures = runJob(
["/usr/local/bin/ender", "remote-action", "upgrade-support", "upgrade-status"],
hosts,
sleepTime=120,
maxWaits=43200,
interimResult=3,
)
if not failures.isEmpty():
print "ERROR: upgrade job failed. Upgrade aborted", failures
return 1
failures = runJob(["/usr/local/bin/ender", "remote-action", "upgrade-support", "remove-solr-service"], hosts)
if not failures.isEmpty():
print "ERROR: unable to remove solr service from all hosts", failures
return 1
print "Please check for any errors in the output before proceeding"
print "If it appears the Solr index has been upgraded successfully. Please proceed to install the new version of Solr and then run the validate-master step"
return 0
def fullStatus(islandID):
islandMgr = mc.getIslandManager()
island = islandMgr.getIsland(int(islandID))
print "INFO: island status for island", islandID
print "INFO: search URL", islandID, island.getSearchURL()
print "INFO: search Parmeters", islandID, island.getSearchParms()
print "INFO: cluster and shard status for island", islandID
clusterMgr = mc.getClusterManager()
clusters = clusterMgr.enumClusters(islandID)
masters = []
slaves = []
masterURLs = []
slaveURLs = []
for cluster in clusters:
print "INFO: cluster", cluster.getID(), "F:", cluster.isFeedEnabled(), "U:", cluster.isUpdateEnabled(), "P:", cluster.isPurgeEnabled()
locs = clusterMgr.enumClusterLocations(cluster.getID())
for loc in locs:
print "INFO: location", loc.getId(), "F:", loc.isFeedEnabled(), "U:", loc.isUpdateEnabled(), "P:", loc.isPurgeEnabled(), "M:", loc.isManagementEnabled()
slaveURL = URL(loc.getClusterLocationProperty(SolrClusterAdapter.SOLR_SLAVE_HOST_URL_PROP))
slaveHost = slaveURL.getHost()
slaves.append(slaveHost)
masterURL = URL(loc.getClusterLocationProperty(SolrClusterAdapter.SOLR_MASTER_HOST_URL_PROP))
masterHost = masterURL.getHost()
masters.append(masterHost)
masterURL = masterURL.toString() + "/admin/system"
masterURLs.append(masterURL)
slaveURL = slaveURL.toString() + "/admin/system"
slaveURLs.append(slaveURL)
print "INFO: Service Status for island", islandID
service = Service("ems-solr")
service.setThreadCount(1)
responses = service.invoke("status", masters)
for key in responses.keySet():
sr = responses.get(key)
print "INFO: Master", key, sr.getStdout()
responses = service.invoke("status", slaves)
for key in responses.keySet():
sr = responses.get(key)
print "INFO: Slave", key, sr.getStdout()
print "INFO: Version Info for island", islandID
for masterURL in masterURLs:
masterF = None
try:
masterF = urllib.urlopen(masterURL)
except:
print "WARN: Solr not running on", masterURL
continue
try:
masterData = masterF.read()
look = '<str name="solr-spec-version">'
idx = masterData.find(look)
idx = idx + len(look)
end = masterData.find("</str>", idx)
print "INFO:", masterURL, masterData[idx:end]
finally:
masterF.close()
for slaveURL in slaveURLs:
slaveF = None
try:
slaveF = urllib.urlopen(slaveURL)
except:
print "WARN: Solr not running on", slaveURL
continue
try:
slaveData = slaveF.read()
look = '<str name="solr-spec-version">'
idx = slaveData.find(look)
idx = idx + len(look)
end = slaveData.find("</str>", idx)
print "INFO:", slaveURL, slaveData[idx:end]
finally:
slaveF.close()
print "INFO: Upgrade activities"
ulog = "/tmp/.solr_upgrade_log_" + str(islandID)
if os.path.exists(ulog):
for line in open(ulog):
print "INFO:", line.strip()
else:
print "INFO: No activity log found:", ulog
def upgradeSlave(islandID):
clusterMgr = mc.getClusterManager()
clusters = clusterMgr.enumClusters(islandID)
hosts = []
for cluster in clusters:
locs = clusterMgr.enumClusterLocations(cluster.getID())
for loc in locs:
slaveURL = URL(loc.getClusterLocationProperty(SolrClusterAdapter.SOLR_SLAVE_HOST_URL_PROP))
slaveHost = slaveURL.getHost()
masterURL = URL(loc.getClusterLocationProperty(SolrClusterAdapter.SOLR_MASTER_HOST_URL_PROP))
masterURL = masterURL.toString() + "/m1Monitor?command=register"
slaveURL = slaveURL.toString()
slaveURL = slaveURL + "/m1Monitor?command=shutdown"
try:
slaveF = urllib.urlopen(slaveURL)
except:
print "WARN: slave down, expected up", slaveURL
logOutput(slaveF)
slaveF.close()
try:
masterF = urllib.urlopen(masterURL)
except:
print "ERROR: master down, must be up", masterURL
return 1
logOutput(masterF)
masterF.close()
hosts.append(slaveHost)
islandMgr = mc.getIslandManager()
retries = 5
done = False
while not done and retries > 0:
done = True
island = islandMgr.getIsland(islandID)
parms = island.getSearchParms()
for cluster in clusters:
locs = clusterMgr.enumClusterLocations(cluster.getID())
for loc in locs:
slaveURL = URL(loc.getClusterLocationProperty(SolrClusterAdapter.SOLR_SLAVE_HOST_URL_PROP)).toString()
if slaveURL in parms:
print slaveURL, "still in search params, waiting for update"
done = False
retries = retries - 1
if not done and retries == 0:
print "Unable to start slave upgrade because search URL does not show masters handling all search requests"
return 1
print "Stopping Solr on slaves"
service = Service("ems-solr")
service.setThreadCount(1)
service.invoke("stop", hosts)
print "Removing Solr service from slaves to prepare for upgrade"
failures = runJob(["/usr/local/bin/ender", "remote-action", "upgrade-support", "remove-solr-service"], hosts)
if failures.isEmpty():
print "INFO: Search is now successfully running on masters, please use Extropy to upgrade slaves"
else:
print "WARN: check logs for errors before upgrading slaves. Host(s) reporting falilure:", failures
return 0
