def doInit(additionalVariables):
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_balancer] Beginning doInit()")
doInit_common(additionalVariables)
additionalVariables.add(RuntimeContextVariable("hadoop_enabler_balancer_BALANCER_RECEIVED_REQUESTS", "",
RuntimeContextVariable.STRING_TYPE))
additionalVariables.add(RuntimeContextVariable("hadoop_enabler_balancer_BALANCER_NEW_REQUEST_FLAG", "False",
RuntimeContextVariable.STRING_TYPE))
additionalVariables.add(RuntimeContextVariable("hadoop_enabler_balancer_BALANCER_EXIT_FLAG", "False",
RuntimeContextVariable.STRING_TYPE))
"""
Create New Thread objects for processing requests and save for later
"""
logger = ContainerUtils.getLogger(proxy)
slstdout = StreamToLogger(logger,"STDOUT")
slstderr = StreamToLogger(logger,"STDERR")
lock = threading.Lock()
processReqThread = processRequestsThread(slstdout, slstderr, lock)
additionalVariables.add(RuntimeContextVariable("hadoop_enabler_balancer_BALANCER_PROCESS_REQUEST_THREAD", processReqThread,
RuntimeContextVariable.OBJECT_TYPE))
checkForReqThread = checkForRequestsThread(slstdout, slstderr, lock)
additionalVariables.add(RuntimeContextVariable("hadoop_enabler_balancer_BALANCER_CHECKFOR_REQUEST_THREAD", checkForReqThread,
RuntimeContextVariable.OBJECT_TYPE))
proxy.doInit(additionalVariables)
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_balancer] Exiting doInit()")
def createEnvironmentScript():
hadoop_bin_dir = proxy.getContainer().getRuntimeContext().getVariable('hadoop_enabler_HADOOP_BIN_DIR').getValue()
if ContainerUtils.isWindows():
environmentFilename = os.path.join(hadoop_bin_dir, "hadoop_enabler_ENVIRONMENT.bat")
else:
environmentFilename = os.path.join(hadoop_bin_dir, "hadoop_enabler_ENVIRONMENT.sh")
environmentFile = open(environmentFilename, 'w')
print>>environmentFile, "###################################################"
print>>environmentFile, "# Generated by Hodoop Enabler"
print>>environmentFile, "# " + str(datetime.datetime.now())
print>>environmentFile, "#"
print>>environmentFile, "# This file sets all the ENVIRONMENT type runtimecontext"
print>>environmentFile, "# variables defined by this enabler."
print>>environmentFile, "#"
print>>environmentFile, "###################################################"
runtimeContext = proxy.getContainer().getRuntimeContext()
for i in range (0, (runtimeContext.getVariableCount() - 1)):
variable = runtimeContext.getVariable(i)
ContainerUtils.getLogger(proxy).fine("[hadoop_enabler_common] " + str(variable.getName()) + " has type [" + str(variable.getTypeInt()) + "]")
if (variable.getTypeInt() == RuntimeContextVariable.ENVIRONMENT_TYPE):
print>>environmentFile, variable.getName() + "=" + str(variable.getValue())
environmentFile.close()
def getBalancerRequestsFromQueue():
hadoop_home_dir = proxy.getContainer().getRuntimeContext().getVariable('hadoop_enabler_HADOOP_HOME_DIR').getValue()
commandline = os.path.join(hadoop_home_dir, "bin", "hadoop") + " fs -ls " + balancerqueue_dir
#Older versions of hadoop (e.g 1.0.4 and 1.1.2) return 255 if file does not exist. Newer versions return 1
output = runCommand(commandline, stdout=PIPE, expectedReturnCodes=[0, 255, 1], suppressOutput=True)
requests = []
if (output[0] == 0):
reqsFileList = output[1].splitlines()
numOfRequests=0
for i in range(1, len(reqsFileList)):
if not reqsFileList[i].startswith("Found"):
""" Workaround - With 2.0.3-alpha a warning message for missing compression libraries is being printed to STDOUT during ls command """
if not reqsFileList[i].count("Unable to load native-hadoop library for your platform... using builtin-java classes where applicable") > 0:
ContainerUtils.getLogger(proxy).fine("[hadoop_enabler_balancer] Found request in queue [" + str(reqsFileList[i]) + "]")
lineTokens = reqsFileList[i].split()
if len(lineTokens) == 8:
filepath = lineTokens[7]
requests.append(filepath)
numOfRequests=numOfRequests+1
else:
ContainerUtils.getLogger(proxy).warning("[hadoop_enabler_balancer] Ignoring Line " + str(i + 1) + " of queue listing for " + balancerqueue_dir +
"Line has unexpected format. Full listing is:" +
str(reqsFileList) )
if numOfRequests > 0:
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_balancer] "+ str(numOfRequests) +" requests found in [" + str(balancerqueue_dir) + "].")
elif (output[0] == 255) or (output[0] == 1):
ContainerUtils.getLogger(proxy).fine("[hadoop_enabler_balancer] No requests found in [" + str(balancerqueue_dir) + "].")
else:
ContainerUtils.getLogger(proxy).warning("[hadoop_enabler_balancer] Unexpected return code " + str(output[0]) +
"while trying to get requests from [" + str(balancerqueue_dir) + "].")
return requests
def doInstall(info):
" do install of activation info"
logger.info("OracleDatabaseContainer: doInstall:Enter")
hostname = "localhost"
try:
hostname = InetAddress.getLocalHost().getCanonicalHostName()
except:
type, value, traceback = sys.exc_info()
logger.severe("Hostname error:" + ` value `)
dbInstallOption = getVariableValue("DB_INSTALL_OPTION")
if dbInstallOption == "INSTALL_DB_AND_CONFIG":
globalLockString = "OracleEnabler-" + hostname
ContainerUtils.releaseGlobalLock(globalLockString)
logger.info("Released global lock with name: " + globalLockString)
try:
oracleDatabase = getVariableValue("ORACLE_DATABASE_OBJECT")
if oracleDatabase:
oracleDatabase.installActivationInfo(info)
except:
type, value, traceback = sys.exc_info()
logger.severe(
"Unexpected error in OracleDatabaseContainer:doInstall:" +
` value `)
logger.info("OracleDatabaseContainer: doInstall:Exit")
def startNode(self):
logInfo("startNode:Enter")
self.__environ = os.environ.copy()
### Starting Node
self.__ELASTIC_ARGS = "-f -Des.pidfile=" + self.__pidfile
if ContainerUtils.isWindows():
self.__ELASTIC_CMD = os.path.join(self.__bindir,
"elasticsearch.bat ")
else:
self.__ELASTIC_CMD = os.path.join(self.__bindir, "elasticsearch ")
self.__CMD = self.__ELASTIC_CMD + self.__ELASTIC_ARGS
logInfo("StartUp Command to be used : " + self.__CMD)
args = shlex.split(self.__CMD)
process = Popen(args,
stdout=None,
stderr=None,
env=self.__environ,
shell=False)
time.sleep(20)
logInfo("Start return Code : " + str(process.returncode))
logInfo("finding the archivesDir")
global archivesDir
archiveMgmtFeature = ContainerUtils.getFeatureInfo(
"Archive Management Support", proxy.container,
proxy.container.currentDomain)
archivesDir = os.path.join(self.__enginedir,
archiveMgmtFeature.archiveDirectory)
logInfo("Found archives dir " + archivesDir)
self.__toggleCheck = True
logInfo("startNode:Exit")
def getStatistic(name):
ContainerUtils.getLogger(proxy).fine("[hadoop_enabler_balancer] Beginning getStatistic()")
return (getStatistic_common(name))
ContainerUtils.getLogger(proxy).fine("[hadoop_enabler_balancer] Exiting getStatistic()")
def doInstall(info):
" do install of activation info"
logger.info("OracleDatabaseContainer: doInstall:Enter")
hostname = "localhost";
try:
hostname = InetAddress.getLocalHost().getCanonicalHostName()
except:
type, value, traceback = sys.exc_info()
logger.severe("Hostname error:" + `value`)
dbInstallOption = getVariableValue("DB_INSTALL_OPTION")
if dbInstallOption == "INSTALL_DB_AND_CONFIG":
globalLockString = "OracleEnabler-" + hostname
ContainerUtils.releaseGlobalLock(globalLockString)
logger.info("Released global lock with name: " + globalLockString)
try:
oracleDatabase = getVariableValue("ORACLE_DATABASE_OBJECT")
if oracleDatabase:
oracleDatabase.installActivationInfo(info)
except:
type, value, traceback = sys.exc_info()
logger.severe("Unexpected error in OracleDatabaseContainer:doInstall:" + `value`)
logger.info("OracleDatabaseContainer: doInstall:Exit")
def stopBalancer():
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_balancer] Stopping existing Balancer if any")
hadoop_bin_dir = proxy.getContainer().getRuntimeContext().getVariable('hadoop_enabler_HADOOP_BIN_DIR').getValue()
#runCommand("sh " + os.path.join(hadoop_bin_dir, "stop-balancer.sh"))
""" Wordaround: version 0.23.7 returns 1 when successful """
if getHadoopVersion().startswith("0.23"):
runCommand(getScript("stop-balancer.sh"), expectedReturnCodes=[1])
else:
runCommand(getScript("stop-balancer.sh"))
def moveContentFiles():
hadoop_home_dir = proxy.getContainer().getRuntimeContext().getVariable('hadoop_enabler_HADOOP_HOME_DIR').getValue()
work_dir = proxy.getContainer().getRuntimeContext().getVariable('CONTAINER_WORK_DIR').getValue()
if ContainerUtils.isWindows():
pass
else:
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_common] Copying enabler content files to version-specific sub-directory.")
runCommand("cp -rv " + work_dir + "/hadoop-content/* " + hadoop_home_dir, shell=True)
def changePermissions():
hadoop_bin_dir = proxy.getContainer().getRuntimeContext().getVariable('hadoop_enabler_HADOOP_BIN_DIR').getValue()
hadoop_sbin_dir = proxy.getContainer().getRuntimeContext().getVariable('hadoop_enabler_HADOOP_SBIN_DIR').getValue()
java_home_dir = proxy.getContainer().getRuntimeContext().getVariable('GRIDLIB_JAVA_HOME').getValue()
if ContainerUtils.isWindows():
pass
else:
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_common] Changing file permissions on directory " + hadoop_bin_dir)
commandline = "chmod -Rv u+x " + hadoop_bin_dir
runCommand(commandline)
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_common] Changing file permissions on directory " + hadoop_sbin_dir)
commandline = "chmod -Rv u+x " + hadoop_sbin_dir
runCommand(commandline)
java_bin_dir = os.path.join(java_home_dir, "bin")
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_common] Changing file permissions on directory " + java_bin_dir)
commandline = "chmod -Rv u+x " + java_bin_dir
runCommand(commandline)
java_jrebin_dir = os.path.join(java_home_dir, "jre", "bin")
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_common] Changing file permissions on directory " + java_jrebin_dir)
commandline = "chmod -Rv u+x " + java_jrebin_dir
runCommand(commandline)
def getStatistic_common(name):
memoryBean = ManagementFactory.getMemoryMXBean()
if name == "enabler_HEAP_MEMORY_USAGE":
bytes = memoryBean.getHeapMemoryUsage().getUsed()
return bytes / 1024 / 1024 # convert to MiB
elif name == "enabler_NON_HEAP_MEMORY_USAGE":
bytes = memoryBean.getNonHeapMemoryUsage().getUsed()
return bytes / 1024 / 1024 # convert to MiB
elif name == "enabler_DATANODE_DECOMMISION_REQUESTS":
hadoop_home_dir = proxy.getContainer().getRuntimeContext().getVariable('hadoop_enabler_HADOOP_HOME_DIR').getValue()
if ContainerUtils.isWindows():
raise Exception("[hadoop_enabler_common] Windows not yet implemented in by this enabler.")
else:
commandline = "sh " + os.path.join(hadoop_home_dir, "bin", "hadoop") + " fs -count " + decommissionqueue_dir
output = runCommand(commandline, expectedReturnCodes=[0, 255], suppressOutput=True)
if (output[0] == 0):
stdout = str(output[1])
count = int(stdout.split()[1])
return int(count)
elif (output[0] == 255):
# Decommission request directory doesn't exist. Not expected to exist until the some datanode posts the first request
return int(0)
else:
ContainerUtils.getLogger(proxy).warning("[hadoop_enabler_common] Unexpected return code [" + str(output[0]) +
"] while attempting to retrieve statistic enabler_DATANODE_DECOMMISION_REQUESTS statistic. Assuming 0.")
print output
return int(0)
elif name.startswith('enabler_DISK_'):
tmpdir = proxy.getContainer().getRuntimeContext().getVariable('hadoop_enabler_TMP_DIR').getValue()
if name == "enabler_DISK_SPACE_FREE":
blocks = int(getStatistic_disk(tmpdir)[0])
return blocks / 1024 / 1024 # convert 1024-byte blocks to GiB
#return available
elif name == "enabler_DISK_SPACE_USED":
blocks = int(getStatistic_disk(tmpdir)[1])
return blocks / 1024 / 1024 # convert 1024-byte blocks to GiB
#return used
elif name == "enabler_DISK_SPACE_USED_PERCENT":
return getStatistic_disk(tmpdir)[2]
#return int(percent[:-1])
else:
raise Exception("[hadoop_enabler_common] Unsupported statistic type requested [" + str(name) + "]")
else:
raise Exception("[hadoop_enabler_common] Unsupported statistic type requested [" + str(name) + "]")
def isContainerRunning():
port = runtimeContext.getVariable('MYSQL_PORT').getValue()
basedir = runtimeContext.getVariable('MYSQL_BASE_DIR').getValue()
user = runtimeContext.getVariable('MYSQL_USER').getValue()
pw = runtimeContext.getVariable('MYSQL_PW').getValue()
bindir = os.path.join(basedir, "bin")
host = socket.gethostname()
status = call([os.path.join(bindir, "mysqladmin"), "--port=" + port, "-h", host, "--user="******"--password="******"ping"])
ContainerUtils.getLogger(proxy).info("mysqladmin ping returns " + str(status))
if status == 0:
return True
else:
return False
def isContainerRunning():
processRequetsThread = proxy.getContainer().getRuntimeContext().getVariable('hadoop_enabler_balancer_BALANCER_PROCESS_REQUEST_THREAD').getValue()
checkForRequetsThread = proxy.getContainer().getRuntimeContext().getVariable('hadoop_enabler_balancer_BALANCER_CHECKFOR_REQUEST_THREAD').getValue()
status = processRequetsThread.isAlive() and checkForRequetsThread.isAlive()
if status == True:
pass
else:
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_balancer.isContainerRunning()] Return status is: " + str(status))
return status
def __lock(self):
"get build lock"
logger.info("Acquire build lock:" + self.__buildLock)
self.__locked = ContainerUtils.acquireGlobalLock(
self.__buildLock, self.__lockExpire, self.__lockWait)
if not self.__locked:
raise Exception("Unable to acquire build lock:" + self.__buildLock)
def archiveDeploy(self, archiveName, archiveLocators, properties):
try:
if self.__role != "primary":
raise Exception("Not a primary Swam manage node")
archiveZip = str(
ContainerUtils.retrieveAndConfigureArchiveFile(
proxy.container, archiveName, archiveLocators, properties))
if archiveZip[-4:] != ".zip":
raise Exception(
"Archive must be a ZIP file containing Docker Compose project"
)
logger.info("Deploying archive:" + archiveZip)
project = getArchiveDeployProperty("project-name", properties,
None)
if not project:
raise Exception(
"project-name is required: Must be unique within this Docker swarm cluster"
)
dir = os.path.join(self.__deploydir, archiveName, project)
if os.path.isdir(dir):
raise Exception("Archive is already deployed")
extractZip(archiveZip, dir)
composeFile = getDockerComposeFile(dir)
copyContainerEnvironment()
os.environ["DOCKER_HOST"] = "tcp://" + self.__manageAddr
os.environ["COMPOSE_HTTP_TIMEOUT"] = "300"
cmdlist = [
self.__dockerCompose, "--file", composeFile, "--project-name",
project, "create", "--force-recreate"
]
logger.info("Executing:" + list2str(cmdlist))
retcode = call(cmdlist)
logger.info("Return code:" + str(retcode))
if retcode != 0:
try:
self.archiveUndeploy(archiveName, properties)
except:
pass
finally:
if os.path.isdir(dir):
distutils.dir_util.remove_tree(dir)
path = os.path.join(self.__deploydir, archiveName)
os.rmdir(path)
raise Exception("Archive deploy failed:" + archiveName)
except:
type, value, traceback = sys.exc_info()
logger.severe("archiveDeploy error:" + ` value `)
raise
def __lock(self):
"get global lock"
self.__locked = ContainerUtils.acquireGlobalLock(
self.__clusterConfigDir, self.__lockExpire, self.__lockWait)
if not self.__locked:
raise Exception("Unable to acquire global lock:" +
self.__clusterConfigDir)
def runCommand(commandline, stdin=None, stdout=None, expectedReturnCodes=None, suppressOutput=None, shell=None):
if (expectedReturnCodes == None): expectedReturnCodes = [0]
if (suppressOutput == None): suppressOutput = False
if (shell == None): shell = False
stderr = None
if (suppressOutput):
stdout=PIPE
stderr=PIPE
else:
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_common] Running command [" + commandline + "]")
if shell:
args = commandline
else:
args = shlex.split(commandline)
os.unsetenv("LD_LIBRARY_PATH")
os.unsetenv("LD_PRELOAD")
if stdin == None:
p = Popen(args, stdout=stdout, stdin=None, stderr=stderr, shell=shell)
output = p.communicate()
else:
p = Popen(args, stdout=stdout, stdin=PIPE, stderr=stderr, shell=shell)
output = p.communicate(input=stdin)
outputlist = [p.returncode]
for item in output:
outputlist.append(item)
if (outputlist[0] in expectedReturnCodes ):
if not (suppressOutput):
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_common] Command return code was [" + str(outputlist[0]) + "]")
printStdoutPipe(stdout, outputlist)
else:
ContainerUtils.getLogger(proxy).severe("[hadoop_enabler_common] Return code " + str(outputlist[0]) +
" was not in list of expected return codes" + str(expectedReturnCodes))
if (suppressOutput):
ContainerUtils.getLogger(proxy).severe("[hadoop_enabler_common] Command was [" + commandline + "]")
printStdoutPipe(stdout, outputlist)
ContainerUtils.getLogger(proxy).finer("[hadoop_enabler_common] exiting runCommand(). Returning outputlist:" + (str(outputlist)))
return outputlist
def killOrphans():
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_common] Killing any orphaned process on this engine remaining from a previous execution")
hadoop_home_dir = proxy.getContainer().getRuntimeContext().getVariable('hadoop_enabler_HADOOP_HOME_DIR').getValue()
if ContainerUtils.isWindows():
raise Exception("[hadoop_enabler_common] Windows not yet implemented in by this enabler.")
else:
commandline = "ps x"
output = runCommand(commandline, expectedReturnCodes=[0, 255], stdout=PIPE, suppressOutput=True)
for line in output[1].splitlines():
if hadoop_home_dir in line:
if "java" in line:
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_common] Issuing Kill command for orphaned process [" + str(line) + "]")
pid = int(line.split(None, 1)[0])
os.kill(pid, signal.SIGKILL)
def __configureDomain(self):
"configure domain"
configureDomainCommand = "configDomain.sh"
if ContainerUtils.isWindows():
configureDomainCommand = "configDomain.bat"
cmdfile = os.path.join(self.__baseDir, "bin", configureDomainCommand)
logger.info("Configure domain")
self.__copyContainerEnvironment()
retcode = call([cmdfile])
logger.info("Return code: " + str(retcode))
def __configureDomain(self):
"configure domain"
configureDomainCommand = "configDomain.sh"
if ContainerUtils.isWindows():
configureDomainCommand = "configDomain.bat"
cmdfile = os.path.join(self.__baseDir, "bin", configureDomainCommand)
logger.info("Configure domain")
self.__copyContainerEnvironment()
retcode = call([cmdfile])
logger.info("Return code: " + str(retcode))
def archiveDeploy(archiveName, archiveLocators):
logInfo("deploying archive " + archiveName)
try:
elastic = getVariableValue("ELASTICSEARCH_NODE_OBJECT")
except:
type, value, traceback = sys.exc_info()
logSevere("Unexpected error in ElasticSearch:archiveDeploy:" +
` value `)
else:
try:
elastic.killNode()
except:
pass
else:
elastic.installPlugins(archiveName, archivesDir)
elastic.startNode()
ContainerUtils.retrieveAndConfigureArchiveFile(proxy.container,
archiveName,
archiveLocators, None)
logInfo("End of deploying archive " + archiveName)
def getStatistic_disk(directory):
if ContainerUtils.isWindows():
raise Exception("[hadoop_enabler_common] Windows not yet implemented in by this enabler.")
return[0,0,0]
else:
df = Popen(["df", directory], stdout=PIPE)
output = df.communicate()[0]
device, size, used, available, percent, mountpoint = output.split("\n")[1].split()
return [available, used, int(percent[:-1])]
def archiveDeploy(self, archiveName, archiveLocators, properties):
try:
if self.__role != "primary":
raise Exception("Not a primary Swam manage node")
archiveZip = str(ContainerUtils.retrieveAndConfigureArchiveFile(proxy.container, archiveName, archiveLocators, properties))
if archiveZip[-4:] != ".zip":
raise Exception("Archive must be a ZIP file containing Docker Compose project")
logger.info("Deploying archive:" + archiveZip)
project = getArchiveDeployProperty("project-name", properties, None)
if not project:
raise Exception("project-name is required: Must be unique within this Docker swarm cluster")
dir = os.path.join(self.__deploydir, archiveName, project)
if os.path.isdir(dir):
raise Exception("Archive is already deployed")
extractZip(archiveZip, dir)
composeFile=getDockerComposeFile(dir)
copyContainerEnvironment()
os.environ["DOCKER_HOST"] = "tcp://" + self.__manageAddr
os.environ["COMPOSE_HTTP_TIMEOUT"] = "300"
cmdlist = [self.__dockerCompose, "--file", composeFile, "--project-name", project, "create", "--force-recreate"]
logger.info("Executing:"+ list2str(cmdlist))
retcode = call(cmdlist)
logger.info("Return code:" + str(retcode))
if retcode != 0:
try:
self.archiveUndeploy(archiveName, properties)
except:
pass
finally:
if os.path.isdir(dir):
distutils.dir_util.remove_tree(dir)
path = os.path.join(self.__deploydir, archiveName)
os.rmdir(path)
raise Exception("Archive deploy failed:" + archiveName)
except:
type, value, traceback = sys.exc_info()
logger.severe("archiveDeploy error:" + `value`)
raise
def doShutdown():
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_balancer] Beginning doShutdown()")
proxy.getContainer().getRuntimeContext().getVariable('hadoop_enabler_balancer_BALANCER_EXIT_FLAG').setValue("True")
stopBalancer()
processRequetsThread = proxy.getContainer().getRuntimeContext().getVariable('hadoop_enabler_balancer_BALANCER_PROCESS_REQUEST_THREAD').getValue()
if processRequetsThread.isAlive():
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_balancer] Waiting for Processing thread to exit.")
processRequetsThread.join()
checkForReqThread = proxy.getContainer().getRuntimeContext().getVariable('hadoop_enabler_balancer_BALANCER_CHECKFOR_REQUEST_THREAD').getValue()
if checkForReqThread.isAlive():
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_balancer] Waiting for Queue monitor thread to exit.")
checkForReqThread.join()
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_balancer] Exiting doShutdown()")
def deleteHDFSFile(file):
hadoop_bin_dir = proxy.getContainer().getRuntimeContext().getVariable('hadoop_enabler_HADOOP_BIN_DIR').getValue()
commandline = os.path.join(hadoop_bin_dir, "hadoop") + " fs -rm " + str(file)
output = runCommand(commandline, expectedReturnCodes=[0, 255])
if (output[0] == 0):
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_common] Successfully deleted [" + str(file) + "]")
elif (output[0] == 255):
ContainerUtils.getLogger(proxy).warning("[hadoop_enabler_common] File already deleted [" + str(file) + "]. Continuing Processing")
else:
ContainerUtils.getLogger(proxy).severe("[hadoop_enabler_common] Unexpected return code [" + str(output[0]) + "] when attempting to delete.]")
def rcvTrue(rcv):
ContainerUtils.getLogger(proxy).finer("[hadoop_enabler_common] checking runtimecontext variable [" + str(rcv) + "]")
rcvvalue = proxy.getContainer().getRuntimeContext().getVariable(rcv).getValue()
ContainerUtils.getLogger(proxy).finest("[hadoop_enabler_common] value is [" + str(rcvvalue) + "].")
if (str(rcvvalue).lower() in ("yes", "y", "true", "t", "1")):
result = True
elif (str(rcvvalue).lower() in ("no", "n", "false", "f", "0")):
result = False
else:
raise Exception("[hadoop_enabler_common] Invalid value for boolean conversion: [" + str(rcvvalue) + "]")
ContainerUtils.getLogger(proxy).finer("[hadoop_enabler_common] Exiting Checking enabler flag. Result is [" + str(result) + "]")
return result
def run(self):
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_balancer] Starting checkForRequestsThread.run()")
"""
Redirect stdout and stderr for this thread.
"""
sys.stdout = self.slstdout
sys.stderr = self.slstderr
try:
checkForRequests(self.lock)
except:
ContainerUtils.getLogger(proxy).severe("[hadoop_enabler_balancer] Unexpected error from checkForRequests thread")
traceback.print_exc()
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_balancer] Exiting checkForRequestsThread.run()")
def doStart():
"do start"
logger.info("Enter OracleCoherenceEnabler:doStart")
# install archives: we need to install them before starting the oracle coherence server
archiveFeatureInfo = ContainerUtils.getFeatureInfo(
Feature.ARCHIVE_FEATURE_NAME, proxy.container,
proxy.container.currentDomain)
if archiveFeatureInfo != None:
rootDirPath = runtimeContext.rootDirectory.absolutePath
archiveDir = archiveFeatureInfo.archiveDirectory
archiveDirPath = os.path.join(rootDirPath, archiveDir)
if os.path.isdir(archiveDirPath):
deploymentDir = getVariableValue(
"COHERENCE_RUNTIME_DEPLOYMENT_DIR")
logger.info("OracleCoherenceEnabler:doStart:Extracting archives:" +
archiveDirPath + " --> " + deploymentDir)
uzip = UnZipFile()
for entry in os.listdir(archiveDirPath):
archiveFile = os.path.join(archiveDirPath, entry)
if os.path.isfile(archiveFile):
logger.info(
"OracleCoherenceEnabler:doStart:Extracting archive:" +
archiveFile)
try:
uzip.extract(archiveFile, deploymentDir)
except:
type, value, traceback = sys.exc_info()
logger.severe(
"OracleCoherenceEnabler:doStart:Error extracting archive file:"
+ ` value `)
# we do not want the archive feature to install the archives
archiveFeatureInfo.setInstallArchivesViaFileCopy(False)
# start coherence server
proxy.doStart()
logger.info("Exit OracleCoherenceEnabler:doStart")
def checkForRequests(lock):
pollperiod = getContainerRunningConditionPollPeriod()/1000
while not rcvTrue('hadoop_enabler_balancer_BALANCER_EXIT_FLAG'):
lock.acquire()
receivedReqs = getBalancerReceivedRequests()
currentReqs = getBalancerRequestsFromQueue()
ContainerUtils.getLogger(proxy).fine("[hadoop_enabler_balancer] Current request list [" + str(currentReqs) + "].")
ContainerUtils.getLogger(proxy).fine("[hadoop_enabler_balancer] Previous request.list [" + str(receivedReqs) + "].")
if receivedReqs != currentReqs:
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_balancer] Balancer Requests have changed since last checked.")
proxy.getContainer().getRuntimeContext().getVariable('hadoop_enabler_balancer_BALANCER_NEW_REQUEST_FLAG').setValue("True")
setBalancerReceivedRequests(currentReqs)
stopBalancer()
lock.release()
time.sleep(pollperiod)
def processRequests(lock):
pollperiod = getContainerRunningConditionPollPeriod()/1000
while not rcvTrue('hadoop_enabler_balancer_BALANCER_EXIT_FLAG'):
timer = threading.Timer(pollperiod, retryTimerElapsed)
timer.start()
lock.acquire()
if rcvTrue("hadoop_enabler_balancer_BALANCER_NEW_REQUEST_FLAG"):
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_balancer] Beginning processing of new requests")
receivedReqs = getBalancerReceivedRequests()
local_receivedReqs = list(receivedReqs)
proxy.getContainer().getRuntimeContext().getVariable('hadoop_enabler_balancer_BALANCER_NEW_REQUEST_FLAG').setValue("False")
lock.release()
runBalancer()
lock.acquire()
if not rcvTrue("hadoop_enabler_balancer_BALANCER_NEW_REQUEST_FLAG"):
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_balancer] Deleting processed requests")
for request in local_receivedReqs:
deleteHDFSFile(request)
setBalancerReceivedRequests([])
lock.release()
if (rcvTrue("hadoop_enabler_balancer_BALANCER_NEW_REQUEST_FLAG") or rcvTrue("hadoop_enabler_balancer_BALANCER_EXIT_FLAG")):
if timer.isAlive():
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_balancer] Terminating retry timer early")
timer.cancel()
else:
timer.join()
def doStart():
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_balancer] Beginning doStart()")
hadoop_home_dir = proxy.getContainer().getRuntimeContext().getVariable('hadoop_enabler_HADOOP_HOME_DIR').getValue()
doStart_common()
processRequetsThread = proxy.getContainer().getRuntimeContext().getVariable('hadoop_enabler_balancer_BALANCER_PROCESS_REQUEST_THREAD').getValue()
processRequetsThread.start()
startupdelay = 0
if (True): #TODO Base decision to delay on whether Namenode just started or has been running for a while
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_balancer] Hadoop cluster is just starting. Waiting [" + str(startupdelay) +
"] seconds before processing any balancer requests so more datanodes can join.")
else:
startupdelay = 0
time.sleep(startupdelay)
checkForReqThread = proxy.getContainer().getRuntimeContext().getVariable('hadoop_enabler_balancer_BALANCER_CHECKFOR_REQUEST_THREAD').getValue()
checkForReqThread.start()
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_balancer] Exiting doStart()")
def __lock(self):
"get global lock"
self.__locked = ContainerUtils.acquireGlobalLock(
self.__swarmid, self.__lockExpire, self.__lockWait)
if not self.__locked:
raise "Unable to acquire global lock:" + self.__swarmid
def getContainerRunningConditionPollPeriod():
pollperiod = proxy.getContainer().getRuntimeContext().getVariable('hadoop_enabler_ENABLER_RUNNING_POLLPERIOD').getValue()
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_common] Will verify enabler is running every " + str(pollperiod) + " seconds.")
return float(pollperiod) * 1000
def __unlock(self):
"unlock global lock"
logger.info("Release build lock:" + self.__buildLock)
if self.__locked:
ContainerUtils.releaseGlobalLock(self.__buildLock)
def printStdoutPipe(stdout, outputlist):
if (stdout == PIPE):
ContainerUtils.getLogger(proxy).info("[hadoop_enabler_common] Command STDOUT:")
print outputlist[1]
def __init__(self, additionalVariables):
" initialize oracle database"
self.__hostname = "localhost";
try:
self.__hostname = InetAddress.getLocalHost().getCanonicalHostName()
except:
type, value, traceback = sys.exc_info()
logger.severe("Hostname error:" + `value`)
additionalVariables.add(RuntimeContextVariable("ORACLE_HOSTNAME", self.__hostname, RuntimeContextVariable.ENVIRONMENT_TYPE))
dbPassword = getVariableValue("DB_PASSWORD_ALL")
if dbPassword and dbPassword.strip():
self.__sysPassword = dbPassword
additionalVariables.add(RuntimeContextVariable("SYS_PWD", dbPassword, RuntimeContextVariable.ENVIRONMENT_TYPE))
additionalVariables.add(RuntimeContextVariable("DBSNMP_PWD", dbPassword, RuntimeContextVariable.ENVIRONMENT_TYPE));
additionalVariables.add(RuntimeContextVariable("SYSMAN_PWD", dbPassword, RuntimeContextVariable.ENVIRONMENT_TYPE))
additionalVariables.add(RuntimeContextVariable("SYSTEM_PWD", dbPassword, RuntimeContextVariable.ENVIRONMENT_TYPE));
else:
self.__sysPassword = getVariableValue("SYS_PWD")
dbDataLocation = getVariableValue("DB_DATA_LOC")
if dbDataLocation and os.path.isdir(dbDataLocation):
dbName = getVariableValue("DB_NAME")
dbDataDir = os.path.join(dbDataLocation, dbName)
if os.path.isdir(dbDataDir):
logger.info("DB Data directory already exists:" + dbDataDir + "; Setting DB_INSTALL_OPTION to INSTALL_DB_SWONLY")
additionalVariables.add(RuntimeContextVariable( "DB_INSTALL_OPTION", "INSTALL_DB_SWONLY", RuntimeContextVariable.ENVIRONMENT_TYPE))
tcpPort = getVariableValue("TCP_PORT");
self.__serviceName = getVariableValue("DB_GLOBAL_NAME")
sb = StringBuilder("jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)")
sb.append("(HOST=").append(self.__hostname).append(")")
sb.append("(PORT=").append(tcpPort).append("))")
sb.append("(CONNECT_DATA=(SERVICE_NAME=").append(self.__serviceName).append(")))")
self.__oracleServiceUrl = sb.toString()
logger.info("Oracle listener service URL:" + self.__oracleServiceUrl)
self.__jdbcUrl = "jdbc:oracle:thin:@" + self.__hostname +":"+ tcpPort + ":" + self.__serviceName
runtimeContext.addVariable(RuntimeContextVariable("JDBC_URL", self.__jdbcUrl, RuntimeContextVariable.STRING_TYPE, "Oracle Thin Driver JDBC Url", True, RuntimeContextVariable.NO_INCREMENT))
oracleDriver = "oracle.jdbc.OracleDriver"
runtimeContext.addVariable(RuntimeContextVariable("JDBC_DRIVER", oracleDriver, RuntimeContextVariable.STRING_TYPE, "Oracle Thin Driver class", True, RuntimeContextVariable.NO_INCREMENT))
self.__dbControl = Boolean.parseBoolean(getVariableValue("CONFIG_DBCONTROL", "false"))
if self.__dbControl:
self.__dbCtrlPort = getVariableValue("DBCONTROL_HTTP_PORT")
additionalVariables.add(RuntimeContextVariable( "HTTPS_PORT", self.__dbCtrlPort, RuntimeContextVariable.STRING_TYPE))
oracleDir = getVariableValue("ORACLE_DIR")
self.__markerFilePath = os.path.join(oracleDir, ".#dsoracle")
self.__maintFilePath = getVariableValue("ORACLE_MAINT_FILE")
dbInstallOption = getVariableValue("DB_INSTALL_OPTION")
if dbInstallOption == "INSTALL_DB_AND_CONFIG":
globalLockString = "OracleEnabler-" + self.__hostname
logger.info("Requesting Global Lock with name: " + globalLockString)
domain = proxy.getContainer().getCurrentDomain()
options = domain.getOptions()
maxActivationTimeOut = options.getProperty(Options.MAX_ACTIVATION_TIME_IN_SECONDS)
lockTimeOut = Long.parseLong(maxActivationTimeOut) * 1000
acquired = ContainerUtils.acquireGlobalLock(globalLockString, lockTimeOut , lockTimeOut)
if acquired:
logger.info("Acquired Global lock with name: " + globalLockString)
else:
logger.severe("Could not acquire Global lock with name: " + globalLockString)
raise Exception("Could not acquire Global lock with name: " + globalLockString)
def __lock(self):
"get build lock"
logger.info("Acquire build lock:" + self.__buildLock)
self.__locked = ContainerUtils.acquireGlobalLock(self.__buildLock, self.__lockExpire, self.__lockWait)
if not self.__locked:
raise Exception("Unable to acquire build lock:" + self.__buildLock)
def __unlock(self):
"unlock global lock"
logger.info("Release build lock:" + self.__buildLock)
if self.__locked:
ContainerUtils.releaseGlobalLock(self.__buildLock)
def __unlock(self):
"unlock global lock"
if self.__locked:
ContainerUtils.releaseGlobalLock(self.__clusterConfigDir)
self.__locked = None
"dfs.datanode.address",
"dfs.datanode.http.address",
"dfs.datanode.ipc.address",
"dfs.hosts.exclude",
"dfs.name.dir",
"dfs.data.dir",
"mapred.job.tracker",
"mapred.job.tracker.http.address",
"mapred.task.tracker.http.address"
]
try: proxy
except NameError:
globals()['proxy'] = inspect.currentframe().f_back.f_globals['proxy']
else: pass
logger = ContainerUtils.getLogger(proxy)
def getDynamicGridlibDependencies():
logger.info("[hadoop_enabler_common] Beginning getDynamicGridlibDependencies()")
hadoopVersion = getHadoopVersion()
logger.info("[hadoop_enabler_common] Hadoop Distribution version is [" + str(hadoopVersion) +"]")
defaultDomainGridlib = GridlibInfo()
defaultDomainGridlib.name = "default-domain-type"
logger.info("[hadoop_enabler_common] Adding Hadoop distribution dependency")
gridlib = GridlibInfo()
gridlib.name = "hadoop-distribution"
gridlib.version = str(hadoopVersion)
def __unlock(self):
"unlock global lock"
if self.__locked:
ContainerUtils.releaseGlobalLock(self.__swarmid)
self.__locked = None
def doInitHadoopProps(userProp_RCVname, userPropFile_RCVname, additionalVariables):
ContainerUtils.getLogger(proxy).info("[hadoop_enabler] Checking for user property file to augment [" + str(userProp_RCVname) + "].")
userPropsRCV = proxy.getContainer().getRuntimeContext().getVariable(userProp_RCVname)
userPropsPredefined = False
if (userPropsRCV == None):
userProps = ""
else:
userPropsPredefined = True
userProps = userPropsRCV.getValue()
ContainerUtils.getLogger(proxy).info("[hadoop_enabler] User properties variable ${" + userProp_RCVname +
"} is was imported or pre-defined on component. Starting value is [" + str(userProps) + "].")
userPropFile = proxy.getContainer().getRuntimeContext().getVariable(userPropFile_RCVname).getValue()
if (userPropFile != "") and os.path.isfile(userPropFile):
ContainerUtils.getLogger(proxy).info("[hadoop_enabler] User property file found [" + str(userPropFile) + "].")
xmldoc = minidom.parse(userPropFile)
propertylist = xmldoc.getElementsByTagName('property')
if propertylist == None or len(propertylist) == 0:
ContainerUtils.getLogger(proxy).warning("[hadoop_enabler] No property elements found in user property file.")
else:
for element in propertylist:
nameElements = element.getElementsByTagName("name")
name = getNodeText(nameElements[0])
isReserved = False
for reservedproperty in reservedproperties:
if reservedproperty.count(name) > 0:
isReserved = True
break
if isReserved:
ContainerUtils.getLogger(proxy).warning("[hadoop_enabler] The property [" + str(name) + "] is managed by the Hadoop Enabler. Will ignore user supplied value.")
else:
ContainerUtils.getLogger(proxy).info("[hadoop_enabler] Applying user property [" + str(element.toxml()) + "].")
userProps = userProps + element.toxml()
if userPropsPredefined:
proxy.getContainer().getRuntimeContext().getVariable(userProp_RCVname).setValue(userProps)
else:
additionalVariables.add(RuntimeContextVariable(userProp_RCVname, userProps, RuntimeContextVariable.STRING_TYPE,"User Supplied Hadoop properties" , False, RuntimeContextVariable.NO_INCREMENT))
def __init__(self, additionalVariables):
" initialize oracle database"
self.__hostname = "localhost"
try:
self.__hostname = InetAddress.getLocalHost().getCanonicalHostName()
except:
type, value, traceback = sys.exc_info()
logger.severe("Hostname error:" + ` value `)
additionalVariables.add(
RuntimeContextVariable("ORACLE_HOSTNAME", self.__hostname,
RuntimeContextVariable.ENVIRONMENT_TYPE,
"Oracle Hostname", True,
RuntimeContextVariable.NO_INCREMENT))
listenAddress = getVariableValue("LISTEN_ADDRESS")
additionalVariables.add(
RuntimeContextVariable("ORACLE_LISTEN_ADDRESS", listenAddress,
RuntimeContextVariable.ENVIRONMENT_TYPE,
"Oracle Listen Address", True,
RuntimeContextVariable.NO_INCREMENT))
dbPassword = getVariableValue("DB_PASSWORD_ALL")
if dbPassword and dbPassword.strip():
self.__sysPassword = dbPassword
additionalVariables.add(
RuntimeContextVariable(
"SYS_PWD", dbPassword,
RuntimeContextVariable.ENVIRONMENT_TYPE))
additionalVariables.add(
RuntimeContextVariable(
"DBSNMP_PWD", dbPassword,
RuntimeContextVariable.ENVIRONMENT_TYPE))
additionalVariables.add(
RuntimeContextVariable(
"SYSMAN_PWD", dbPassword,
RuntimeContextVariable.ENVIRONMENT_TYPE))
additionalVariables.add(
RuntimeContextVariable(
"SYSTEM_PWD", dbPassword,
RuntimeContextVariable.ENVIRONMENT_TYPE))
else:
self.__sysPassword = getVariableValue("SYS_PWD")
dbDataLocation = getVariableValue("DB_DATA_LOC")
if dbDataLocation and os.path.isdir(dbDataLocation):
dbName = getVariableValue("DB_NAME")
dbDataDir = os.path.join(dbDataLocation, dbName)
if os.path.isdir(dbDataDir):
logger.info("DB Data directory already exists:" + dbDataDir +
"; Setting DB_INSTALL_OPTION to INSTALL_DB_SWONLY")
additionalVariables.add(
RuntimeContextVariable(
"DB_INSTALL_OPTION", "INSTALL_DB_SWONLY",
RuntimeContextVariable.ENVIRONMENT_TYPE))
tcpPort = getVariableValue("TCP_PORT")
self.__serviceName = getVariableValue("DB_GLOBAL_NAME")
sb = StringBuilder(
"jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)")
sb.append("(HOST=").append(self.__hostname).append(")")
sb.append("(PORT=").append(tcpPort).append("))")
sb.append("(CONNECT_DATA=(SERVICE_NAME=").append(
self.__serviceName).append(")))")
self.__oracleServiceUrl = sb.toString()
logger.info("Oracle listener service URL:" + self.__oracleServiceUrl)
self.__jdbcUrl = "jdbc:oracle:thin:@" + self.__hostname + ":" + tcpPort + ":" + self.__serviceName
additionalVariables.add(
RuntimeContextVariable("JDBC_URL", self.__jdbcUrl,
RuntimeContextVariable.STRING_TYPE,
"Oracle Thin Driver JDBC Url", True,
RuntimeContextVariable.NO_INCREMENT))
oracleDriver = "oracle.jdbc.OracleDriver"
additionalVariables.add(
RuntimeContextVariable("JDBC_DRIVER", oracleDriver,
RuntimeContextVariable.STRING_TYPE,
"Oracle Thin Driver class", True,
RuntimeContextVariable.NO_INCREMENT))
self.__dbControl = Boolean.parseBoolean(
getVariableValue("CONFIG_DBCONTROL", "false"))
if self.__dbControl:
self.__dbCtrlPort = getVariableValue("DBCONTROL_HTTP_PORT")
additionalVariables.add(
RuntimeContextVariable("HTTPS_PORT", self.__dbCtrlPort,
RuntimeContextVariable.STRING_TYPE))
oracleDir = getVariableValue("ORACLE_DIR")
self.__markerFilePath = os.path.join(oracleDir, ".#dsoracle")
self.__maintFilePath = getVariableValue("ORACLE_MAINT_FILE")
dbInstallOption = getVariableValue("DB_INSTALL_OPTION")
if dbInstallOption == "INSTALL_DB_AND_CONFIG":
globalLockString = "OracleEnabler-" + self.__hostname
logger.info("Requesting Global Lock with name: " +
globalLockString)
domain = proxy.getContainer().getCurrentDomain()
options = domain.getOptions()
maxActivationTimeOut = options.getProperty(
Options.MAX_ACTIVATION_TIME_IN_SECONDS)
lockTimeOut = Long.parseLong(maxActivationTimeOut) * 1000
acquired = ContainerUtils.acquireGlobalLock(
globalLockString, lockTimeOut, lockTimeOut)
if acquired:
logger.info("Acquired Global lock with name: " +
globalLockString)
else:
logger.severe("Could not acquire Global lock with name: " +
globalLockString)
raise Exception("Could not acquire Global lock with name: " +
globalLockString)
