def dump_command_to_json(self, command):
"""
Converts command to json file and returns file path
"""
# Perform few modifications to stay compatible with the way in which
# site.pp files are generated by manifestGenerator.py
public_fqdn = hostname.public_hostname()
command['public_hostname'] = public_fqdn
# Now, dump the json file
command_type = command['commandType']
from ActionQueue import ActionQueue # To avoid cyclic dependency
if command_type == ActionQueue.STATUS_COMMAND:
# These files are frequently created, thats why we don't
# store them all, but only the latest one
file_path = os.path.join(self.tmp_dir, "status_command.json")
else:
task_id = command['taskId']
command['clusterHostInfo'] = manifestGenerator.decompressClusterHostInfo(command['clusterHostInfo'])
file_path = os.path.join(self.tmp_dir, "command-{0}.json".format(task_id))
# Json may contain passwords, that's why we need proper permissions
if os.path.isfile(file_path):
os.unlink(file_path)
with os.fdopen(os.open(file_path, os.O_WRONLY | os.O_CREAT,
0600), 'w') as f:
content = json.dumps(command, sort_keys = False, indent = 4)
f.write(content)
def dump_command_to_json(self, command):
"""
Converts command to json file and returns file path
"""
# Perform few modifications to stay compatible with the way in which
# site.pp files are generated by manifestGenerator.py
public_fqdn = hostname.public_hostname()
command['public_hostname'] = public_fqdn
# Now, dump the json file
command_type = command['commandType']
from ActionQueue import ActionQueue # To avoid cyclic dependency
if command_type == ActionQueue.STATUS_COMMAND:
# These files are frequently created, thats why we don't
# store them all, but only the latest one
file_path = os.path.join(self.tmp_dir, "status_command.json")
else:
task_id = command['taskId']
#command['clusterHostInfo'] = manifestGenerator.decompressClusterHostInfo(command['clusterHostInfo'])
file_path = os.path.join(self.tmp_dir, "command-{0}.json".format(task_id))
# Json may contain passwords, that's why we need proper permissions
if os.path.isfile(file_path):
os.unlink(file_path)
with os.fdopen(os.open(file_path, os.O_WRONLY | os.O_CREAT,
0600), 'w') as f:
content = json.dumps(command, sort_keys=False, indent=4)
# patch content
# ${AGENT_WORK_ROOT} -> AgentConfig.getWorkRootPath()
# ${AGENT_LOG_ROOT} -> AgentConfig.getLogPath()
content = content.replace("${AGENT_WORK_ROOT}",
self.config.getWorkRootPath())
content = content.replace("${AGENT_LOG_ROOT}", self.config.getLogPath())
f.write(content)
def __init__(self, config, controller, agentToggleLogger):
self.config = config
self.controller = controller
self.tmp_dir = config.getResolvedPath(AgentConfig.APP_TASK_DIR)
self.python_executor = PythonExecutor(self.tmp_dir, config,
agentToggleLogger)
self.status_commands_stdout = os.path.realpath(
posixpath.join(self.tmp_dir, 'status_command_stdout.txt'))
self.status_commands_stderr = os.path.realpath(
posixpath.join(self.tmp_dir, 'status_command_stderr.txt'))
self.public_fqdn = hostname.public_hostname()
self.stored_command = {}
self.allocated_ports = {}
self.log_folders = {}
self.allocated_ports_set = set()
# Clean up old status command files if any
try:
os.unlink(self.status_commands_stdout)
os.unlink(self.status_commands_stderr)
except OSError:
pass # Ignore fail
self.base_dir = os.path.realpath(
posixpath.join(config.getResolvedPath(AgentConfig.APP_PACKAGE_DIR),
"package"))
def __init__(self, config, controller):
self.config = config
self.tmp_dir = config.get('agent', 'prefix')
self.force_https_protocol = config.get_force_https_protocol()
self.exec_tmp_dir = Constants.AGENT_TMP_DIR
self.file_cache = FileCache(config)
self.status_commands_stdout = os.path.join(
self.tmp_dir, 'status_command_stdout.txt')
self.status_commands_stderr = os.path.join(
self.tmp_dir, 'status_command_stderr.txt')
self.public_fqdn = hostname.public_hostname(config)
# cache reset will be called on every agent registration
controller.registration_listeners.append(self.file_cache.reset)
# Construct the hadoop credential lib JARs path
self.credential_shell_lib_path = os.path.join(
config.get('security', 'credential_lib_dir',
self.DEFAULT_CREDENTIAL_SHELL_LIB_PATH), '*')
self.credential_conf_dir = config.get('security',
'credential_conf_dir',
self.DEFAULT_CREDENTIAL_CONF_DIR)
self.credential_shell_cmd = config.get(
'security', 'credential_shell_cmd',
self.DEFAULT_CREDENTIAL_SHELL_CMD)
# Clean up old status command files if any
try:
os.unlink(self.status_commands_stdout)
os.unlink(self.status_commands_stderr)
except OSError:
pass # Ignore fail
self.commands_in_progress_lock = threading.RLock()
self.commands_in_progress = {}
def build(self, id='-1'):
timestamp = int(time.time() * 1000)
version = self.read_agent_version()
register = {'responseId': int(id),
'timestamp': timestamp,
'hostname': self.config.getLabel(),
'publicHostname': hostname.public_hostname(),
'agentVersion': version
}
return register
def build(self, id='-1'):
timestamp = int(time.time() * 1000)
version = self.read_agent_version()
register = {
'responseId': int(id),
'timestamp': timestamp,
'hostname': self.config.getLabel(),
'publicHostname': hostname.public_hostname(),
'agentVersion': version
}
return register
def build(self, actualState, expectedState, allocated_ports, id='-1'):
timestamp = int(time.time() * 1000)
version = self.read_agent_version()
register = {'responseId': int(id),
'timestamp': timestamp,
'hostname': self.config.getLabel(),
'publicHostname': hostname.public_hostname(),
'agentVersion': version,
'actualState': actualState,
'expectedState': expectedState,
'allocatedPorts': allocated_ports
}
return register
def build(self, actualState, expectedState, allocated_ports, id='-1'):
timestamp = int(time.time() * 1000)
version = self.read_agent_version()
register = {
'responseId': int(id),
'timestamp': timestamp,
'hostname': self.config.getLabel(),
'publicHostname': hostname.public_hostname(),
'agentVersion': version,
'actualState': actualState,
'expectedState': expectedState,
'allocatedPorts': allocated_ports
}
return register
def build(self, id='-1'):
global clusterId, clusterDefinitionRevision, firstContact
timestamp = int(time.time()*1000)
version = self.read_agent_version()
register = { 'responseId' : int(id),
'timestamp' : timestamp,
'hostname' : self.config.getLabel(),
'currentPingPort' : 8670,
'publicHostname' : hostname.public_hostname(),
'hardwareProfile' : self.hardware.get(),
'agentEnv' : {},
'agentVersion' : version
}
return register
def build(self, id='-1'):
global clusterId, clusterDefinitionRevision, firstContact
timestamp = int(time.time() * 1000)
version = self.read_agent_version()
register = {
'responseId': int(id),
'timestamp': timestamp,
'hostname': self.config.getLabel(),
'currentPingPort': 8670,
'publicHostname': hostname.public_hostname(),
'hardwareProfile': self.hardware.get(),
'agentEnv': {},
'agentVersion': version
}
return register
def __init__(self, config, controller):
self.config = config
self.tmp_dir = config.getResolvedPath(AgentConfig.APP_TASK_DIR)
self.python_executor = PythonExecutor(self.tmp_dir, config)
self.status_commands_stdout = os.path.join(
self.tmp_dir, 'status_command_stdout.txt')
self.status_commands_stderr = os.path.join(
self.tmp_dir, 'status_command_stderr.txt')
self.public_fqdn = hostname.public_hostname()
self.applied_configs = {}
# Clean up old status command files if any
try:
os.unlink(self.status_commands_stdout)
os.unlink(self.status_commands_stderr)
except OSError:
pass # Ignore fail
self.base_dir = os.path.join(
config.getResolvedPath(AgentConfig.APP_PACKAGE_DIR), "package")
def __init__(self, config, controller):
self.config = config
self.tmp_dir = config.get('agent', 'prefix')
self.file_cache = FileCache(config)
self.python_executor = PythonExecutor(self.tmp_dir, config)
self.status_commands_stdout = os.path.join(self.tmp_dir,
'status_command_stdout.txt')
self.status_commands_stderr = os.path.join(self.tmp_dir,
'status_command_stderr.txt')
self.public_fqdn = hostname.public_hostname()
# cache reset will be called on every agent registration
controller.registration_listeners.append(self.file_cache.reset)
# Clean up old status command files if any
try:
os.unlink(self.status_commands_stdout)
os.unlink(self.status_commands_stderr)
except OSError:
pass # Ignore fail
def __init__(self, config, controller):
self.config = config
self.tmp_dir = config.get('agent', 'prefix')
self.file_cache = FileCache(config)
self.python_executor = PythonExecutor(self.tmp_dir, config)
self.status_commands_stdout = os.path.join(self.tmp_dir,
'status_command_stdout.txt')
self.status_commands_stderr = os.path.join(self.tmp_dir,
'status_command_stderr.txt')
self.public_fqdn = hostname.public_hostname()
# cache reset will be called on every agent registration
controller.registration_listeners.append(self.file_cache.reset)
# Clean up old status command files if any
try:
os.unlink(self.status_commands_stdout)
os.unlink(self.status_commands_stderr)
except OSError:
pass # Ignore fail
def __init__(self, config, controller):
self.config = config
self.tmp_dir = config.getResolvedPath(AgentConfig.APP_TASK_DIR)
self.python_executor = PythonExecutor(self.tmp_dir, config)
self.status_commands_stdout = os.path.join(self.tmp_dir,
'status_command_stdout.txt')
self.status_commands_stderr = os.path.join(self.tmp_dir,
'status_command_stderr.txt')
self.public_fqdn = hostname.public_hostname()
self.applied_configs = {}
# Clean up old status command files if any
try:
os.unlink(self.status_commands_stdout)
os.unlink(self.status_commands_stderr)
except OSError:
pass # Ignore fail
self.base_dir = os.path.join(
config.getResolvedPath(AgentConfig.APP_PACKAGE_DIR), "package")
def build(self, id='-1'):
global clusterId, clusterDefinitionRevision, firstContact
timestamp = int(time.time()*1000)
hostInfo = HostInfo()
agentEnv = { }
hostInfo.register(agentEnv)
version = self.read_agent_version()
register = { 'responseId' : int(id),
'timestamp' : timestamp,
'hostname' : hostname.hostname(),
'publicHostname' : hostname.public_hostname(),
'hardwareProfile' : self.hardware.get(),
'agentEnv' : agentEnv,
'agentVersion' : version
}
return register
def __init__(self, config, controller):
self.config = config
self.tmp_dir = config.get('agent', 'prefix')
self.exec_tmp_dir = Constants.AGENT_TMP_DIR
self.file_cache = FileCache(config)
self.status_commands_stdout = os.path.join(self.tmp_dir,
'status_command_stdout.txt')
self.status_commands_stderr = os.path.join(self.tmp_dir,
'status_command_stderr.txt')
self.public_fqdn = hostname.public_hostname(config)
# cache reset will be called on every agent registration
controller.registration_listeners.append(self.file_cache.reset)
# Clean up old status command files if any
try:
os.unlink(self.status_commands_stdout)
os.unlink(self.status_commands_stderr)
except OSError:
pass # Ignore fail
self.commands_in_progress_lock = threading.RLock()
self.commands_in_progress = {}
def build(self, id='-1'):
global clusterId, clusterDefinitionRevision, firstContact
timestamp = int(time.time()*1000)
hostInfo = HostInfo(self.config)
agentEnv = { }
hostInfo.register(agentEnv, False, False)
version = self.agentInfo.version()
current_ping_port = self.config.get('agent','current_ping_port')
register = { 'responseId' : int(id),
'timestamp' : timestamp,
'hostname' : hostname.hostname(),
'currentPingPort' : int(current_ping_port),
'publicHostname' : hostname.public_hostname(),
'hardwareProfile' : self.hardware.get(),
'agentEnv' : agentEnv,
'agentVersion' : version
}
return register
def __init__(self, config, controller):
self.config = config
self.tmp_dir = config.get('agent', 'prefix')
self.exec_tmp_dir = Constants.AGENT_TMP_DIR
self.file_cache = FileCache(config)
self.status_commands_stdout = os.path.join(
self.tmp_dir, 'status_command_stdout.txt')
self.status_commands_stderr = os.path.join(
self.tmp_dir, 'status_command_stderr.txt')
self.public_fqdn = hostname.public_hostname(config)
# cache reset will be called on every agent registration
controller.registration_listeners.append(self.file_cache.reset)
# Clean up old status command files if any
try:
os.unlink(self.status_commands_stdout)
os.unlink(self.status_commands_stderr)
except OSError:
pass # Ignore fail
self.commands_in_progress_lock = threading.RLock()
self.commands_in_progress = {}
def build(self, version, id='-1'):
global clusterId, clusterDefinitionRevision, firstContact
timestamp = int(time.time()*1000)
hostInfo = HostInfo(self.config)
agentEnv = { }
hostInfo.register(agentEnv, False, False)
current_ping_port = self.config.get('agent','current_ping_port')
register = { 'responseId' : int(id),
'timestamp' : timestamp,
'hostname' : hostname.hostname(self.config),
'currentPingPort' : int(current_ping_port),
'publicHostname' : hostname.public_hostname(self.config),
'hardwareProfile' : self.hardware.get(),
'agentEnv' : agentEnv,
'agentVersion' : version,
'prefix' : self.config.get('agent', 'prefix')
}
return register
def __init__(self, config, controller, agentToggleLogger):
self.config = config
self.controller = controller
self.tmp_dir = config.getResolvedPath(AgentConfig.APP_TASK_DIR)
self.python_executor = PythonExecutor(self.tmp_dir, config, agentToggleLogger)
self.status_commands_stdout = os.path.realpath(posixpath.join(self.tmp_dir,
'status_command_stdout.txt'))
self.status_commands_stderr = os.path.realpath(posixpath.join(self.tmp_dir,
'status_command_stderr.txt'))
self.public_fqdn = hostname.public_hostname()
self.stored_command = {}
self.allocated_ports = {}
self.log_folders = {}
self.allocated_ports_set = set()
# Clean up old status command files if any
try:
os.unlink(self.status_commands_stdout)
os.unlink(self.status_commands_stderr)
except OSError:
pass # Ignore fail
self.base_dir = os.path.realpath(posixpath.join(
config.getResolvedPath(AgentConfig.APP_PACKAGE_DIR), "package"))
def build(self, commandResult, id='-1',
componentsMapped=False):
timestamp = int(time.time() * 1000)
queueResult = self.actionQueue.result()
logger.info("Queue result: " + pformat(queueResult))
nodeStatus = {"status": "HEALTHY",
"cause": "NONE"}
heartbeat = {'responseId': int(id),
'timestamp': timestamp,
'hostname': self.config.getLabel(),
'nodeStatus': nodeStatus,
'fqdn': hostname.public_hostname()
}
commandsInProgress = False
if not self.actionQueue.commandQueue.empty():
commandsInProgress = True
if len(queueResult) != 0:
heartbeat['reports'] = []
for report in queueResult['reports']:
if report['reportResult']:
del report['reportResult']
heartbeat['reports'].append(report)
else:
# dropping the result but only recording the status
commandResult["commandStatus"] = report["status"]
pass
if len(heartbeat['reports']) > 0:
# There may be IN_PROGRESS tasks
commandsInProgress = True
pass
# For first request/heartbeat assume no components are mapped
if int(id) == 0:
componentsMapped = False
# create a report of command results (there is only one component at any point)
if len(heartbeat['reports']) > 0:
for report in heartbeat['reports']:
commandResult["commandStatus"] = report["status"]
pass
# Process component status - it can have internal or external STATUS requests and CONFIG requests
if queueResult['componentStatus']:
componentStatuses = []
for componentStatus in queueResult['componentStatus']:
if componentStatus['reportResult']:
del componentStatus['reportResult']
componentStatuses.append(componentStatus)
else:
commandResult["healthStatus"] = componentStatus["status"]
pass
if len(componentStatuses) > 0:
heartbeat['componentStatus'] = componentStatuses
logger.info("Sending heartbeat with response id: " + str(id) + " and "
"timestamp: " + str(timestamp) +
". Command(s) in progress: " + repr(commandsInProgress) +
". Components mapped: " + repr(componentsMapped))
logger.debug("Heartbeat : " + pformat(heartbeat))
return heartbeat
def writeHostnames(outputFile):
fqdn = hostname.hostname()
public_fqdn = hostname.public_hostname()
outputFile.write('$myhostname' + " = '" + fqdn + "'" + os.linesep)
outputFile.write('$public_hostname' + " = '" + public_fqdn + "'" + os.linesep)
def build(self, commandResult, id='-1', componentsMapped=False):
timestamp = int(time.time() * 1000)
queueResult = self.actionQueue.result()
self.agentToggleLogger.log("Queue result: " + pformat(queueResult))
nodeStatus = {"status": "HEALTHY", "cause": "NONE"}
if not self.actionQueue.componentPackage == '':
logger.info("Add package to heartbeat: " +
self.actionQueue.componentPackage)
heartbeat = {
'responseId': int(id),
'timestamp': timestamp,
'hostname': self.config.getLabel(),
'nodeStatus': nodeStatus,
'package': self.actionQueue.componentPackage,
'fqdn': hostname.public_hostname()
}
commandsInProgress = False
if not self.actionQueue.commandQueue.empty():
commandsInProgress = True
if len(queueResult) != 0:
heartbeat['reports'] = []
for report in queueResult['reports']:
if report['reportResult']:
del report['reportResult']
heartbeat['reports'].append(report)
else:
# dropping the result but only recording the status
commandResult["commandStatus"] = report["status"]
pass
if len(heartbeat['reports']) > 0:
# There may be IN_PROGRESS tasks
commandsInProgress = True
pass
# For first request/heartbeat assume no components are mapped
if int(id) == 0:
componentsMapped = False
# create a report of command results (there is only one component at any point)
if len(heartbeat['reports']) > 0:
for report in heartbeat['reports']:
commandResult["commandStatus"] = report["status"]
pass
# Process component status - it can have internal or external STATUS requests and CONFIG requests
if queueResult['componentStatus']:
componentStatuses = []
for componentStatus in queueResult['componentStatus']:
if componentStatus['reportResult']:
del componentStatus['reportResult']
componentStatuses.append(componentStatus)
else:
commandResult["healthStatus"] = componentStatus["status"]
pass
if len(componentStatuses) > 0:
heartbeat['componentStatus'] = componentStatuses
self.agentToggleLogger.log("Sending heartbeat with response id: " +
str(id) + " and "
"timestamp: " + str(timestamp) +
". Command(s) in progress: " +
repr(commandsInProgress) +
". Components mapped: " +
repr(componentsMapped))
logger.debug("Heartbeat : " + pformat(heartbeat))
return heartbeat
def writeHostnames(outputFile):
fqdn = hostname.hostname()
public_fqdn = hostname.public_hostname()
outputFile.write('$myhostname' + " = '" + fqdn + "'" + os.linesep)
outputFile.write('$public_hostname' + " = '" + public_fqdn + "'" +
os.linesep)
def public_fqdn(self): hostname.public_hostname(self.config)