def setUp(self):
# disable stdout
out = StringIO.StringIO()
sys.stdout = out
# generate sample config
tmpdir = tempfile.gettempdir()
self.agentToggleLogger = AgentToggleLogger("info")
def __init__(self, config, controller, agentToggleLogger):
super(ActionQueue, self).__init__()
self.queueOutAgentToggleLogger = agentToggleLogger
self.queueInAgentToggleLogger = AgentToggleLogger("info")
self.commandQueue = Queue.Queue()
self.commandStatuses = CommandStatusDict(
callback_action=self.status_update_callback)
self.config = config
self.controller = controller
self._stop = threading.Event()
self.tmpdir = config.getResolvedPath(AgentConfig.APP_TASK_DIR)
self.componentPackage = ''
self.customServiceOrchestrator = CustomServiceOrchestrator(
config, controller, self.queueOutAgentToggleLogger)
self.dockerManager = DockerManager(self.tmpdir,
config.getWorkRootPath(),
config.getLogPath(),
self.customServiceOrchestrator)
self.yarnDockerManager = YarnDockerManager(
self.tmpdir, config.getWorkRootPath(), config.getLogPath(),
self.customServiceOrchestrator)
def __init__(self, config, controller, agentToggleLogger):
super(ActionQueue, self).__init__()
self.queueOutAgentToggleLogger = agentToggleLogger
self.queueInAgentToggleLogger = AgentToggleLogger("info")
self.commandQueue = Queue.Queue()
self.commandStatuses = CommandStatusDict(callback_action=
self.status_update_callback)
self.config = config
self.controller = controller
self._stop = threading.Event()
self.tmpdir = config.getResolvedPath(AgentConfig.APP_TASK_DIR)
self.customServiceOrchestrator = CustomServiceOrchestrator(config,
controller,
self.queueOutAgentToggleLogger)
def setUp(self, hostname_method, NetUtil_mock, lockMock, threadMock):
#Controller.logger = MagicMock()
lockMock.return_value = MagicMock()
NetUtil_mock.return_value = MagicMock()
hostname_method.return_value = "test_hostname"
config = MagicMock()
config.get.return_value = "something"
config.getResolvedPath.return_value = "something"
self.controller = Controller.Controller(config)
self.controller.netutil.MINIMUM_INTERVAL_BETWEEN_HEARTBEATS = 0.1
self.controller.netutil.HEARTBEAT_NOT_IDDLE_INTERVAL_SEC = 0.1
self.agentToggleLogger = AgentToggleLogger("info")
self.controller.actionQueue = ActionQueue.ActionQueue(config, self.controller, self.agentToggleLogger)
def setUp(self):
self.agentToggleLogger = AgentToggleLogger("info")
class TestAgentToggleLogger(unittest.TestCase):
def setUp(self):
self.agentToggleLogger = AgentToggleLogger("info")
def test_adjustLogLevel(self):
from ActionQueue import ActionQueue
# log level is set to info here (during initialization)
# run an execution command first
self.agentToggleLogger.adjustLogLevelAtStart(ActionQueue.EXECUTION_COMMAND)
assert self.agentToggleLogger.logLevel == "info"
self.agentToggleLogger.adjustLogLevelAtEnd(ActionQueue.EXECUTION_COMMAND)
assert self.agentToggleLogger.logLevel == "info"
# run a status command now
self.agentToggleLogger.adjustLogLevelAtStart(ActionQueue.STATUS_COMMAND)
assert self.agentToggleLogger.logLevel == "info"
self.agentToggleLogger.adjustLogLevelAtEnd(ActionQueue.STATUS_COMMAND)
assert self.agentToggleLogger.logLevel == "debug"
# run a status command again
self.agentToggleLogger.adjustLogLevelAtStart(ActionQueue.STATUS_COMMAND)
assert self.agentToggleLogger.logLevel == "debug"
self.agentToggleLogger.adjustLogLevelAtEnd(ActionQueue.STATUS_COMMAND)
assert self.agentToggleLogger.logLevel == "debug"
# now an execution command shows up
self.agentToggleLogger.adjustLogLevelAtStart(ActionQueue.EXECUTION_COMMAND)
assert self.agentToggleLogger.logLevel == "info"
self.agentToggleLogger.adjustLogLevelAtEnd(ActionQueue.EXECUTION_COMMAND)
assert self.agentToggleLogger.logLevel == "info"
class ActionQueue(threading.Thread):
""" Action Queue for the agent. We pick one command at a time from the queue
and execute it
"""
STATUS_COMMAND = 'STATUS_COMMAND'
EXECUTION_COMMAND = 'EXECUTION_COMMAND'
IN_PROGRESS_STATUS = 'IN_PROGRESS'
COMPLETED_STATUS = 'COMPLETED'
FAILED_STATUS = 'FAILED'
STORE_APPLIED_CONFIG = 'record_config'
AUTO_RESTART = 'auto_restart'
def __init__(self, config, controller, agentToggleLogger):
super(ActionQueue, self).__init__()
self.queueOutAgentToggleLogger = agentToggleLogger
self.queueInAgentToggleLogger = AgentToggleLogger("info")
self.commandQueue = Queue.Queue()
self.commandStatuses = CommandStatusDict(callback_action=
self.status_update_callback)
self.config = config
self.controller = controller
self._stop = threading.Event()
self.tmpdir = config.getResolvedPath(AgentConfig.APP_TASK_DIR)
self.customServiceOrchestrator = CustomServiceOrchestrator(config,
controller,
self.queueOutAgentToggleLogger)
def stop(self):
self._stop.set()
def stopped(self):
return self._stop.isSet()
def put(self, commands):
for command in commands:
self.queueInAgentToggleLogger.adjustLogLevelAtStart(command['commandType'])
message = "Adding " + command['commandType'] + " for service " + \
command['serviceName'] + " of cluster " + \
command['clusterName'] + " to the queue."
self.queueInAgentToggleLogger.log(message)
self.queueInAgentToggleLogger.adjustLogLevelAtEnd(command['commandType'])
logger.debug(pprint.pformat(command))
self.commandQueue.put(command)
def empty(self):
return self.commandQueue.empty()
def run(self):
while not self.stopped():
time.sleep(2)
command = self.commandQueue.get() # Will block if queue is empty
self.queueOutAgentToggleLogger.adjustLogLevelAtStart(command['commandType'])
self.process_command(command)
self.queueOutAgentToggleLogger.adjustLogLevelAtEnd(command['commandType'])
logger.info("ActionQueue stopped.")
def process_command(self, command):
logger.debug("Took an element of Queue: " + pprint.pformat(command))
# make sure we log failures
try:
if command['commandType'] == self.EXECUTION_COMMAND:
self.execute_command(command)
elif command['commandType'] == self.STATUS_COMMAND:
self.execute_status_command(command)
else:
logger.error("Unrecognized command " + pprint.pformat(command))
except Exception, err:
# Should not happen
traceback.print_exc()
logger.warn(err)
def setUp(self):
# disable stdout
out = StringIO.StringIO()
sys.stdout = out
self.agentToggleLogger = AgentToggleLogger("info")
def setUp(self):
out = StringIO.StringIO()
sys.stdout = out
# save original open() method for later use
self.original_open = open
self.agentToggleLogger = AgentToggleLogger("info")
def setUp(self):
self.agentToggleLogger = AgentToggleLogger("info")
class TestAgentToggleLogger(unittest.TestCase):
def setUp(self):
self.agentToggleLogger = AgentToggleLogger("info")
def test_adjustLogLevel(self):
from ActionQueue import ActionQueue
# log level is set to info here (during initialization)
# run an execution command first
self.agentToggleLogger.adjustLogLevelAtStart(
ActionQueue.EXECUTION_COMMAND)
assert self.agentToggleLogger.logLevel == "info"
self.agentToggleLogger.adjustLogLevelAtEnd(
ActionQueue.EXECUTION_COMMAND)
assert self.agentToggleLogger.logLevel == "info"
# run a status command now
self.agentToggleLogger.adjustLogLevelAtStart(
ActionQueue.STATUS_COMMAND)
assert self.agentToggleLogger.logLevel == "info"
self.agentToggleLogger.adjustLogLevelAtEnd(ActionQueue.STATUS_COMMAND)
assert self.agentToggleLogger.logLevel == "debug"
# run a status command again
self.agentToggleLogger.adjustLogLevelAtStart(
ActionQueue.STATUS_COMMAND)
assert self.agentToggleLogger.logLevel == "debug"
self.agentToggleLogger.adjustLogLevelAtEnd(ActionQueue.STATUS_COMMAND)
assert self.agentToggleLogger.logLevel == "debug"
# now an execution command shows up
self.agentToggleLogger.adjustLogLevelAtStart(
ActionQueue.EXECUTION_COMMAND)
assert self.agentToggleLogger.logLevel == "info"
self.agentToggleLogger.adjustLogLevelAtEnd(
ActionQueue.EXECUTION_COMMAND)
assert self.agentToggleLogger.logLevel == "info"
class ActionQueue(threading.Thread):
""" Action Queue for the agent. We pick one command at a time from the queue
and execute it
"""
STATUS_COMMAND = 'STATUS_COMMAND'
EXECUTION_COMMAND = 'EXECUTION_COMMAND'
IN_PROGRESS_STATUS = 'IN_PROGRESS'
COMPLETED_STATUS = 'COMPLETED'
FAILED_STATUS = 'FAILED'
STORE_APPLIED_CONFIG = 'record_config'
AUTO_RESTART = 'auto_restart'
docker_mode = False
yarn_docker_mode = False
def __init__(self, config, controller, agentToggleLogger):
super(ActionQueue, self).__init__()
self.queueOutAgentToggleLogger = agentToggleLogger
self.queueInAgentToggleLogger = AgentToggleLogger("info")
self.commandQueue = Queue.Queue()
self.commandStatuses = CommandStatusDict(
callback_action=self.status_update_callback)
self.config = config
self.controller = controller
self._stop = threading.Event()
self.tmpdir = config.getResolvedPath(AgentConfig.APP_TASK_DIR)
self.componentPackage = ''
self.customServiceOrchestrator = CustomServiceOrchestrator(
config, controller, self.queueOutAgentToggleLogger)
self.dockerManager = DockerManager(self.tmpdir,
config.getWorkRootPath(),
config.getLogPath(),
self.customServiceOrchestrator)
self.yarnDockerManager = YarnDockerManager(
self.tmpdir, config.getWorkRootPath(), config.getLogPath(),
self.customServiceOrchestrator)
def stop(self):
self._stop.set()
def stopped(self):
return self._stop.isSet()
def put(self, commands):
for command in commands:
self.queueInAgentToggleLogger.adjustLogLevelAtStart(
command['commandType'])
message = "Adding " + command['commandType'] + " for service " + \
command['serviceName'] + " of cluster " + \
command['clusterName'] + " to the queue."
self.queueInAgentToggleLogger.log(message)
self.queueInAgentToggleLogger.adjustLogLevelAtEnd(
command['commandType'])
logger.debug(pprint.pformat(command))
self.commandQueue.put(command)
def empty(self):
return self.commandQueue.empty()
def run(self):
while not self.stopped():
time.sleep(2)
command = self.commandQueue.get() # Will block if queue is empty
self.queueOutAgentToggleLogger.adjustLogLevelAtStart(
command['commandType'])
self.process_command(command)
self.queueOutAgentToggleLogger.adjustLogLevelAtEnd(
command['commandType'])
logger.info("ActionQueue stopped.")
def process_command(self, command):
logger.debug("Took an element of Queue: " + pprint.pformat(command))
# make sure we log failures
try:
if command['commandType'] == self.EXECUTION_COMMAND:
self.execute_command(command)
elif command['commandType'] == self.STATUS_COMMAND:
self.execute_status_command(command)
else:
logger.error("Unrecognized command " + pprint.pformat(command))
except Exception, err:
# Should not happen
traceback.print_exc()
logger.warn(err)
