class HeronExecutorTest(unittest.TestCase):
"""Unittest for Heron Executor"""
def get_expected_shell_command(container_id):
return 'heron_shell_binary --port=shell-port ' \
'--log_file_prefix=log-files/heron-shell-%s.log ' \
'--secret=topid' % container_id
def build_packing_plan(self, instance_distribution):
packing_plan = PackingPlan()
for container_id in list(instance_distribution.keys()):
container_plan = packing_plan.container_plans.add()
container_plan.id = int(container_id)
for (component_name, global_task_id, component_index) in instance_distribution[container_id]:
instance_plan = container_plan.instance_plans.add()
instance_plan.component_name = component_name
instance_plan.task_id = int(global_task_id)
instance_plan.component_index = int(component_index)
return packing_plan
# pylint: disable=no-self-argument
def get_expected_metricsmgr_command(container_id):
return "heron_java_home/bin/java -Xmx1024M -XX:+PrintCommandLineFlags " \
"-Djava.net.preferIPv4Stack=true -verbosegc " \
"-XX:+UseConcMarkSweepGC -XX:+CMSScavengeBeforeRemark -XX:TargetSurvivorRatio=90 " \
"-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps " \
"-XX:+PrintGCCause -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 " \
"-XX:GCLogFileSize=100M -XX:+PrintPromotionFailure -XX:+PrintTenuringDistribution " \
"-XX:+PrintHeapAtGC -XX:+HeapDumpOnOutOfMemoryError -XX:ParallelGCThreads=4 " \
"-Xloggc:log-files/gc.metricsmgr-%d.log " \
"-cp metricsmgr_classpath org.apache.heron.metricsmgr.MetricsManager " \
"--id=metricsmgr-%d --port=metricsmgr_port " \
"--topology=topname --cluster=cluster --role=role --environment=environ " \
"--topology-id=topid " \
"--system-config-file=%s --override-config-file=%s " \
"--sink-config-file=metrics_sinks_config_file" %\
(container_id, container_id, INTERNAL_CONF_PATH, OVERRIDE_PATH)
def get_expected_metricscachemgr_command():
return "heron_java_home/bin/java -Xmx1024M -XX:+PrintCommandLineFlags " \
"-Djava.net.preferIPv4Stack=true -verbosegc " \
"-XX:+UseConcMarkSweepGC -XX:+CMSScavengeBeforeRemark -XX:TargetSurvivorRatio=90 " \
"-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps " \
"-XX:+PrintGCCause -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 " \
"-XX:GCLogFileSize=100M -XX:+PrintPromotionFailure -XX:+PrintTenuringDistribution " \
"-XX:+PrintHeapAtGC -XX:+HeapDumpOnOutOfMemoryError -XX:ParallelGCThreads=4 " \
"-Xloggc:log-files/gc.metricscache.log " \
"-cp metricscachemgr_classpath org.apache.heron.metricscachemgr.MetricsCacheManager " \
"--metricscache_id metricscache-0 --master_port metricscachemgr_masterport " \
"--stats_port metricscachemgr_statsport --topology_name topname --topology_id topid " \
"--system_config_file %s --override_config_file %s " \
"--sink_config_file metrics_sinks_config_file " \
"--cluster cluster --role role --environment environ" %\
(INTERNAL_CONF_PATH, OVERRIDE_PATH)
def get_expected_healthmgr_command():
return "heron_java_home/bin/java -Xmx1024M -XX:+PrintCommandLineFlags " \
"-Djava.net.preferIPv4Stack=true -verbosegc " \
"-XX:+UseConcMarkSweepGC -XX:+CMSScavengeBeforeRemark -XX:TargetSurvivorRatio=90 " \
"-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps " \
"-XX:+PrintGCCause -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 " \
"-XX:GCLogFileSize=100M -XX:+PrintPromotionFailure -XX:+PrintTenuringDistribution " \
"-XX:+PrintHeapAtGC -XX:+HeapDumpOnOutOfMemoryError -XX:ParallelGCThreads=4 " \
"-Xloggc:log-files/gc.healthmgr.log " \
"-cp scheduler_classpath:healthmgr_classpath " \
"org.apache.heron.healthmgr.HealthManager --cluster cluster --role role " \
"--environment environ --topology_name topname --metricsmgr_port metricsmgr_port"
def get_expected_instance_command(component_name, instance_id, container_id):
instance_name = "container_%d_%s_%d" % (container_id, component_name, instance_id)
return "heron_java_home/bin/java -Xmx320M -Xms320M -Xmn160M -XX:MaxMetaspaceSize=128M " \
"-XX:MetaspaceSize=128M -XX:ReservedCodeCacheSize=64M -XX:+PrintCommandLineFlags " \
"-Djava.net.preferIPv4Stack=true -verbosegc " \
"-XX:+UseConcMarkSweepGC -XX:+CMSScavengeBeforeRemark -XX:TargetSurvivorRatio=90 " \
"-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps " \
"-XX:+PrintGCCause -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 " \
"-XX:GCLogFileSize=100M -XX:+PrintPromotionFailure -XX:+PrintTenuringDistribution " \
"-XX:+PrintHeapAtGC -XX:+HeapDumpOnOutOfMemoryError -XX:ParallelGCThreads=4 " \
"-Xloggc:log-files/gc.%s.log " \
"-cp instance_classpath:classpath -XX:+HeapDumpOnOutOfMemoryError " \
"org.apache.heron.instance.HeronInstance -topology_name topname -topology_id topid " \
"-instance_id %s -component_name %s -task_id %d -component_index 0 -stmgr_id stmgr-%d " \
"-stmgr_port tmaster_controller_port -metricsmgr_port metricsmgr_port " \
"-system_config_file %s -override_config_file %s" \
% (instance_name, instance_name, component_name, instance_id,
container_id, INTERNAL_CONF_PATH, OVERRIDE_PATH)
MockPOpen.set_next_pid(37)
expected_processes_container_0 = [
ProcessInfo(MockPOpen(), 'heron-tmaster',
'tmaster_binary --topology_name=topname --topology_id=topid '
'--zkhostportlist=zknode --zkroot=zkroot --myhost=%s --master_port=master_port '
'--controller_port=tmaster_controller_port --stats_port=tmaster_stats_port '
'--config_file=%s --override_config_file=%s '
'--metrics_sinks_yaml=metrics_sinks_config_file '
'--metricsmgr_port=metricsmgr_port '
'--ckptmgr_port=ckptmgr-port' % (HOSTNAME, INTERNAL_CONF_PATH, OVERRIDE_PATH)),
ProcessInfo(MockPOpen(), 'heron-shell-0', get_expected_shell_command(0)),
ProcessInfo(MockPOpen(), 'metricsmgr-0', get_expected_metricsmgr_command(0)),
ProcessInfo(MockPOpen(), 'heron-metricscache', get_expected_metricscachemgr_command()),
ProcessInfo(MockPOpen(), 'heron-healthmgr', get_expected_healthmgr_command()),
]
MockPOpen.set_next_pid(37)
expected_processes_container_1 = [
ProcessInfo(MockPOpen(), 'stmgr-1',
'stmgr_binary --topology_name=topname --topology_id=topid '
'--topologydefn_file=topdefnfile --zkhostportlist=zknode --zkroot=zkroot '
'--stmgr_id=stmgr-1 '
'--instance_ids=container_1_word_3,container_1_exclaim1_2,container_1_exclaim1_1 '
'--myhost=%s --data_port=master_port '
'--local_data_port=tmaster_controller_port --metricsmgr_port=metricsmgr_port '
'--shell_port=shell-port --config_file=%s --override_config_file=%s '
'--ckptmgr_port=ckptmgr-port --ckptmgr_id=ckptmgr-1 '
'--metricscachemgr_mode=cluster'
% (HOSTNAME, INTERNAL_CONF_PATH, OVERRIDE_PATH)),
ProcessInfo(MockPOpen(), 'container_1_word_3', get_expected_instance_command('word', 3, 1)),
ProcessInfo(MockPOpen(), 'container_1_exclaim1_1',
get_expected_instance_command('exclaim1', 1, 1)),
ProcessInfo(MockPOpen(), 'container_1_exclaim1_2',
get_expected_instance_command('exclaim1', 2, 1)),
ProcessInfo(MockPOpen(), 'heron-shell-1', get_expected_shell_command(1)),
ProcessInfo(MockPOpen(), 'metricsmgr-1', get_expected_metricsmgr_command(1)),
]
MockPOpen.set_next_pid(37)
expected_processes_container_7 = [
ProcessInfo(MockPOpen(), 'container_7_word_11', get_expected_instance_command('word', 11, 7)),
ProcessInfo(MockPOpen(), 'container_7_exclaim1_210',
get_expected_instance_command('exclaim1', 210, 7)),
ProcessInfo(MockPOpen(), 'stmgr-7',
'stmgr_binary --topology_name=topname --topology_id=topid '
'--topologydefn_file=topdefnfile --zkhostportlist=zknode --zkroot=zkroot '
'--stmgr_id=stmgr-7 '
'--instance_ids=container_7_word_11,container_7_exclaim1_210 --myhost=%s '
'--data_port=master_port '
'--local_data_port=tmaster_controller_port --metricsmgr_port=metricsmgr_port '
'--shell_port=shell-port --config_file=%s --override_config_file=%s '
'--ckptmgr_port=ckptmgr-port --ckptmgr_id=ckptmgr-7 '
'--metricscachemgr_mode=cluster'
% (HOSTNAME, INTERNAL_CONF_PATH, OVERRIDE_PATH)),
ProcessInfo(MockPOpen(), 'metricsmgr-7', get_expected_metricsmgr_command(7)),
ProcessInfo(MockPOpen(), 'heron-shell-7', get_expected_shell_command(7)),
]
def setUp(self):
MockPOpen.set_next_pid(37)
self.maxDiff = None
self.executor_0 = MockExecutor(self.get_args(0))
self.executor_1 = MockExecutor(self.get_args(1))
self.executor_7 = MockExecutor(self.get_args(7))
self.packing_plan_expected = self.build_packing_plan({
1:[('word', '3', '0'), ('exclaim1', '2', '0'), ('exclaim1', '1', '0')],
7:[('word', '11', '0'), ('exclaim1', '210', '0')],
})
# ./heron-executor <shardid> <topname> <topid> <topdefnfile>
# <zknode> <zkroot> <tmaster_binary> <stmgr_binary>
# <metricsmgr_classpath> <instance_jvm_opts_in_base64> <classpath>
# <master_port> <tmaster_controller_port> <tmaster_stats_port> <heron_internals_config_file>
# <override_config_file> <component_rammap> <component_jvm_opts_in_base64> <pkg_type>
# <topology_bin_file> <heron_java_home> <shell-port> <heron_shell_binary> <metricsmgr_port>
# <cluster> <role> <environ> <instance_classpath> <metrics_sinks_config_file>
# <scheduler_classpath> <scheduler_port> <python_instance_binary>
@staticmethod
def get_args(shard_id):
executor_args = [
("--shard", shard_id),
("--topology-name", "topname"),
("--topology-id", "topid"),
("--topology-defn-file", "topdefnfile"),
("--state-manager-connection", "zknode"),
("--state-manager-root", "zkroot"),
("--state-manager-config-file", "state_manager_config_file"),
("--tmaster-binary", "tmaster_binary"),
("--stmgr-binary", "stmgr_binary"),
("--metrics-manager-classpath", "metricsmgr_classpath"),
("--instance-jvm-opts", "LVhYOitIZWFwRHVtcE9uT3V0T2ZNZW1vcnlFcnJvcg(61)(61)"),
("--classpath", "classpath"),
("--master-port", "master_port"),
("--tmaster-controller-port", "tmaster_controller_port"),
("--tmaster-stats-port", "tmaster_stats_port"),
("--heron-internals-config-file", INTERNAL_CONF_PATH),
("--override-config-file", OVERRIDE_PATH),
("--component-ram-map", "exclaim1:536870912,word:536870912"),
("--component-jvm-opts", ""),
("--pkg-type", "jar"),
("--topology-binary-file", "topology_bin_file"),
("--heron-java-home", "heron_java_home"),
("--shell-port", "shell-port"),
("--heron-shell-binary", "heron_shell_binary"),
("--metrics-manager-port", "metricsmgr_port"),
("--cluster", "cluster"),
("--role", "role"),
("--environment", "environ"),
("--instance-classpath", "instance_classpath"),
("--metrics-sinks-config-file", "metrics_sinks_config_file"),
("--scheduler-classpath", "scheduler_classpath"),
("--scheduler-port", "scheduler_port"),
("--python-instance-binary", "python_instance_binary"),
("--cpp-instance-binary", "cpp_instance_binary"),
("--metricscache-manager-classpath", "metricscachemgr_classpath"),
("--metricscache-manager-master-port", "metricscachemgr_masterport"),
("--metricscache-manager-stats-port", "metricscachemgr_statsport"),
("--is-stateful", "is_stateful_enabled"),
("--checkpoint-manager-classpath", "ckptmgr_classpath"),
("--checkpoint-manager-port", "ckptmgr-port"),
("--checkpoint-manager-ram", "1073741824"),
("--stateful-config-file", "stateful_config_file"),
("--health-manager-mode", "cluster"),
("--health-manager-classpath", "healthmgr_classpath"),
("--metricscache-manager-mode", "cluster")
]
args = ("%s=%s" % (arg[0], (str(arg[1]))) for arg in executor_args)
command = "./heron-executor %s" % (" ".join(args))
return command.split()
def test_update_packing_plan(self):
self.executor_0.update_packing_plan(self.packing_plan_expected)
self.assertEqual(self.packing_plan_expected, self.executor_0.packing_plan)
self.assertEqual({1: "stmgr-1", 7: "stmgr-7"}, self.executor_0.stmgr_ids)
self.assertEqual(
{0: "metricsmgr-0", 1: "metricsmgr-1", 7: "metricsmgr-7"}, self.executor_0.metricsmgr_ids)
self.assertEqual(
{0: "heron-shell-0", 1: "heron-shell-1", 7: "heron-shell-7"}, self.executor_0.heron_shell_ids)
def test_launch_container_0(self):
self.do_test_launch(self.executor_0, self.expected_processes_container_0)
def test_launch_container_1(self):
self.do_test_launch(self.executor_1, self.expected_processes_container_1)
def test_launch_container_7(self):
self.do_test_launch(self.executor_7, self.expected_processes_container_7)
def do_test_launch(self, executor, expected_processes):
executor.update_packing_plan(self.packing_plan_expected)
executor.launch()
monitored_processes = executor.processes_to_monitor
# convert to (pid, name, command)
found_processes = list([(process_info.pid, process_info.name, process_info.command_str) for process_info in executor.processes])
found_monitored = list([(pinfo[0], pinfo[1].name, pinfo[1].command_str) for pinfo in list(monitored_processes.items())])
found_processes.sort(key=lambda tuple: tuple[0])
found_monitored.sort(key=lambda tuple: tuple[0])
print("do_test_commands - found_processes: %s found_monitored: %s" \
% (found_processes, found_monitored))
self.assertEqual(found_processes, found_monitored)
print("do_test_commands - expected_processes: %s monitored_processes: %s" \
% (expected_processes, monitored_processes))
self.assert_processes(expected_processes, monitored_processes)
def test_change_instance_dist_container_1(self):
MockPOpen.set_next_pid(37)
self.executor_1.update_packing_plan(self.packing_plan_expected)
current_commands = self.executor_1.get_commands_to_run()
temp_dict = dict(
list(map((lambda process_info: (process_info.name, process_info.command.split(' '))),
self.expected_processes_container_1)))
current_json = json.dumps(current_commands, sort_keys=True, cls=CommandEncoder).split(' ')
temp_json = json.dumps(temp_dict, sort_keys=True).split(' ')
print("current_json: %s" % current_json)
print("temp_json: %s" % temp_json)
# better test error report
for (s1, s2) in zip(current_json, temp_json):
self.assertEqual(s1, s2)
# update instance distribution
new_packing_plan = self.build_packing_plan(
{1:[('word', '3', '0'), ('word', '2', '0'), ('exclaim1', '1', '0')]})
self.executor_1.update_packing_plan(new_packing_plan)
updated_commands = self.executor_1.get_commands_to_run()
# get the commands to kill, keep and start and verify
commands_to_kill, commands_to_keep, commands_to_start = \
self.executor_1.get_command_changes(current_commands, updated_commands)
self.assertEqual(['container_1_exclaim1_2', 'stmgr-1'], sorted(commands_to_kill.keys()))
self.assertEqual(
['container_1_exclaim1_1', 'container_1_word_3', 'heron-shell-1', 'metricsmgr-1'],
sorted(commands_to_keep.keys()))
self.assertEqual(['container_1_word_2', 'stmgr-1'], sorted(commands_to_start.keys()))
def assert_processes(self, expected_processes, found_processes):
self.assertEqual(len(expected_processes), len(found_processes))
for expected_process in expected_processes:
self.assert_process(expected_process, found_processes)
def assert_process(self, expected_process, found_processes):
pid = expected_process.pid
self.assertTrue(found_processes[pid])
self.assertEqual(expected_process.name, found_processes[pid].name)
self.assertEqual(expected_process.command, found_processes[pid].command_str)
self.assertEqual(1, found_processes[pid].attempts)
def _run_process(self, name, cmd, env=None): popen = MockPOpen() self.processes.append(ProcessInfo(popen, name, cmd)) return popen
class HeronExecutorTest(unittest.TestCase):
"""Unittest for Heron Executor"""
dist_expected = {
1: [('word', '3', '0'), ('exclaim1', '2', '0'), ('exclaim1', '1', '0')]
}
shell_command_expected = 'heron_shell_binary --port=shell-port ' \
'--log_file_prefix=fake_dir/heron-shell.log'
# pylint: disable=no-self-argument
def get_expected_metricsmgr_command(container_id):
return "heron_java_home/bin/java -Xmx1024M -XX:+PrintCommandLineFlags -verbosegc " \
"-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:+PrintGCCause " \
"-XX:+PrintPromotionFailure -XX:+PrintTenuringDistribution -XX:+PrintHeapAtGC " \
"-XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC -XX:+PrintCommandLineFlags " \
"-Xloggc:log-files/gc.metricsmgr.log -Djava.net.preferIPv4Stack=true " \
"-cp metricsmgr_classpath com.twitter.heron.metricsmgr.MetricsManager metricsmgr-%d " \
"metricsmgr_port topname topid %s " \
"metrics_sinks_config_file" % (container_id, INTERNAL_CONF_PATH)
def get_expected_instance_command(component_name,
instance_id,
container_id=1):
instance_name = "container_%d_%s_%d" % (container_id, component_name,
instance_id)
return "heron_java_home/bin/java -Xmx320M -Xms320M -Xmn160M -XX:MaxPermSize=128M " \
"-XX:PermSize=128M -XX:ReservedCodeCacheSize=64M -XX:+CMSScavengeBeforeRemark " \
"-XX:TargetSurvivorRatio=90 -XX:+PrintCommandLineFlags -verbosegc -XX:+PrintGCDetails " \
"-XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:+PrintGCCause " \
"-XX:+PrintPromotionFailure -XX:+PrintTenuringDistribution -XX:+PrintHeapAtGC " \
"-XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC -XX:ParallelGCThreads=4 " \
"-Xloggc:log-files/gc.%s.log -XX:+HeapDumpOnOutOfMemoryError " \
"-Djava.net.preferIPv4Stack=true -cp instance_classpath:classpath " \
"com.twitter.heron.instance.HeronInstance topname topid %s %s %d 0 stmgr-%d " \
"master_port metricsmgr_port %s" \
% (instance_name, instance_name, component_name, instance_id,
container_id, INTERNAL_CONF_PATH)
MockPOpen.set_next_pid(37)
expected_processes_container_0 = [
ProcessInfo(MockPOpen(), 'heron-shell-0', shell_command_expected),
ProcessInfo(MockPOpen(), 'metricsmgr-0',
get_expected_metricsmgr_command(0)),
ProcessInfo(
MockPOpen(), 'heron-tmaster', 'tmaster_binary master_port '
'tmaster_controller_port tmaster_stats_port '
'topname topid zknode zkroot stmgr-1 '
'%s metrics_sinks_config_file metricsmgr_port' %
INTERNAL_CONF_PATH),
]
MockPOpen.set_next_pid(37)
expected_processes_container_1 = [
ProcessInfo(
MockPOpen(), 'stmgr-1',
'stmgr_binary topname topid topdefnfile zknode zkroot stmgr-1 '
'container_1_word_3,container_1_exclaim1_2,container_1_exclaim1_1 master_port '
'metricsmgr_port shell-port %s' % INTERNAL_CONF_PATH),
ProcessInfo(MockPOpen(), 'container_1_word_3',
get_expected_instance_command('word', 3)),
ProcessInfo(MockPOpen(), 'container_1_exclaim1_1',
get_expected_instance_command('exclaim1', 1)),
ProcessInfo(MockPOpen(), 'container_1_exclaim1_2',
get_expected_instance_command('exclaim1', 2)),
ProcessInfo(MockPOpen(), 'heron-shell-1', shell_command_expected),
ProcessInfo(MockPOpen(), 'metricsmgr-1',
get_expected_metricsmgr_command(1)),
]
def setUp(self):
MockPOpen.set_next_pid(37)
self.maxDiff = None
self.executor_0 = MockExecutor(self.get_args(0))
self.executor_1 = MockExecutor(self.get_args(1))
# ./heron-executor <shardid> <topname> <topid> <topdefnfile>
# <instance_distribution> <zknode> <zkroot> <tmaster_binary> <stmgr_binary>
# <metricsmgr_classpath> <instance_jvm_opts_in_base64> <classpath>
# <master_port> <tmaster_controller_port> <tmaster_stats_port> <heron_internals_config_file>
# <component_rammap> <component_jvm_opts_in_base64> <pkg_type> <topology_bin_file>
# <heron_java_home> <shell-port> <heron_shell_binary> <metricsmgr_port>
# <cluster> <role> <environ> <instance_classpath> <metrics_sinks_config_file>
# <scheduler_classpath> <scheduler_port> <python_instance_binary>
@staticmethod
def get_args(shard_id):
return ("""
./heron-executor %d topname topid topdefnfile
1:word:3:0:exclaim1:2:0:exclaim1:1:0 zknode zkroot tmaster_binary stmgr_binary
metricsmgr_classpath "LVhYOitIZWFwRHVtcE9uT3V0T2ZNZW1vcnlFcnJvcg==" classpath
master_port tmaster_controller_port tmaster_stats_port
%s exclaim1:536870912,word:536870912 "" jar topology_bin_file
heron_java_home shell-port heron_shell_binary metricsmgr_port
cluster role environ instance_classpath metrics_sinks_config_file
scheduler_classpath scheduler_port python_instance_binary
""" % (shard_id, INTERNAL_CONF_PATH)).replace("\n", '').split()
def test_parse_instance_distribution(self):
dist_found = self.executor_0.parse_instance_distribution(
"1:word:3:0:exclaim1:2:0:exclaim1:1:0")
self.assertEquals(self.dist_expected, dist_found)
def test_update_instance_distribution(self):
self.executor_0.update_instance_distribution(self.dist_expected)
self.assertEquals(self.dist_expected,
self.executor_0.instance_distribution)
self.assertEquals(["stmgr-1"], self.executor_0.stmgr_ids)
self.assertEquals(["metricsmgr-0", "metricsmgr-1"],
self.executor_0.metricsmgr_ids)
self.assertEquals(["heron-shell-0", "heron-shell-1"],
self.executor_0.heron_shell_ids)
def test_launch_container_0(self):
self.do_test_launch(self.executor_0,
self.expected_processes_container_0)
def test_launch_container_1(self):
self.do_test_launch(self.executor_1,
self.expected_processes_container_1)
def do_test_launch(self, executor, expected_processes):
executor.update_instance_distribution(self.dist_expected)
executor.launch()
monitored_processes = executor.processes_to_monitor
# convert to (pid, name, command)
found_processes = map(
lambda (process_info):
(process_info.pid, process_info.name, process_info.command_str),
executor.processes)
found_monitored = map(
lambda (pid, process_info):
(pid, process_info.name, process_info.command_str),
monitored_processes.items())
print "do_test_commands - found_processes: %s found_monitored: %s" \
% (found_processes, found_monitored)
self.assertEquals(found_processes, found_monitored)
print "do_test_commands - expected_processes: %s monitored_processes: %s" \
% (expected_processes, monitored_processes)
self.assert_processes(expected_processes, monitored_processes)
def test_change_instance_dist_container_1(self):
MockPOpen.set_next_pid(37)
self.executor_1.update_instance_distribution(self.dist_expected)
current_commands = self.executor_1.get_commands_to_run()
self.assertEquals(
dict(
map((lambda (process_info):
(process_info.name, process_info.command.split(' '))),
self.expected_processes_container_1)), current_commands)
# update instance distribution
new_distribution = \
self.executor_1.parse_instance_distribution("1:word:3:0:word:2:0:exclaim1:1:0")
self.executor_1.update_instance_distribution(new_distribution)
updated_commands = self.executor_1.get_commands_to_run()
# get the commands to kill, keep and start and verify
commands_to_kill, commands_to_keep, commands_to_start = \
self.executor_1.get_command_changes(current_commands, updated_commands)
self.assertEquals(['container_1_exclaim1_2', 'stmgr-1'],
sorted(commands_to_kill.keys()))
self.assertEquals([
'container_1_exclaim1_1', 'container_1_word_3', 'heron-shell-1',
'metricsmgr-1'
], sorted(commands_to_keep.keys()))
self.assertEquals(['container_1_word_2', 'stmgr-1'],
sorted(commands_to_start.keys()))
def assert_processes(self, expected_processes, found_processes):
self.assertEquals(len(expected_processes), len(found_processes))
for expected_process in expected_processes:
self.assert_process(expected_process, found_processes)
def assert_process(self, expected_process, found_processes):
pid = expected_process.pid
self.assertTrue(found_processes[pid])
self.assertEquals(expected_process.name, found_processes[pid].name)
self.assertEquals(expected_process.command,
found_processes[pid].command_str)
self.assertEquals(1, found_processes[pid].attempts)
class HeronExecutorTest(unittest.TestCase):
"""Unittest for Heron Executor"""
def get_expected_shell_command(container_id):
return 'heron_shell_binary --port=shell-port ' \
'--log_file_prefix=fake_dir/heron-shell-%s.log ' \
'--secret=topid' % container_id
def build_packing_plan(self, instance_distribution):
packing_plan = PackingPlan()
for container_id in instance_distribution.keys():
container_plan = packing_plan.container_plans.add()
container_plan.id = int(container_id)
for (component_name, global_task_id, component_index) in instance_distribution[container_id]:
instance_plan = container_plan.instance_plans.add()
instance_plan.component_name = component_name
instance_plan.task_id = int(global_task_id)
instance_plan.component_index = int(component_index)
return packing_plan
# pylint: disable=no-self-argument
def get_expected_metricsmgr_command(container_id):
return "heron_java_home/bin/java -Xmx1024M -XX:+PrintCommandLineFlags -verbosegc " \
"-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:+PrintGCCause " \
"-XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=100M " \
"-XX:+PrintPromotionFailure -XX:+PrintTenuringDistribution -XX:+PrintHeapAtGC " \
"-XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC -XX:+PrintCommandLineFlags " \
"-Xloggc:log-files/gc.metricsmgr.log -Djava.net.preferIPv4Stack=true " \
"-cp metricsmgr_classpath com.twitter.heron.metricsmgr.MetricsManager metricsmgr-%d " \
"metricsmgr_port topname topid %s " \
"metrics_sinks_config_file" % (container_id, INTERNAL_CONF_PATH)
def get_expected_metricscachemgr_command():
return "heron_java_home/bin/java -Xmx1024M -XX:+PrintCommandLineFlags -verbosegc " \
"-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:+PrintGCCause " \
"-XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=100M " \
"-XX:+PrintPromotionFailure -XX:+PrintTenuringDistribution -XX:+PrintHeapAtGC " \
"-XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC -XX:+PrintCommandLineFlags " \
"-Xloggc:log-files/gc.metricscache.log -Djava.net.preferIPv4Stack=true " \
"-cp metricscachemgr_classpath com.twitter.heron.metricscachemgr.MetricsCacheManager " \
"--metricscache_id metricscache-0 --master_port metricscachemgr_masterport " \
"--stats_port metricscachemgr_statsport --topology_name topname --topology_id topid " \
"--system_config_file %s --sink_config_file metrics_sinks_config_file " \
"--cluster cluster --role role --environment environ --verbose" % (INTERNAL_CONF_PATH)
def get_expected_healthmgr_command():
return "heron_java_home/bin/java -Xmx1024M -XX:+PrintCommandLineFlags -verbosegc " \
"-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:+PrintGCCause " \
"-XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=100M " \
"-XX:+PrintPromotionFailure -XX:+PrintTenuringDistribution -XX:+PrintHeapAtGC " \
"-XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC -XX:+PrintCommandLineFlags " \
"-Xloggc:log-files/gc.healthmgr.log -Djava.net.preferIPv4Stack=true " \
"-cp scheduler_classpath:healthmgr_classpath " \
"com.twitter.heron.healthmgr.HealthManager --cluster cluster --role role " \
"--environment environ --topology_name topname"
def get_expected_instance_command(component_name, instance_id, container_id):
instance_name = "container_%d_%s_%d" % (container_id, component_name, instance_id)
return "heron_java_home/bin/java -Xmx320M -Xms320M -Xmn160M -XX:MaxMetaspaceSize=128M " \
"-XX:MetaspaceSize=128M -XX:ReservedCodeCacheSize=64M -XX:+CMSScavengeBeforeRemark " \
"-XX:TargetSurvivorRatio=90 -XX:+PrintCommandLineFlags -verbosegc -XX:+PrintGCDetails " \
"-XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:+PrintGCCause " \
"-XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=100M " \
"-XX:+PrintPromotionFailure -XX:+PrintTenuringDistribution -XX:+PrintHeapAtGC " \
"-XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC -XX:ParallelGCThreads=4 " \
"-Xloggc:log-files/gc.%s.log -XX:+HeapDumpOnOutOfMemoryError " \
"-Djava.net.preferIPv4Stack=true -cp instance_classpath:classpath " \
"com.twitter.heron.instance.HeronInstance topname topid %s %s %d 0 stmgr-%d " \
"master_port metricsmgr_port %s" \
% (instance_name, instance_name, component_name, instance_id,
container_id, INTERNAL_CONF_PATH)
MockPOpen.set_next_pid(37)
expected_processes_container_0 = [
ProcessInfo(MockPOpen(), 'heron-tmaster',
'tmaster_binary %s master_port '
'tmaster_controller_port tmaster_stats_port '
'topname topid zknode zkroot '
'%s metrics_sinks_config_file metricsmgr_port '
'ckptmgr-port' % (HOSTNAME, INTERNAL_CONF_PATH )),
ProcessInfo(MockPOpen(), 'heron-shell-0', get_expected_shell_command(0)),
ProcessInfo(MockPOpen(), 'metricsmgr-0', get_expected_metricsmgr_command(0)),
ProcessInfo(MockPOpen(), 'heron-metricscache', get_expected_metricscachemgr_command()),
ProcessInfo(MockPOpen(), 'heron-healthmgr', get_expected_healthmgr_command()),
]
MockPOpen.set_next_pid(37)
expected_processes_container_1 = [
ProcessInfo(MockPOpen(), 'stmgr-1',
'stmgr_binary topname topid topdefnfile zknode zkroot stmgr-1 '
'container_1_word_3,container_1_exclaim1_2,container_1_exclaim1_1 %s master_port '
'metricsmgr_port shell-port %s ckptmgr-port ckptmgr-1'
% (HOSTNAME, INTERNAL_CONF_PATH)),
ProcessInfo(MockPOpen(), 'container_1_word_3', get_expected_instance_command('word', 3, 1)),
ProcessInfo(MockPOpen(), 'container_1_exclaim1_1',
get_expected_instance_command('exclaim1', 1, 1)),
ProcessInfo(MockPOpen(), 'container_1_exclaim1_2',
get_expected_instance_command('exclaim1', 2, 1)),
ProcessInfo(MockPOpen(), 'heron-shell-1', get_expected_shell_command(1)),
ProcessInfo(MockPOpen(), 'metricsmgr-1', get_expected_metricsmgr_command(1)),
]
MockPOpen.set_next_pid(37)
expected_processes_container_7 = [
ProcessInfo(MockPOpen(), 'container_7_word_11', get_expected_instance_command('word', 11, 7)),
ProcessInfo(MockPOpen(), 'container_7_exclaim1_210',
get_expected_instance_command('exclaim1', 210, 7)),
ProcessInfo(MockPOpen(), 'stmgr-7',
'stmgr_binary topname topid topdefnfile zknode zkroot stmgr-7 '
'container_7_word_11,container_7_exclaim1_210 %s master_port '
'metricsmgr_port shell-port %s ckptmgr-port ckptmgr-7'
% (HOSTNAME, INTERNAL_CONF_PATH)),
ProcessInfo(MockPOpen(), 'metricsmgr-7', get_expected_metricsmgr_command(7)),
ProcessInfo(MockPOpen(), 'heron-shell-7', get_expected_shell_command(7)),
]
def setUp(self):
MockPOpen.set_next_pid(37)
self.maxDiff = None
self.executor_0 = MockExecutor(self.get_args(0))
self.executor_1 = MockExecutor(self.get_args(1))
self.executor_7 = MockExecutor(self.get_args(7))
self.packing_plan_expected = self.build_packing_plan({
1:[('word', '3', '0'), ('exclaim1', '2', '0'), ('exclaim1', '1', '0')],
7:[('word', '11', '0'), ('exclaim1', '210', '0')],
})
# ./heron-executor <shardid> <topname> <topid> <topdefnfile>
# <zknode> <zkroot> <tmaster_binary> <stmgr_binary>
# <metricsmgr_classpath> <instance_jvm_opts_in_base64> <classpath>
# <master_port> <tmaster_controller_port> <tmaster_stats_port> <heron_internals_config_file>
# <component_rammap> <component_jvm_opts_in_base64> <pkg_type> <topology_bin_file>
# <heron_java_home> <shell-port> <heron_shell_binary> <metricsmgr_port>
# <cluster> <role> <environ> <instance_classpath> <metrics_sinks_config_file>
# <scheduler_classpath> <scheduler_port> <python_instance_binary>
@staticmethod
def get_args(shard_id):
return ("""
./heron-executor %d topname topid topdefnfile
zknode zkroot tmaster_binary stmgr_binary
metricsmgr_classpath "LVhYOitIZWFwRHVtcE9uT3V0T2ZNZW1vcnlFcnJvcg==" classpath
master_port tmaster_controller_port tmaster_stats_port
%s exclaim1:536870912,word:536870912 "" jar topology_bin_file
heron_java_home shell-port heron_shell_binary metricsmgr_port
cluster role environ instance_classpath metrics_sinks_config_file
scheduler_classpath scheduler_port python_instance_binary
metricscachemgr_classpath metricscachemgr_masterport metricscachemgr_statsport
is_stateful_enabled ckptmgr_classpath ckptmgr-port stateful_config_file
healthmgr_mode healthmgr_classpath
""" % (shard_id, INTERNAL_CONF_PATH)).replace("\n", '').split()
def test_update_packing_plan(self):
self.executor_0.update_packing_plan(self.packing_plan_expected)
self.assertEquals(self.packing_plan_expected, self.executor_0.packing_plan)
self.assertEquals({1: "stmgr-1", 7: "stmgr-7"}, self.executor_0.stmgr_ids)
self.assertEquals(
{0: "metricsmgr-0", 1: "metricsmgr-1", 7: "metricsmgr-7"}, self.executor_0.metricsmgr_ids)
self.assertEquals(
{0: "heron-shell-0", 1: "heron-shell-1", 7: "heron-shell-7"}, self.executor_0.heron_shell_ids)
def test_launch_container_0(self):
self.do_test_launch(self.executor_0, self.expected_processes_container_0)
def test_launch_container_1(self):
self.do_test_launch(self.executor_1, self.expected_processes_container_1)
def test_launch_container_7(self):
self.do_test_launch(self.executor_7, self.expected_processes_container_7)
def do_test_launch(self, executor, expected_processes):
executor.update_packing_plan(self.packing_plan_expected)
executor.launch()
monitored_processes = executor.processes_to_monitor
# convert to (pid, name, command)
found_processes = map(lambda (process_info):
(process_info.pid, process_info.name, process_info.command_str),
executor.processes)
found_monitored = map(lambda (pid, process_info):
(pid, process_info.name, process_info.command_str),
monitored_processes.items())
found_processes.sort(key=lambda tuple: tuple[0])
found_monitored.sort(key=lambda tuple: tuple[0])
print "do_test_commands - found_processes: %s found_monitored: %s" \
% (found_processes, found_monitored)
self.assertEquals(found_processes, found_monitored)
print "do_test_commands - expected_processes: %s monitored_processes: %s" \
% (expected_processes, monitored_processes)
self.assert_processes(expected_processes, monitored_processes)
def test_change_instance_dist_container_1(self):
MockPOpen.set_next_pid(37)
self.executor_1.update_packing_plan(self.packing_plan_expected)
current_commands = self.executor_1.get_commands_to_run()
self.assertEquals(dict(
map((lambda (process_info): (process_info.name, process_info.command.split(' '))),
self.expected_processes_container_1)), current_commands)
# update instance distribution
new_packing_plan = self.build_packing_plan(
{1:[('word', '3', '0'), ('word', '2', '0'), ('exclaim1', '1', '0')]})
self.executor_1.update_packing_plan(new_packing_plan)
updated_commands = self.executor_1.get_commands_to_run()
# get the commands to kill, keep and start and verify
commands_to_kill, commands_to_keep, commands_to_start = \
self.executor_1.get_command_changes(current_commands, updated_commands)
self.assertEquals(['container_1_exclaim1_2', 'stmgr-1'], sorted(commands_to_kill.keys()))
self.assertEquals(
['container_1_exclaim1_1', 'container_1_word_3', 'heron-shell-1', 'metricsmgr-1'],
sorted(commands_to_keep.keys()))
self.assertEquals(['container_1_word_2', 'stmgr-1'], sorted(commands_to_start.keys()))
def assert_processes(self, expected_processes, found_processes):
self.assertEquals(len(expected_processes), len(found_processes))
for expected_process in expected_processes:
self.assert_process(expected_process, found_processes)
def assert_process(self, expected_process, found_processes):
pid = expected_process.pid
self.assertTrue(found_processes[pid])
self.assertEquals(expected_process.name, found_processes[pid].name)
self.assertEquals(expected_process.command, found_processes[pid].command_str)
self.assertEquals(1, found_processes[pid].attempts)
class HeronExecutorTest(unittest.TestCase):
"""Unittest for Heron Executor"""
def get_expected_shell_command(container_id):
return 'heron_shell_binary --port=shell-port ' \
'--log_file_prefix=log-files/heron-shell-%s.log ' \
'--secret=topid' % container_id
def build_packing_plan(self, instance_distribution):
packing_plan = PackingPlan()
for container_id in list(instance_distribution.keys()):
container_plan = packing_plan.container_plans.add()
container_plan.id = int(container_id)
for (component_name, global_task_id,
component_index) in instance_distribution[container_id]:
instance_plan = container_plan.instance_plans.add()
instance_plan.component_name = component_name
instance_plan.task_id = int(global_task_id)
instance_plan.component_index = int(component_index)
return packing_plan
# pylint: disable=no-self-argument
def get_expected_metricsmgr_command(container_id):
return "heron_java_home/bin/java -Xmx1024M -XX:+PrintCommandLineFlags " \
"-Djava.net.preferIPv4Stack=true " \
"-XX:+UseG1GC -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication " \
"-XX:MaxGCPauseMillis=100 -XX:InitiatingHeapOccupancyPercent=30 " \
"-XX:ParallelGCThreads=4 " \
"-cp metricsmgr_classpath org.apache.heron.metricsmgr.MetricsManager " \
f"--id=metricsmgr-{int(container_id)} --port=metricsmgr_port " \
"--topology=topname --cluster=cluster --role=role --environment=environ " \
"--topology-id=topid " \
f"--system-config-file={INTERNAL_CONF_PATH} --override-config-file={OVERRIDE_PATH} " \
"--sink-config-file=metrics_sinks_config_file"
def get_expected_metricscachemgr_command():
return "heron_java_home/bin/java -Xmx1024M -XX:+PrintCommandLineFlags " \
"-Djava.net.preferIPv4Stack=true " \
"-XX:+UseG1GC -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication " \
"-XX:MaxGCPauseMillis=100 -XX:InitiatingHeapOccupancyPercent=30 " \
"-XX:ParallelGCThreads=4 " \
"-cp metricscachemgr_classpath org.apache.heron.metricscachemgr.MetricsCacheManager " \
"--metricscache_id metricscache-0 --server_port metricscachemgr_serverport " \
"--stats_port metricscachemgr_statsport --topology_name topname --topology_id topid " \
f"--system_config_file {INTERNAL_CONF_PATH} --override_config_file {OVERRIDE_PATH} " \
"--sink_config_file metrics_sinks_config_file " \
"--cluster cluster --role role --environment environ"
def get_expected_healthmgr_command():
return "heron_java_home/bin/java -Xmx1024M -XX:+PrintCommandLineFlags " \
"-Djava.net.preferIPv4Stack=true " \
"-XX:+UseG1GC -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication " \
"-XX:MaxGCPauseMillis=100 -XX:InitiatingHeapOccupancyPercent=30 " \
"-XX:ParallelGCThreads=4 " \
"-cp scheduler_classpath:healthmgr_classpath " \
"org.apache.heron.healthmgr.HealthManager --cluster cluster --role role " \
"--environment environ --topology_name topname --metricsmgr_port metricsmgr_port"
def get_expected_instance_command(component_name, instance_id,
container_id):
instance_name = f"container_{int(container_id)}_{component_name}_{int(instance_id)}"
return "heron_java_home/bin/java -Xmx320M -Xms320M -XX:MaxMetaspaceSize=128M " \
"-XX:MetaspaceSize=128M -XX:ReservedCodeCacheSize=64M -XX:+PrintCommandLineFlags " \
"-Djava.net.preferIPv4Stack=true " \
"-XX:+UseG1GC -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication " \
"-XX:MaxGCPauseMillis=100 -XX:InitiatingHeapOccupancyPercent=30 " \
"-XX:ParallelGCThreads=4 " \
"-cp instance_classpath:classpath -XX:+HeapDumpOnOutOfMemoryError " \
"org.apache.heron.instance.HeronInstance -topology_name topname -topology_id topid " \
f"-instance_id {instance_name} -component_name {component_name} " \
f"-task_id {int(instance_id)} -component_index 0 -stmgr_id stmgr-{int(container_id)} " \
"-stmgr_port tmanager_controller_port -metricsmgr_port metricsmgr_port " \
f"-system_config_file {INTERNAL_CONF_PATH} -override_config_file {OVERRIDE_PATH}"
MockPOpen.set_next_pid(37)
expected_processes_container_0 = [
ProcessInfo(
MockPOpen(), 'heron-tmanager',
'tmanager_binary --topology_name=topname --topology_id=topid '
f'--zkhostportlist=zknode --zkroot=zkroot --myhost={HOSTNAME} --server_port=server_port '
'--controller_port=tmanager_controller_port --stats_port=tmanager_stats_port '
f'--config_file={INTERNAL_CONF_PATH} --override_config_file={OVERRIDE_PATH} '
'--metrics_sinks_yaml=metrics_sinks_config_file '
'--metricsmgr_port=metricsmgr_port --ckptmgr_port=ckptmgr-port'),
ProcessInfo(MockPOpen(), 'heron-metricscache',
get_expected_metricscachemgr_command()),
ProcessInfo(MockPOpen(), 'heron-healthmgr',
get_expected_healthmgr_command()),
ProcessInfo(MockPOpen(), 'metricsmgr-0',
get_expected_metricsmgr_command(0)),
ProcessInfo(MockPOpen(), 'heron-shell-0',
get_expected_shell_command(0)),
]
MockPOpen.set_next_pid(37)
expected_processes_container_1 = [
ProcessInfo(
MockPOpen(), 'stmgr-1',
'stmgr_binary --topology_name=topname --topology_id=topid '
'--topologydefn_file=topdefnfile --zkhostportlist=zknode --zkroot=zkroot '
'--stmgr_id=stmgr-1 '
'--instance_ids=container_1_word_3,container_1_exclaim1_2,container_1_exclaim1_1 '
f'--myhost={HOSTNAME} --data_port=server_port '
'--local_data_port=tmanager_controller_port --metricsmgr_port=metricsmgr_port '
f'--shell_port=shell-port --config_file={INTERNAL_CONF_PATH} --override_config_file={OVERRIDE_PATH} '
'--ckptmgr_port=ckptmgr-port --ckptmgr_id=ckptmgr-1 '
'--metricscachemgr_mode=cluster'),
ProcessInfo(MockPOpen(), 'metricsmgr-1',
get_expected_metricsmgr_command(1)),
ProcessInfo(MockPOpen(), 'container_1_word_3',
get_expected_instance_command('word', 3, 1)),
ProcessInfo(MockPOpen(), 'container_1_exclaim1_2',
get_expected_instance_command('exclaim1', 2, 1)),
ProcessInfo(MockPOpen(), 'container_1_exclaim1_1',
get_expected_instance_command('exclaim1', 1, 1)),
ProcessInfo(MockPOpen(), 'heron-shell-1',
get_expected_shell_command(1)),
]
MockPOpen.set_next_pid(37)
expected_processes_container_7 = [
ProcessInfo(
MockPOpen(), 'stmgr-7',
'stmgr_binary --topology_name=topname --topology_id=topid '
'--topologydefn_file=topdefnfile --zkhostportlist=zknode --zkroot=zkroot '
'--stmgr_id=stmgr-7 '
f'--instance_ids=container_7_word_11,container_7_exclaim1_210 --myhost={HOSTNAME} '
'--data_port=server_port '
'--local_data_port=tmanager_controller_port --metricsmgr_port=metricsmgr_port '
f'--shell_port=shell-port --config_file={INTERNAL_CONF_PATH} --override_config_file={OVERRIDE_PATH} '
'--ckptmgr_port=ckptmgr-port --ckptmgr_id=ckptmgr-7 '
'--metricscachemgr_mode=cluster'),
ProcessInfo(MockPOpen(), 'metricsmgr-7',
get_expected_metricsmgr_command(7)),
ProcessInfo(MockPOpen(), 'container_7_word_11',
get_expected_instance_command('word', 11, 7)),
ProcessInfo(MockPOpen(), 'container_7_exclaim1_210',
get_expected_instance_command('exclaim1', 210, 7)),
ProcessInfo(MockPOpen(), 'heron-shell-7',
get_expected_shell_command(7)),
]
def setUp(self):
MockPOpen.set_next_pid(37)
self.maxDiff = None
self.executor_0 = MockExecutor(self.get_args(0))
self.executor_1 = MockExecutor(self.get_args(1))
self.executor_7 = MockExecutor(self.get_args(7))
self.packing_plan_expected = self.build_packing_plan({
1: [('word', '3', '0'), ('exclaim1', '2', '0'),
('exclaim1', '1', '0')],
7: [('word', '11', '0'), ('exclaim1', '210', '0')],
})
# ./heron-executor <shardid> <topname> <topid> <topdefnfile>
# <zknode> <zkroot> <tmanager_binary> <stmgr_binary>
# <metricsmgr_classpath> <instance_jvm_opts_in_base64> <classpath>
# <server_port> <tmanager_controller_port> <tmanager_stats_port> <heron_internals_config_file>
# <override_config_file> <component_rammap> <component_jvm_opts_in_base64> <pkg_type>
# <topology_bin_file> <heron_java_home> <shell-port> <heron_shell_binary> <metricsmgr_port>
# <cluster> <role> <environ> <instance_classpath> <metrics_sinks_config_file>
# <scheduler_classpath> <scheduler_port> <python_instance_binary>
@staticmethod
def get_args(shard_id):
executor_args = [
("--shard", shard_id), ("--topology-name", "topname"),
("--topology-id", "topid"),
("--topology-defn-file", "topdefnfile"),
("--state-manager-connection", "zknode"),
("--state-manager-root", "zkroot"),
("--state-manager-config-file", "state_manager_config_file"),
("--tmanager-binary", "tmanager_binary"),
("--stmgr-binary", "stmgr_binary"),
("--metrics-manager-classpath", "metricsmgr_classpath"),
("--instance-jvm-opts",
"LVhYOitIZWFwRHVtcE9uT3V0T2ZNZW1vcnlFcnJvcg(61)(61)"),
("--classpath", "classpath"), ("--server-port", "server_port"),
("--tmanager-controller-port", "tmanager_controller_port"),
("--tmanager-stats-port", "tmanager_stats_port"),
("--heron-internals-config-file", INTERNAL_CONF_PATH),
("--override-config-file", OVERRIDE_PATH),
("--component-ram-map", "exclaim1:536870912,word:536870912"),
("--component-jvm-opts", ""), ("--pkg-type", "jar"),
("--topology-binary-file", "topology_bin_file"),
("--heron-java-home", "heron_java_home"),
("--shell-port", "shell-port"),
("--heron-shell-binary", "heron_shell_binary"),
("--metrics-manager-port", "metricsmgr_port"),
("--cluster", "cluster"), ("--role", "role"),
("--environment", "environ"),
("--instance-classpath", "instance_classpath"),
("--metrics-sinks-config-file", "metrics_sinks_config_file"),
("--scheduler-classpath", "scheduler_classpath"),
("--scheduler-port", "scheduler_port"),
("--python-instance-binary", "python_instance_binary"),
("--cpp-instance-binary", "cpp_instance_binary"),
("--metricscache-manager-classpath", "metricscachemgr_classpath"),
("--metricscache-manager-server-port",
"metricscachemgr_serverport"),
("--metricscache-manager-stats-port", "metricscachemgr_statsport"),
("--is-stateful", "is_stateful_enabled"),
("--checkpoint-manager-classpath", "ckptmgr_classpath"),
("--checkpoint-manager-port", "ckptmgr-port"),
("--checkpoint-manager-ram", "1073741824"),
("--stateful-config-file", "stateful_config_file"),
("--health-manager-mode", "cluster"),
("--health-manager-classpath", "healthmgr_classpath"),
("--metricscache-manager-mode", "cluster")
]
args = [f"{k}={v}" for k, v in executor_args]
ctx = cli.make_context('heron-executor', args)
return argparse.Namespace(**ctx.params)
def test_update_packing_plan(self):
self.executor_0.update_packing_plan(self.packing_plan_expected)
self.assertEqual(self.packing_plan_expected,
self.executor_0.packing_plan)
self.assertEqual({
1: "stmgr-1",
7: "stmgr-7"
}, self.executor_0.stmgr_ids)
self.assertEqual(
{
0: "metricsmgr-0",
1: "metricsmgr-1",
7: "metricsmgr-7"
}, self.executor_0.metricsmgr_ids)
self.assertEqual(
{
0: "heron-shell-0",
1: "heron-shell-1",
7: "heron-shell-7"
}, self.executor_0.heron_shell_ids)
def test_launch_container_0(self):
self.do_test_launch(self.executor_0,
self.expected_processes_container_0)
def test_launch_container_1(self):
self.do_test_launch(self.executor_1,
self.expected_processes_container_1)
def test_launch_container_7(self):
self.do_test_launch(self.executor_7,
self.expected_processes_container_7)
def do_test_launch(self, executor, expected_processes):
executor.update_packing_plan(self.packing_plan_expected)
executor.launch()
monitored_processes = executor.processes_to_monitor
# convert to (pid, name, command)
found_processes = list([(process_info.pid, process_info.name,
process_info.command_str)
for process_info in executor.processes])
found_monitored = list([(pinfo[0], pinfo[1].name, pinfo[1].command_str)
for pinfo in list(monitored_processes.items())
])
found_processes.sort(key=lambda tuple: tuple[0])
found_monitored.sort(key=lambda tuple: tuple[0])
print("found_processes:")
pprint(found_processes)
print("found_monitored:")
pprint(found_monitored)
self.assertEqual(found_processes, found_monitored)
print("expected_processes:")
pprint(expected_processes)
print("monitored_processes:")
pprint(monitored_processes)
self.assert_processes(expected_processes, monitored_processes)
def test_change_instance_dist_container_1(self):
MockPOpen.set_next_pid(37)
self.executor_1.update_packing_plan(self.packing_plan_expected)
current_commands = self.executor_1.get_commands_to_run()
temp_dict = dict(
list(
map((lambda process_info:
(process_info.name, process_info.command.split(' '))),
self.expected_processes_container_1)))
current_json = json.dumps(current_commands,
sort_keys=True,
cls=CommandEncoder).split(' ')
temp_json = json.dumps(temp_dict, sort_keys=True).split(' ')
print(f"current_json: {current_json}")
print(f"temp_json: {temp_json}")
# better test error report
for (s1, s2) in zip(current_json, temp_json):
self.assertEqual(s1, s2)
# update instance distribution
new_packing_plan = self.build_packing_plan({
1: [('word', '3', '0'), ('word', '2', '0'), ('exclaim1', '1', '0')]
})
self.executor_1.update_packing_plan(new_packing_plan)
updated_commands = self.executor_1.get_commands_to_run()
# get the commands to kill, keep and start and verify
commands_to_kill, commands_to_keep, commands_to_start = \
self.executor_1.get_command_changes(current_commands, updated_commands)
self.assertEqual(['container_1_exclaim1_2', 'stmgr-1'],
sorted(commands_to_kill.keys()))
self.assertEqual([
'container_1_exclaim1_1', 'container_1_word_3', 'heron-shell-1',
'metricsmgr-1'
], sorted(commands_to_keep.keys()))
self.assertEqual(['container_1_word_2', 'stmgr-1'],
sorted(commands_to_start.keys()))
def assert_processes(self, expected_processes, found_processes):
self.assertEqual(len(expected_processes), len(found_processes))
for expected_process in expected_processes:
self.assert_process(expected_process, found_processes)
def assert_process(self, expected_process, found_processes):
pid = expected_process.pid
self.assertTrue(found_processes[pid])
self.assertEqual(expected_process.name, found_processes[pid].name)
self.assertEqual(expected_process.command,
found_processes[pid].command_str)
self.assertEqual(1, found_processes[pid].attempts)