def test_sizer_run_setsizes_singlehost(self, mock_ssh):
m_stdout = ("1001\t/path/to/data/testTopic1-0\n"
"1002\t/path/to/data/testTopic1-1\n"
"2001\t/path/to/data/testTopic2-0\n"
"2002\t/path/to/data/testTopic2-1\n")
mock_ssh.set_default(stdout=m_stdout.encode('utf-8'))
sizer = SizerSSH(self.args, self.cluster)
sizer.get_partition_sizes()
compare([call.Popen(['ssh', 'brokerhost1.example.com', 'du -sk /path/to/data/*'], stdout=PIPE, stderr=ANY)], mock_ssh.mock.method_calls)
assert self.cluster.topics['testTopic1'].partitions[0].size == 1001
assert self.cluster.topics['testTopic1'].partitions[1].size == 1002
assert self.cluster.topics['testTopic2'].partitions[0].size == 2001
assert self.cluster.topics['testTopic2'].partitions[1].size == 2002
def test_sizer_run_badinput(self):
self.create_cluster_onehost()
m_stdout = StringIO(u'foo\nbar\nbaz\n')
m_stderr = StringIO()
m_stdin = StringIO()
self.mock_exec_command.return_value = (m_stdin, m_stdout, m_stderr)
sizer = SizerSSH(self.args, self.cluster)
sizer.get_partition_sizes()
self.mock_connect.assert_called_once_with(ANY, "brokerhost1.example.com", allow_agent=True)
self.mock_exec_command.assert_called_once_with(ANY, "du -sk /path/to/data/*")
assert self.cluster.topics['testTopic1'].partitions[0].size == 0
assert self.cluster.topics['testTopic1'].partitions[1].size == 0
assert self.cluster.topics['testTopic2'].partitions[0].size == 0
assert self.cluster.topics['testTopic2'].partitions[1].size == 0
def test_sizer_regex_good(self):
sizer = SizerSSH(self.args, self.cluster)
match_obj = sizer.size_re.match("1001\t/path/to/data/testTopic1-0\n")
assert match_obj is not None
assert match_obj.group(1) == "1001"
assert match_obj.group(2) == "testTopic1"
assert match_obj.group(3) == "0"
def test_sizer_run_setsizes_singlehost(self):
self.create_cluster_onehost()
m_stdout = StringIO(u'1001\t/path/to/data/testTopic1-0\n'
"1002\t/path/to/data/testTopic1-1\n"
"2001\t/path/to/data/testTopic2-0\n"
"2002\t/path/to/data/testTopic2-1\n")
m_stderr = StringIO()
m_stdin = StringIO()
self.mock_exec_command.return_value = (m_stdin, m_stdout, m_stderr)
sizer = SizerSSH(self.args, self.cluster)
sizer.get_partition_sizes()
self.mock_connect.assert_called_once_with(ANY, "brokerhost1.example.com", allow_agent=True)
self.mock_exec_command.assert_called_once_with(ANY, "du -sk /path/to/data/*")
assert self.cluster.topics['testTopic1'].partitions[0].size == 1001
assert self.cluster.topics['testTopic1'].partitions[1].size == 1002
assert self.cluster.topics['testTopic2'].partitions[0].size == 2001
assert self.cluster.topics['testTopic2'].partitions[1].size == 2002
def test_sizer_run_setsizes_singlehost(self, mock_ssh):
m_stdout = ("1001\t/path/to/data/testTopic1-0\n"
"1002\t/path/to/data/testTopic1-1\n"
"2001\t/path/to/data/testTopic2-0\n"
"2002\t/path/to/data/testTopic2-1\n")
mock_ssh.set_default(stdout=m_stdout.encode('utf-8'))
sizer = SizerSSH(self.args, self.cluster)
sizer.get_partition_sizes()
compare([
call.Popen(
['ssh', 'brokerhost1.example.com', 'du -sk /path/to/data/*'],
stdout=PIPE,
stderr=ANY)
], mock_ssh.mock.method_calls)
assert self.cluster.topics['testTopic1'].partitions[0].size == 1001
assert self.cluster.topics['testTopic1'].partitions[1].size == 1002
assert self.cluster.topics['testTopic2'].partitions[0].size == 2001
assert self.cluster.topics['testTopic2'].partitions[1].size == 2002
def test_sizer_run_setsizes_twohost(self):
self.create_cluster_twohost()
m_stdout_1 = StringIO(u'1001\t/path/to/data/testTopic1-0\n'
u'1002\t/path/to/data/testTopic1-1\n'
u'2001\t/path/to/data/testTopic2-0\n'
u'2002\t/path/to/data/testTopic2-1\n')
m_stdout_2 = StringIO(u'1001\t/path/to/data/testTopic1-0\n'
u'4002\t/path/to/data/testTopic1-1\n'
u'1011\t/path/to/data/testTopic2-0\n'
u'2002\t/path/to/data/testTopic2-1\n')
m_stderr = StringIO()
m_stdin = StringIO()
self.mock_exec_command.side_effect = [(m_stdin, m_stdout_1, m_stderr), (m_stdin, m_stdout_2, m_stderr)]
sizer = SizerSSH(self.args, self.cluster)
sizer.get_partition_sizes()
self.mock_connect.assert_has_calls([call(ANY, "brokerhost1.example.com", allow_agent=True), call(ANY, "brokerhost2.example.com", allow_agent=True)])
self.mock_exec_command.assert_has_calls([call(ANY, "du -sk /path/to/data/*"), call(ANY, "du -sk /path/to/data/*")])
assert self.cluster.topics['testTopic1'].partitions[0].size == 1001
assert self.cluster.topics['testTopic1'].partitions[1].size == 4002
assert self.cluster.topics['testTopic2'].partitions[0].size == 2001
assert self.cluster.topics['testTopic2'].partitions[1].size == 2002
def test_process_df_match_badpartition(self):
sizer = SizerSSH(self.args, self.cluster)
match_obj = sizer.size_re.match("5002\t/path/to/data/testTopic1-2\n")
sizer.process_df_match(match_obj, 1)
def test_process_df_match_badtopic(self):
sizer = SizerSSH(self.args, self.cluster)
match_obj = sizer.size_re.match("5002\t/path/to/data/badTopic1-1\n")
sizer.process_df_match(match_obj, 1)
def test_process_df_match(self):
sizer = SizerSSH(self.args, self.cluster)
match_obj = sizer.size_re.match("5002\t/path/to/data/testTopic1-1\n")
sizer.process_df_match(match_obj, 1)
assert self.cluster.topics['testTopic1'].partitions[1].size == 5002
def test_sizer_regex_bad(self):
sizer = SizerSSH(self.args, self.cluster)
assert sizer.size_re.match("foo") is None
def test_sizer_run_missing_host(self):
self.cluster.brokers[1].hostname = None
sizer = SizerSSH(self.args, self.cluster)
self.assertRaises(UnknownBrokerException, sizer.get_partition_sizes)
def test_sizer_create(self):
sizer = SizerSSH(self.args, self.cluster)
assert isinstance(sizer, SizerSSH)
def test_process_df_match_badpartition(self):
sizer = SizerSSH(self.args, self.cluster)
match_obj = sizer.size_re.match("5002\t/path/to/data/testTopic1-2\n")
sizer.process_df_match(match_obj, 1)
def test_process_df_match_badtopic(self):
sizer = SizerSSH(self.args, self.cluster)
match_obj = sizer.size_re.match("5002\t/path/to/data/badTopic1-1\n")
sizer.process_df_match(match_obj, 1)
def test_process_df_match(self):
sizer = SizerSSH(self.args, self.cluster)
match_obj = sizer.size_re.match("5002\t/path/to/data/testTopic1-1\n")
sizer.process_df_match(match_obj, 1)
assert self.cluster.topics['testTopic1'].partitions[1].size == 5002
