def test_get_subdirs_return_val(self, mock_safe_execute):
ptns = map(str, range(15))
hdfsdirs = ["%s/%s" % (self.hdfspath, p) for p in ptns]
ls_output = "\n".join(hdfsdirs)
mock_safe_execute.return_value = tse.ShellResult(0, ls_output, "")
subdirs = self.hm.get_subdirs(self.hdfspath)
self.assertListEqual(subdirs, ptns)
def test_get_newdirs_shell(self, mock_safe_execute):
mock_safe_execute.return_value = tse.ShellResult(
0, self.returned_dirs, "")
_ = self.hm.get_newdirs(self.hdfspath, self.lastdir, self.loadts,
self.process_delay_hrs)
mock_safe_execute.assert_called_with("hadoop fs -ls %s" %
self.hdfspath)
def test_get_newdirs_delay(self, mock_safe_execute):
mock_safe_execute.return_value = tse.ShellResult(
0, self.returned_dirs, "")
loadts = self.loadts - dt.timedelta(hours=8)
newdirs = self.hm.get_newdirs(self.hdfspath, self.lastdir, loadts,
self.process_delay_hrs)
self.assertListEqual(newdirs, [])
def test_create_partition_hdfs_path_exists_partition_does_not_exist(
self, mock_chk_ptn, mock_pth_exists, mock_safe_exec):
ptn_path = "/foo/2016/08/12/14/0"
mock_pth_exists.return_value = True
mock_chk_ptn.return_value = False
mock_safe_exec.return_value = tse.ShellResult(0, "", "")
self.hm.create_partition(ptn_path)
mock_pth_exists.assert_called_with(ptn_path)
def test_get_primary_namenode_return_val_namenode_found(
self, mock_safe_execute):
namenodes = ["nn1", "nn2"]
webhdfs_path = "/foo/bar"
hdfs_user = "******"
mock_safe_execute.return_value = tse.ShellResult(0, "FileStatus", "")
nn = self.hm.get_primary_namenode(namenodes, webhdfs_path, hdfs_user)
self.assertEqual(nn, "nn1")
def test_get_primary_namenode_return_val_namenode_not_found(
self, mock_safe_execute):
namenodes = ["nn1", "nn2"]
webhdfs_path = "/foo/bar"
hdfs_user = "******"
mock_safe_execute.return_value = tse.ShellResult(0, "", "")
self.assertIsNone(
self.hm.get_primary_namenode(namenodes, webhdfs_path, hdfs_user))
def test_get_status_val(self):
self.mock_exec.return_value = tse.ShellResult(0, INFO_MSG, "")
jobstatus = self.om.get_status("0436651-160203234824430-oozie-oozi-W")
self.assertDictEqual(
jobstatus, {
"overall": "KILLED",
"fail": "OK",
":start:": "OK",
"set-input": "ERROR"
})
def test_get_counts_val(self):
self.mock_exec.return_value = tse.ShellResult(0, COUNTER_MSG, "")
counts = self.om.get_counts("0436835-160203234824430-oozie-oozi-W")
self.assertDictEqual(
counts, {
"map_input_records": 12466493,
"map_output_records": 28701197,
"reduce_input_records": 28701197,
"reduce_output_records": 28701197,
"skipped": 9
})
def test_load_decompress(self, mock_vexec):
mock_vexec.return_value = tse.ShellResult(
0, "Rows Loaded\n-----\n3100\n", "")
nrows = self.tvm.load("foo", "bar", "foo", "bar", "foo", "decompress")
self.assertEqual(nrows, "3100")
def test_shell_result_not_enough_args(self):
with self.assertRaises(TypeError):
shell_exec.ShellResult(1, 2)
def test_shell_result_all_args(self):
self.assertEqual(
shell_exec.ShellResult("123", "456", "789").__repr__(),
"retcode = 123\noutput = 456\nerror = 789")
def test_launch_val(self):
self.mock_exec.return_value = tse.ShellResult(0, "job: 12-34", "")
jobid = self.om.launch(propfile="foo")
self.assertEqual(jobid, "12-34")
def test_get_logtrace_val(self):
self.mock_exec.return_value = tse.ShellResult(0, "foo", "")
lt = self.om.get_logtrace("1234")
self.assertEqual(lt, "foo")
def test_check_partition_ptn_doesnt_exist(self, mock_safe_exec):
ptn_str = "/foo/2016/08/12/14/0"
mock_safe_exec.return_value = tse.ShellResult(0,
"/bar/2016/08/12/14/0",
"")
self.assertFalse(self.hm.check_partition(ptn_str))
def test_rollback(self, mock_vexec):
mock_vexec.return_value = tse.ShellResult(0, "count\n-----\n3100\n",
"")
nrows = self.tvm.rollback("schema_foo", "table_foo", "schema_bar",
"table_bar", "keyfoo")
self.assertEqual(nrows, "3100")
def test_get_subdirs_exception(self, mock_safe_execute):
mock_safe_execute.side_effect = tse.ShellException()
mock_safe_execute.return_value = tse.ShellResult(0, "", "")
with self.assertRaises(thex.HdfsManagerException):
_ = self.hm.get_subdirs(self.hdfspath)
def test_get_newdirs_return_val_with_lastdir(self, mock_safe_execute):
mock_safe_execute.return_value = tse.ShellResult(
0, self.returned_dirs, "")
newdirs = self.hm.get_newdirs(self.hdfspath, self.lastdir, self.loadts,
self.process_delay_hrs)
self.assertListEqual(newdirs, self.returned_dirs.split("\n")[1:])
def test_get_subdirs_shell(self, mock_safe_execute):
mock_safe_execute.return_value = tse.ShellResult(0, "", "")
_ = self.hm.get_subdirs(self.hdfspath)
mock_safe_execute.assert_called_with("hadoop fs -ls %s" %
self.hdfspath)
def test_execute_stmt_output(self, mock_safe_execute):
stmt = "select * from foo;"
mock_safe_execute.return_value = tse.ShellResult(0, "a,b,c", "")
result = self.tvm.execute(stmt=stmt)
self.assertEqual(result.output, "a,b,c")
def test_check_partition_ptn_exists(self, mock_safe_exec):
ptn_str = "/foo/2016/08/12/14/0"
mock_safe_exec.return_value = tse.ShellResult(0, ptn_str, "")
self.assertTrue(self.hm.check_partition(ptn_str))
