def test_get_subdirs_return_val(self, mock_safe_execute): ptns = map(str, range(15)) hdfsdirs = ["%s/%s" % (self.hdfspath, p) for p in ptns] ls_output = "\n".join(hdfsdirs) mock_safe_execute.return_value = tse.ShellResult(0, ls_output, "") subdirs = self.hm.get_subdirs(self.hdfspath) self.assertListEqual(subdirs, ptns)
def test_get_newdirs_shell(self, mock_safe_execute): mock_safe_execute.return_value = tse.ShellResult( 0, self.returned_dirs, "") _ = self.hm.get_newdirs(self.hdfspath, self.lastdir, self.loadts, self.process_delay_hrs) mock_safe_execute.assert_called_with("hadoop fs -ls %s" % self.hdfspath)
def test_get_newdirs_delay(self, mock_safe_execute): mock_safe_execute.return_value = tse.ShellResult( 0, self.returned_dirs, "") loadts = self.loadts - dt.timedelta(hours=8) newdirs = self.hm.get_newdirs(self.hdfspath, self.lastdir, loadts, self.process_delay_hrs) self.assertListEqual(newdirs, [])
def test_create_partition_hdfs_path_exists_partition_does_not_exist( self, mock_chk_ptn, mock_pth_exists, mock_safe_exec): ptn_path = "/foo/2016/08/12/14/0" mock_pth_exists.return_value = True mock_chk_ptn.return_value = False mock_safe_exec.return_value = tse.ShellResult(0, "", "") self.hm.create_partition(ptn_path) mock_pth_exists.assert_called_with(ptn_path)
def test_get_primary_namenode_return_val_namenode_found( self, mock_safe_execute): namenodes = ["nn1", "nn2"] webhdfs_path = "/foo/bar" hdfs_user = "******" mock_safe_execute.return_value = tse.ShellResult(0, "FileStatus", "") nn = self.hm.get_primary_namenode(namenodes, webhdfs_path, hdfs_user) self.assertEqual(nn, "nn1")
def test_get_primary_namenode_return_val_namenode_not_found( self, mock_safe_execute): namenodes = ["nn1", "nn2"] webhdfs_path = "/foo/bar" hdfs_user = "******" mock_safe_execute.return_value = tse.ShellResult(0, "", "") self.assertIsNone( self.hm.get_primary_namenode(namenodes, webhdfs_path, hdfs_user))
def test_get_status_val(self): self.mock_exec.return_value = tse.ShellResult(0, INFO_MSG, "") jobstatus = self.om.get_status("0436651-160203234824430-oozie-oozi-W") self.assertDictEqual( jobstatus, { "overall": "KILLED", "fail": "OK", ":start:": "OK", "set-input": "ERROR" })
def test_get_counts_val(self): self.mock_exec.return_value = tse.ShellResult(0, COUNTER_MSG, "") counts = self.om.get_counts("0436835-160203234824430-oozie-oozi-W") self.assertDictEqual( counts, { "map_input_records": 12466493, "map_output_records": 28701197, "reduce_input_records": 28701197, "reduce_output_records": 28701197, "skipped": 9 })
def test_load_decompress(self, mock_vexec): mock_vexec.return_value = tse.ShellResult( 0, "Rows Loaded\n-----\n3100\n", "") nrows = self.tvm.load("foo", "bar", "foo", "bar", "foo", "decompress") self.assertEqual(nrows, "3100")
def test_shell_result_not_enough_args(self): with self.assertRaises(TypeError): shell_exec.ShellResult(1, 2)
def test_shell_result_all_args(self): self.assertEqual( shell_exec.ShellResult("123", "456", "789").__repr__(), "retcode = 123\noutput = 456\nerror = 789")
def test_launch_val(self): self.mock_exec.return_value = tse.ShellResult(0, "job: 12-34", "") jobid = self.om.launch(propfile="foo") self.assertEqual(jobid, "12-34")
def test_get_logtrace_val(self): self.mock_exec.return_value = tse.ShellResult(0, "foo", "") lt = self.om.get_logtrace("1234") self.assertEqual(lt, "foo")
def test_check_partition_ptn_doesnt_exist(self, mock_safe_exec): ptn_str = "/foo/2016/08/12/14/0" mock_safe_exec.return_value = tse.ShellResult(0, "/bar/2016/08/12/14/0", "") self.assertFalse(self.hm.check_partition(ptn_str))
def test_rollback(self, mock_vexec): mock_vexec.return_value = tse.ShellResult(0, "count\n-----\n3100\n", "") nrows = self.tvm.rollback("schema_foo", "table_foo", "schema_bar", "table_bar", "keyfoo") self.assertEqual(nrows, "3100")
def test_get_subdirs_exception(self, mock_safe_execute): mock_safe_execute.side_effect = tse.ShellException() mock_safe_execute.return_value = tse.ShellResult(0, "", "") with self.assertRaises(thex.HdfsManagerException): _ = self.hm.get_subdirs(self.hdfspath)
def test_get_newdirs_return_val_with_lastdir(self, mock_safe_execute): mock_safe_execute.return_value = tse.ShellResult( 0, self.returned_dirs, "") newdirs = self.hm.get_newdirs(self.hdfspath, self.lastdir, self.loadts, self.process_delay_hrs) self.assertListEqual(newdirs, self.returned_dirs.split("\n")[1:])
def test_get_subdirs_shell(self, mock_safe_execute): mock_safe_execute.return_value = tse.ShellResult(0, "", "") _ = self.hm.get_subdirs(self.hdfspath) mock_safe_execute.assert_called_with("hadoop fs -ls %s" % self.hdfspath)
def test_execute_stmt_output(self, mock_safe_execute): stmt = "select * from foo;" mock_safe_execute.return_value = tse.ShellResult(0, "a,b,c", "") result = self.tvm.execute(stmt=stmt) self.assertEqual(result.output, "a,b,c")
def test_check_partition_ptn_exists(self, mock_safe_exec): ptn_str = "/foo/2016/08/12/14/0" mock_safe_exec.return_value = tse.ShellResult(0, ptn_str, "") self.assertTrue(self.hm.check_partition(ptn_str))