def test_create_directory(self): new_dir = HDFS(os.path.join("/tmp", str(uuid.uuid4()))) self.assertFalse(new_dir.exists(), "Directory is already exists") try: new_dir.create_directory() self.assertTrue(new_dir.exists(), "Directory was not created") self.assertTrue(new_dir.is_directory()) finally: new_dir.delete(recursive=True) self.assertFalse(new_dir.exists(), "Directory was not removed")
def test_create(self): new_file = HDFS(os.path.join("/tmp", str(uuid.uuid4()))) new_dir = HDFS(os.path.join("/tmp", str(uuid.uuid4()))) # tets new file creation try: new_file.create(directory=False) self.assertTrue(new_file.exists(), "File was not created") self.assertFalse(new_file.is_directory(), "New file should not be a directory") finally: new_file.delete() self.assertFalse(new_file.exists(), "File was not removed") # test new folder creation try: new_dir.create(directory=True) self.assertTrue(new_dir.exists(), "Directory was not created") self.assertTrue(new_dir.is_directory(), "New file should be a directory") finally: new_dir.delete(recursive=True) self.assertFalse(new_dir.exists(), "Directory was not removed")
def test_create_file(self): new_file = HDFS(os.path.join("/tmp", str(uuid.uuid4()))) self.assertFalse(new_file.exists(), "File is already exists") try: new_file.create_file() self.assertTrue(new_file.exists(), "File was not created") self.assertFalse(new_file.is_directory(), "New file should not be a folder") finally: new_file.delete() self.assertFalse(new_file.exists(), "File was not removed")
def should_create_file_recursively(self): _base_dir = os.path.join("/tmp", str(uuid.uuid4())) _path = os.path.join(_base_dir, str(uuid.uuid4()), str(uuid.uuid4()), "file.txt") _file = HDFS(_path) self.assertFalse(_file.exists(), "File is already exists") try: _file.create_file(recursive=True) self.assertTrue(_file.exists(), "File was not created") self.assertFalse(_file.is_directory(), "New file should not be a directory") finally: HDFS(_base_dir).delete_directory() self.assertFalse(_file.exists(), "File was not removed") self.assertFalse(HDFS(_base_dir).exists(), "Bse dir was not removed")
def should_create_file_recursively(self): _base_dir = os.path.join('/tmp', str(uuid.uuid4())) _path = os.path.join(_base_dir, str(uuid.uuid4()), str(uuid.uuid4()), 'file.txt') _file = HDFS(_path) self.assertFalse(_file.exists(), "File is already exists") try: _file.create_file(recursive=True) self.assertTrue(_file.exists(), "File was not created") self.assertFalse(_file.is_directory(), "New file should not be a directory") finally: HDFS(_base_dir).delete_directory() self.assertFalse(_file.exists(), "File was not removed") self.assertFalse(HDFS(_base_dir).exists(), "Bse dir was not removed")
def test_import_to_hive(self): _path = HDFS(os.path.join('/user', getpass.getuser(), 'table_name')) try: if _path.exists(): _path.delete(recursive=_path.is_directory()) # shell.execute_shell_command('hadoop fs', '-rm -r /user/', getpass.getuser(), '/table_name') cmd = Sqoop.import_data().from_rdbms( host=MYSQL_SERVER, rdbms="mysql", username="******", password_file="{0}/rdbms.password".format(BASE_DIR), database="sqoop_tests").table( table="table_name").to_hive().run() # self.assertEquals(cmd.status, 0, cmd.stderr) # result = shell.execute_shell_command('hadoop fs', '-du -s /user/hive/warehouse/table_name/part-m-*') # self.assertNotEqual(result.stdout.split(' ')[0], '0', result.stdout) finally: shell.execute_shell_command( 'hive', "-e 'DROP TABLE IF EXISTS table_name'")
def test_import_to_hive(self): _path = HDFS(os.path.join('/user', getpass.getuser(), 'table_name')) try: if _path.exists(): _path.delete(recursive=_path.is_directory()) # shell.execute_shell_command('hadoop fs', '-rm -r /user/', getpass.getuser(), '/table_name') cmd = Sqoop.import_data().from_rdbms( host=MYSQL_SERVER, rdbms="mysql", username="******", password_file="{0}/rdbms.password".format(BASE_DIR), database="sqoop_tests" ).table( table="table_name" ).to_hive().run() # self.assertEquals(cmd.status, 0, cmd.stderr) # result = shell.execute_shell_command('hadoop fs', '-du -s /user/hive/warehouse/table_name/part-m-*') # self.assertNotEqual(result.stdout.split(' ')[0], '0', result.stdout) finally: shell.execute_shell_command('hive', "-e 'DROP TABLE IF EXISTS table_name'")