def test_create_directory(self):
new_dir = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
self.assertFalse(new_dir.exists(), "Directory is already exists")
try:
new_dir.create_directory()
self.assertTrue(new_dir.exists(), "Directory was not created")
self.assertTrue(new_dir.is_directory())
finally:
new_dir.delete(recursive=True)
self.assertFalse(new_dir.exists(), "Directory was not removed")
def test_create(self):
new_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
new_dir = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
# tets new file creation
try:
new_file.create(directory=False)
self.assertTrue(new_file.exists(), "File was not created")
self.assertFalse(new_file.is_directory(), "New file should not be a directory")
finally:
new_file.delete()
self.assertFalse(new_file.exists(), "File was not removed")
# test new folder creation
try:
new_dir.create(directory=True)
self.assertTrue(new_dir.exists(), "Directory was not created")
self.assertTrue(new_dir.is_directory(), "New file should be a directory")
finally:
new_dir.delete(recursive=True)
self.assertFalse(new_dir.exists(), "Directory was not removed")
def test_create_file(self):
new_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
self.assertFalse(new_file.exists(), "File is already exists")
try:
new_file.create_file()
self.assertTrue(new_file.exists(), "File was not created")
self.assertFalse(new_file.is_directory(), "New file should not be a folder")
finally:
new_file.delete()
self.assertFalse(new_file.exists(), "File was not removed")
def test_create(self):
new_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
new_dir = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
# tets new file creation
try:
new_file.create(directory=False)
self.assertTrue(new_file.exists(), "File was not created")
self.assertFalse(new_file.is_directory(), "New file should not be a directory")
finally:
new_file.delete()
self.assertFalse(new_file.exists(), "File was not removed")
# test new folder creation
try:
new_dir.create(directory=True)
self.assertTrue(new_dir.exists(), "Directory was not created")
self.assertTrue(new_dir.is_directory(), "New file should be a directory")
finally:
new_dir.delete(recursive=True)
self.assertFalse(new_dir.exists(), "Directory was not removed")
def test_create_file(self):
new_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
self.assertFalse(new_file.exists(), "File is already exists")
try:
new_file.create_file()
self.assertTrue(new_file.exists(), "File was not created")
self.assertFalse(new_file.is_directory(), "New file should not be a folder")
finally:
new_file.delete()
self.assertFalse(new_file.exists(), "File was not removed")
def test_create_directory(self):
new_dir = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
self.assertFalse(new_dir.exists(), "Directory is already exists")
try:
new_dir.create_directory()
self.assertTrue(new_dir.exists(), "Directory was not created")
self.assertTrue(new_dir.is_directory())
finally:
new_dir.delete(recursive=True)
self.assertFalse(new_dir.exists(), "Directory was not removed")
def should_create_file_recursively(self):
_base_dir = os.path.join("/tmp", str(uuid.uuid4()))
_path = os.path.join(_base_dir, str(uuid.uuid4()), str(uuid.uuid4()), "file.txt")
_file = HDFS(_path)
self.assertFalse(_file.exists(), "File is already exists")
try:
_file.create_file(recursive=True)
self.assertTrue(_file.exists(), "File was not created")
self.assertFalse(_file.is_directory(), "New file should not be a directory")
finally:
HDFS(_base_dir).delete_directory()
self.assertFalse(_file.exists(), "File was not removed")
self.assertFalse(HDFS(_base_dir).exists(), "Bse dir was not removed")
def should_create_file_recursively(self):
_base_dir = os.path.join('/tmp', str(uuid.uuid4()))
_path = os.path.join(_base_dir, str(uuid.uuid4()), str(uuid.uuid4()), 'file.txt')
_file = HDFS(_path)
self.assertFalse(_file.exists(), "File is already exists")
try:
_file.create_file(recursive=True)
self.assertTrue(_file.exists(), "File was not created")
self.assertFalse(_file.is_directory(), "New file should not be a directory")
finally:
HDFS(_base_dir).delete_directory()
self.assertFalse(_file.exists(), "File was not removed")
self.assertFalse(HDFS(_base_dir).exists(), "Bse dir was not removed")
def test_import_to_hive(self):
_path = HDFS(os.path.join('/user', getpass.getuser(), 'table_name'))
try:
if _path.exists():
_path.delete(recursive=_path.is_directory())
# shell.execute_shell_command('hadoop fs', '-rm -r /user/', getpass.getuser(), '/table_name')
cmd = Sqoop.import_data().from_rdbms(
host=MYSQL_SERVER,
rdbms="mysql",
username="******",
password_file="{0}/rdbms.password".format(BASE_DIR),
database="sqoop_tests").table(
table="table_name").to_hive().run()
# self.assertEquals(cmd.status, 0, cmd.stderr)
# result = shell.execute_shell_command('hadoop fs', '-du -s /user/hive/warehouse/table_name/part-m-*')
# self.assertNotEqual(result.stdout.split(' ')[0], '0', result.stdout)
finally:
shell.execute_shell_command(
'hive', "-e 'DROP TABLE IF EXISTS table_name'")
def test_import_to_hive(self):
_path = HDFS(os.path.join('/user', getpass.getuser(), 'table_name'))
try:
if _path.exists():
_path.delete(recursive=_path.is_directory())
# shell.execute_shell_command('hadoop fs', '-rm -r /user/', getpass.getuser(), '/table_name')
cmd = Sqoop.import_data().from_rdbms(
host=MYSQL_SERVER,
rdbms="mysql",
username="******",
password_file="{0}/rdbms.password".format(BASE_DIR),
database="sqoop_tests"
).table(
table="table_name"
).to_hive().run()
# self.assertEquals(cmd.status, 0, cmd.stderr)
# result = shell.execute_shell_command('hadoop fs', '-du -s /user/hive/warehouse/table_name/part-m-*')
# self.assertNotEqual(result.stdout.split(' ')[0], '0', result.stdout)
finally:
shell.execute_shell_command('hive', "-e 'DROP TABLE IF EXISTS table_name'")
