def test_equals_file_string(self):
files = self.temp_file()
try:
expend_q = Hive.load_queries_from_string("show tables").use_database("default")
expend_f = Hive.load_queries_from_file(files).use_database("default")
self.assertEqual(expend_q.run().is_ok(), expend_f.run().is_ok())
finally:
self.delete_local(files)
def test_create_database(self):
db_exist = False
try:
msg = "create database if not EXISTS testdb;"
self.assertRaises(HiveCommandError, Hive.load_queries_from_string("drop database testdb").run)
self.assertTrue(Hive.load_queries_from_string(msg).run().is_ok(),
"create database Done")
db_exist = True
self.assertTrue(Hive.load_queries_from_string("show tables").use_database("testdb").run().is_ok(),
"check database in query")
finally:
if db_exist:
Hive.load_queries_from_string("drop database testdb;").run()
def test_exist_table_string(self):
files, msg = self.create_database()
try:
res = Hive.load_queries_from_string(msg)
self.assertEqual(True, res.run().is_ok())
finally:
self.delete_local(files)
def test_use_database(self):
_command = 'hive -e "test" --database hello'
hive = (
Hive.load_queries_from_string(query="test", executor=mock_executor(expected_command=_command))
.use_database("blabla")
.use_database("hello")
)
hive.run()
def test_exist_table_file(self):
files, msg = self.create_database()
msg_f = self.temp_file(msg=msg)
try:
res = Hive.load_queries_from_file(msg_f)
self.assertEqual(True, res.run().is_ok())
finally:
self.delete_local(msg_f)
self.delete_local(files)
def test_load_config(self):
_command = "hive -e \"test\" --define A=B --define C=D --hiveconf hello=world " \
"--hivevar A=B --hivevar C=D --database hive"
metastore = IniFileMetaStore(file=os.path.join(
os.path.dirname(__file__), 'resources/hive/hive.ini'))
hive = Hive.load_preconfigured_job(name='hive test',
config=Configuration.load(
metastore=metastore,
readonly=False, accepts_nulls=True),
executor=mock_executor(expected_command=_command)) \
.with_hive_conf("hello", "world")
hive.run()
def test_load_config(self):
_command = (
'hive -e "test" --define A=B --define C=D --hiveconf hello=world '
"--hivevar A=B --hivevar C=D --database hive"
)
metastore = IniFileMetaStore(file=os.path.join(os.path.dirname(__file__), "resources/hive/hive.ini"))
hive = Hive.load_preconfigured_job(
name="hive test",
config=Configuration.load(metastore=metastore, readonly=False, accepts_nulls=True),
executor=mock_executor(expected_command=_command),
).with_hive_conf("hello", "world")
hive.run()
def test_change_config_hive(self):
try:
hive = Hive.load_queries_from_string("show tables") \
.with_hive_conf("hive.log.dir", "'/tmp/hivelog'") \
.use_database("default")
hive.run()
import os
self.assertEqual(os.path.exists("/tmp/hivelog/hive.log"), True)
finally:
import shutil
shutil.rmtree('/tmp/hivelog')
def test_with_auxpath(self):
_command = "hive " \
"--auxpath dear user,hello "\
"-e \"test\" " \
"--define key=value " \
"--hivevar hello=user " \
"--database hello" \
hive = Hive.load_queries_from_string(query="test", executor=mock_executor(expected_command=_command)) \
.with_auxillary_jars("dear user,hello") \
.add_hivevar("hello", "user") \
.use_database("hello") \
.define_variable("key", "value")
hive.run()
def test_with_auxpath(self):
_command = (
"hive "
"--auxpath dear user,hello "
'-e "test" '
"--define key=value "
"--hivevar hello=user "
"--database hello"
)
hive = (
Hive.load_queries_from_string(query="test", executor=mock_executor(expected_command=_command))
.with_auxillary_jars("dear user,hello")
.add_hivevar("hello", "user")
.use_database("hello")
.define_variable("key", "value")
)
hive.run()
def test_create_table(self):
files = self.temp_file(msg="hello,world")
try:
msg = "create database if not EXISTS testdb;"
self.assertTrue(Hive.load_queries_from_string(msg).run().is_ok())
msg = "create table some_table(strings STRING)" \
"ROW FORMAT DELIMITED " \
"FIELDS TERMINATED BY ','" \
"STORED AS TEXTFILE " \
"Location '/tmp/hive_table';"
hive = Hive.load_queries_from_string(msg).use_database("testdb")
self.assertTrue(hive.use_database("testdb").run().is_ok(), " create table")
msg = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE some_table;".format(files)
hive = Hive.load_queries_from_string(msg).use_database("testdb")
self.assertTrue(hive.run().is_ok(), "load data in table")
self.assertTrue(hdfs.is_file_exists("/tmp/hive_table"), "check tables")
finally:
Hive.load_queries_from_string("drop table some_table;").use_database("testdb").run()
Hive.load_queries_from_string("drop database testdb;").use_database("testdb").run()
self.delete_local(files)
self.delete_file_in_hdfs()
def test_add_hivevar(self):
_command = 'hive -e "test" --hivevar hello=world'
hive = Hive.load_queries_from_string(
query="test", executor=mock_executor(expected_command=_command)
).add_hivevar("hello", "world")
hive.run()
def hive_add_partition(context):
for path in context['new_pathes']:
Hive.load_queries_from_string(
query="USE hive_monitoring; ALTER TABLE data "
"ADD PARTITION(date='{0}') "
"LOCATION '{1}';".format(parser_name(path), path)).run()
"resources/file_12.11.2014_.txt"))
ftp.upload(local_path=os.path.join(os.path.dirname(__file__),
"resources/file_13.11.2014_.txt"))
ftp.upload(local_path=os.path.join(os.path.dirname(__file__),
"resources/file_14.11.2014_.txt"))
# upload file to HDFS/ create directories
hdfs_file = HDFS("{0}/raw/12.11.2014".format(BASE_DIR))
hdfs_file.create(directory=True)
local_file = LocalFS(path=os.path.join(os.path.dirname(__file__),
'resources/file_12.11.2014_.txt'))
local_file.copy_to_hdfs(hdfs_path="{0}/raw/12.11.2014".format(BASE_DIR))
hdfs_file = HDFS("{0}/raw/13.11.2014".format(BASE_DIR))
hdfs_file.create(directory=True)
local_file = LocalFS(path=os.path.join(os.path.dirname(__file__),
'resources/file_13.11.2014_.txt'))
local_file.copy_to_hdfs(hdfs_path="{0}/raw/13.11.2014".format(BASE_DIR))
# create empty local directory 'tmp' in folder 'resources'
local_file = LocalFS(
path=os.path.join(os.path.dirname(__file__), 'resources/tmp'))
if local_file.exists():
local_file.delete_directory()
local_file.create(directory=True)
# create HIVE external table with partition
hive = Hive.load_queries_from_file(
path=os.path.join(os.path.dirname(__file__), "resources/script.hql"))
hive.run()
def test_define_variable(self):
_command = 'hive -e "test" --define hello=world'
hive = Hive.load_queries_from_string(
query="test", executor=mock_executor(expected_command=_command)
).define_variable("hello", "world")
hive.run()
def test___init__(self):
hive = Hive("hello", lambda: "world")
assert True if hive is not None else False
def hive_add_partition(context):
for path in context['new_pathes']:
Hive.load_queries_from_string(query="USE hive_monitoring; ALTER TABLE data "
"ADD PARTITION(date='{0}') "
"LOCATION '{1}';"
.format(parser_name(path), path)).run()
def test_execute_query_string(self):
hive = Hive.load_queries_from_string("show tables").with_hive_conf("A", "B").add_hivevar("A", "B") \
.define_variable("A", "B")
res = hive.run()
self.assertEqual(res.is_ok(), True)
def test_add_hivevar(self):
_command = "hive -e \"test\" --hivevar hello=world"
hive = Hive.load_queries_from_string(query="test", executor=mock_executor(expected_command=_command)) \
.add_hivevar("hello", "world")
hive.run()
def test_define_variable(self):
_command = "hive -e \"test\" --define hello=world"
hive = Hive.load_queries_from_string(query="test", executor=mock_executor(expected_command=_command)) \
.define_variable("hello", "world")
hive.run()
def test_use_database(self):
_command = "hive -e \"test\" --database hello"
hive = Hive.load_queries_from_string(query="test", executor=mock_executor(expected_command=_command)) \
.use_database("blabla") \
.use_database("hello")
hive.run()
if __name__ == "__main__":
hdfs_file = HDFS("{0}/raw".format(BASE_DIR))
if hdfs_file.exists():
hdfs_file.delete(recursive=True)
config = RawConfigParser()
config.read(os.path.join(os.path.dirname(__file__), "resources/ftp_config.ini"))
host_download = config.get("ftp", "host.download")
user_name = config.get("ftp", "user.name")
password = config.get("ftp", "password")
path = config.get("ftp", "path")
ftp = ftp_client(host=host_download,
login=user_name,
password=password,
path="/tmp")
if ftp.exists():
ftp.delete(recursive=True)
local_file = LocalFS(path=os.path.join(os.path.dirname(__file__),
'resources/tmp'))
if local_file.exists():
local_file.delete_directory()
hive = Hive.load_queries_from_string(query="DROP DATABASE IF EXISTS hive_monitoring CASCADE;")
hive.run()
# upload file to directory on FTP
ftp.upload(local_path=os.path.join(os.path.dirname(__file__), "resources/file_12.11.2014_.txt"))
ftp.upload(local_path=os.path.join(os.path.dirname(__file__), "resources/file_13.11.2014_.txt"))
ftp.upload(local_path=os.path.join(os.path.dirname(__file__), "resources/file_14.11.2014_.txt"))
# upload file to HDFS/ create directories
hdfs_file = HDFS("{0}/raw/12.11.2014".format(BASE_DIR))
hdfs_file.create(directory=True)
local_file = LocalFS(path=os.path.join(os.path.dirname(__file__),
'resources/file_12.11.2014_.txt'))
local_file.copy_to_hdfs(hdfs_path="{0}/raw/12.11.2014".format(BASE_DIR))
hdfs_file = HDFS("{0}/raw/13.11.2014".format(BASE_DIR))
hdfs_file.create(directory=True)
local_file = LocalFS(path=os.path.join(os.path.dirname(__file__),
'resources/file_13.11.2014_.txt'))
local_file.copy_to_hdfs(hdfs_path="{0}/raw/13.11.2014".format(BASE_DIR))
# create empty local directory 'tmp' in folder 'resources'
local_file = LocalFS(path=os.path.join(os.path.dirname(__file__),
'resources/tmp'))
if local_file.exists():
local_file.delete_directory()
local_file.create(directory=True)
# create HIVE external table with partition
hive = Hive.load_queries_from_file(path=os.path.join(os.path.dirname(__file__), "resources/script.hql"))
hive.run()
