def test_equals_file_string(self): files = self.temp_file() try: expend_q = Hive.load_queries_from_string("show tables").use_database("default") expend_f = Hive.load_queries_from_file(files).use_database("default") self.assertEqual(expend_q.run().is_ok(), expend_f.run().is_ok()) finally: self.delete_local(files)
def test_create_database(self): db_exist = False try: msg = "create database if not EXISTS testdb;" self.assertRaises(HiveCommandError, Hive.load_queries_from_string("drop database testdb").run) self.assertTrue(Hive.load_queries_from_string(msg).run().is_ok(), "create database Done") db_exist = True self.assertTrue(Hive.load_queries_from_string("show tables").use_database("testdb").run().is_ok(), "check database in query") finally: if db_exist: Hive.load_queries_from_string("drop database testdb;").run()
def test_exist_table_string(self): files, msg = self.create_database() try: res = Hive.load_queries_from_string(msg) self.assertEqual(True, res.run().is_ok()) finally: self.delete_local(files)
def test_use_database(self): _command = 'hive -e "test" --database hello' hive = ( Hive.load_queries_from_string(query="test", executor=mock_executor(expected_command=_command)) .use_database("blabla") .use_database("hello") ) hive.run()
def test_exist_table_file(self): files, msg = self.create_database() msg_f = self.temp_file(msg=msg) try: res = Hive.load_queries_from_file(msg_f) self.assertEqual(True, res.run().is_ok()) finally: self.delete_local(msg_f) self.delete_local(files)
def test_load_config(self): _command = "hive -e \"test\" --define A=B --define C=D --hiveconf hello=world " \ "--hivevar A=B --hivevar C=D --database hive" metastore = IniFileMetaStore(file=os.path.join( os.path.dirname(__file__), 'resources/hive/hive.ini')) hive = Hive.load_preconfigured_job(name='hive test', config=Configuration.load( metastore=metastore, readonly=False, accepts_nulls=True), executor=mock_executor(expected_command=_command)) \ .with_hive_conf("hello", "world") hive.run()
def test_load_config(self): _command = ( 'hive -e "test" --define A=B --define C=D --hiveconf hello=world ' "--hivevar A=B --hivevar C=D --database hive" ) metastore = IniFileMetaStore(file=os.path.join(os.path.dirname(__file__), "resources/hive/hive.ini")) hive = Hive.load_preconfigured_job( name="hive test", config=Configuration.load(metastore=metastore, readonly=False, accepts_nulls=True), executor=mock_executor(expected_command=_command), ).with_hive_conf("hello", "world") hive.run()
def test_change_config_hive(self): try: hive = Hive.load_queries_from_string("show tables") \ .with_hive_conf("hive.log.dir", "'/tmp/hivelog'") \ .use_database("default") hive.run() import os self.assertEqual(os.path.exists("/tmp/hivelog/hive.log"), True) finally: import shutil shutil.rmtree('/tmp/hivelog')
def test_with_auxpath(self): _command = "hive " \ "--auxpath dear user,hello "\ "-e \"test\" " \ "--define key=value " \ "--hivevar hello=user " \ "--database hello" \ hive = Hive.load_queries_from_string(query="test", executor=mock_executor(expected_command=_command)) \ .with_auxillary_jars("dear user,hello") \ .add_hivevar("hello", "user") \ .use_database("hello") \ .define_variable("key", "value") hive.run()
def test_with_auxpath(self): _command = ( "hive " "--auxpath dear user,hello " '-e "test" ' "--define key=value " "--hivevar hello=user " "--database hello" ) hive = ( Hive.load_queries_from_string(query="test", executor=mock_executor(expected_command=_command)) .with_auxillary_jars("dear user,hello") .add_hivevar("hello", "user") .use_database("hello") .define_variable("key", "value") ) hive.run()
def test_create_table(self): files = self.temp_file(msg="hello,world") try: msg = "create database if not EXISTS testdb;" self.assertTrue(Hive.load_queries_from_string(msg).run().is_ok()) msg = "create table some_table(strings STRING)" \ "ROW FORMAT DELIMITED " \ "FIELDS TERMINATED BY ','" \ "STORED AS TEXTFILE " \ "Location '/tmp/hive_table';" hive = Hive.load_queries_from_string(msg).use_database("testdb") self.assertTrue(hive.use_database("testdb").run().is_ok(), " create table") msg = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE some_table;".format(files) hive = Hive.load_queries_from_string(msg).use_database("testdb") self.assertTrue(hive.run().is_ok(), "load data in table") self.assertTrue(hdfs.is_file_exists("/tmp/hive_table"), "check tables") finally: Hive.load_queries_from_string("drop table some_table;").use_database("testdb").run() Hive.load_queries_from_string("drop database testdb;").use_database("testdb").run() self.delete_local(files) self.delete_file_in_hdfs()
def test_add_hivevar(self): _command = 'hive -e "test" --hivevar hello=world' hive = Hive.load_queries_from_string( query="test", executor=mock_executor(expected_command=_command) ).add_hivevar("hello", "world") hive.run()
def hive_add_partition(context): for path in context['new_pathes']: Hive.load_queries_from_string( query="USE hive_monitoring; ALTER TABLE data " "ADD PARTITION(date='{0}') " "LOCATION '{1}';".format(parser_name(path), path)).run()
"resources/file_12.11.2014_.txt")) ftp.upload(local_path=os.path.join(os.path.dirname(__file__), "resources/file_13.11.2014_.txt")) ftp.upload(local_path=os.path.join(os.path.dirname(__file__), "resources/file_14.11.2014_.txt")) # upload file to HDFS/ create directories hdfs_file = HDFS("{0}/raw/12.11.2014".format(BASE_DIR)) hdfs_file.create(directory=True) local_file = LocalFS(path=os.path.join(os.path.dirname(__file__), 'resources/file_12.11.2014_.txt')) local_file.copy_to_hdfs(hdfs_path="{0}/raw/12.11.2014".format(BASE_DIR)) hdfs_file = HDFS("{0}/raw/13.11.2014".format(BASE_DIR)) hdfs_file.create(directory=True) local_file = LocalFS(path=os.path.join(os.path.dirname(__file__), 'resources/file_13.11.2014_.txt')) local_file.copy_to_hdfs(hdfs_path="{0}/raw/13.11.2014".format(BASE_DIR)) # create empty local directory 'tmp' in folder 'resources' local_file = LocalFS( path=os.path.join(os.path.dirname(__file__), 'resources/tmp')) if local_file.exists(): local_file.delete_directory() local_file.create(directory=True) # create HIVE external table with partition hive = Hive.load_queries_from_file( path=os.path.join(os.path.dirname(__file__), "resources/script.hql")) hive.run()
def test_define_variable(self): _command = 'hive -e "test" --define hello=world' hive = Hive.load_queries_from_string( query="test", executor=mock_executor(expected_command=_command) ).define_variable("hello", "world") hive.run()
def test___init__(self): hive = Hive("hello", lambda: "world") assert True if hive is not None else False
def hive_add_partition(context): for path in context['new_pathes']: Hive.load_queries_from_string(query="USE hive_monitoring; ALTER TABLE data " "ADD PARTITION(date='{0}') " "LOCATION '{1}';" .format(parser_name(path), path)).run()
def test_execute_query_string(self): hive = Hive.load_queries_from_string("show tables").with_hive_conf("A", "B").add_hivevar("A", "B") \ .define_variable("A", "B") res = hive.run() self.assertEqual(res.is_ok(), True)
def test_add_hivevar(self): _command = "hive -e \"test\" --hivevar hello=world" hive = Hive.load_queries_from_string(query="test", executor=mock_executor(expected_command=_command)) \ .add_hivevar("hello", "world") hive.run()
def test_define_variable(self): _command = "hive -e \"test\" --define hello=world" hive = Hive.load_queries_from_string(query="test", executor=mock_executor(expected_command=_command)) \ .define_variable("hello", "world") hive.run()
def test_use_database(self): _command = "hive -e \"test\" --database hello" hive = Hive.load_queries_from_string(query="test", executor=mock_executor(expected_command=_command)) \ .use_database("blabla") \ .use_database("hello") hive.run()
if __name__ == "__main__": hdfs_file = HDFS("{0}/raw".format(BASE_DIR)) if hdfs_file.exists(): hdfs_file.delete(recursive=True) config = RawConfigParser() config.read(os.path.join(os.path.dirname(__file__), "resources/ftp_config.ini")) host_download = config.get("ftp", "host.download") user_name = config.get("ftp", "user.name") password = config.get("ftp", "password") path = config.get("ftp", "path") ftp = ftp_client(host=host_download, login=user_name, password=password, path="/tmp") if ftp.exists(): ftp.delete(recursive=True) local_file = LocalFS(path=os.path.join(os.path.dirname(__file__), 'resources/tmp')) if local_file.exists(): local_file.delete_directory() hive = Hive.load_queries_from_string(query="DROP DATABASE IF EXISTS hive_monitoring CASCADE;") hive.run()
# upload file to directory on FTP ftp.upload(local_path=os.path.join(os.path.dirname(__file__), "resources/file_12.11.2014_.txt")) ftp.upload(local_path=os.path.join(os.path.dirname(__file__), "resources/file_13.11.2014_.txt")) ftp.upload(local_path=os.path.join(os.path.dirname(__file__), "resources/file_14.11.2014_.txt")) # upload file to HDFS/ create directories hdfs_file = HDFS("{0}/raw/12.11.2014".format(BASE_DIR)) hdfs_file.create(directory=True) local_file = LocalFS(path=os.path.join(os.path.dirname(__file__), 'resources/file_12.11.2014_.txt')) local_file.copy_to_hdfs(hdfs_path="{0}/raw/12.11.2014".format(BASE_DIR)) hdfs_file = HDFS("{0}/raw/13.11.2014".format(BASE_DIR)) hdfs_file.create(directory=True) local_file = LocalFS(path=os.path.join(os.path.dirname(__file__), 'resources/file_13.11.2014_.txt')) local_file.copy_to_hdfs(hdfs_path="{0}/raw/13.11.2014".format(BASE_DIR)) # create empty local directory 'tmp' in folder 'resources' local_file = LocalFS(path=os.path.join(os.path.dirname(__file__), 'resources/tmp')) if local_file.exists(): local_file.delete_directory() local_file.create(directory=True) # create HIVE external table with partition hive = Hive.load_queries_from_file(path=os.path.join(os.path.dirname(__file__), "resources/script.hql")) hive.run()