def test_apply_local_fs_snapshot(self): _config_file = os.path.join(os.path.dirname(__file__), 'resources', 'bootsrap', 'bootstrap.ini') test_dir = LocalFS('/tmp/data_tmp') if test_dir.exists(): test_dir.delete_directory() try: metastore = IniFileMetaStore(file=_config_file) _config = Configuration.load(metastore) apply_localfs_snapshot(_config) self.assertTrue(test_dir.exists(), "Folder was not created") finally: test_dir.delete_directory()
def test_broker(self): shell.execute_shell_command('fuser -k -n tcp {0}'.format(PORT)) local = LocalFS("/tmp/kafka-test") if not local.exists(): local.create_directory() thread = KafkaThreadBroker() thread.daemon = True thread.start() sleep(TIME) cmd = shell.execute_shell_command('netstat -lntu') self.assertTrue("9010" in cmd.stdout, cmd.stdout) local.delete_directory() shell.execute_shell_command('fuser -k -n tcp {0}'.format(PORT))
def test_merge(self): basedir = os.path.dirname(os.path.realpath(__file__)) local = LocalFS(os.path.join(basedir, "resources", "test_merge")) hdfs_file = HDFS(os.path.join("/tmp", str(uuid.uuid4()))) merged_file = LocalFS(os.path.join(basedir, "resources", "merged.txt")) try: local.copy_to_hdfs(hdfs_file.path) self.assertTrue(hdfs_file.exists(), "Local file was not copied to HDFS") hdfs_file.merge(merged_file.path) self.assertTrue(merged_file.exists(), "merged file was not copied to local fs") finally: hdfs_file.delete_directory() merged_file.delete()
if __name__ == "__main__": hdfs_file = HDFS("{0}/raw".format(BASE_DIR)) if hdfs_file.exists(): hdfs_file.delete(recursive=True) config = RawConfigParser() config.read(os.path.join(os.path.dirname(__file__), "resources/ftp_config.ini")) host_download = config.get("ftp", "host.download") user_name = config.get("ftp", "user.name") password = config.get("ftp", "password") path = config.get("ftp", "path") ftp = ftp_client(host=host_download, login=user_name, password=password, path="/tmp") if ftp.exists(): ftp.delete(recursive=True) local_file = LocalFS(path=os.path.join(os.path.dirname(__file__), 'resources/tmp')) if local_file.exists(): local_file.delete_directory() hive = Hive.load_queries_from_string(query="DROP DATABASE IF EXISTS hive_monitoring CASCADE;") hive.run()
def on_flow_failed(context): local_file = LocalFS( path=os.path.join(os.path.dirname(__file__), 'resources/tmp')) if local_file.exists(): local_file.delete_directory()
def on_flow_failed(context): local_file = LocalFS(path=os.path.join(os.path.dirname(__file__), 'resources/tmp')) if local_file.exists(): local_file.delete_directory()