Python get_hdfs_client Examples

Example #1

File: impala_test_suite.py Project: apache/incubator-impala

 def create_hdfs_client(cls):
   if pytest.config.option.namenode_http_address is None:
     hdfs_client = get_hdfs_client_from_conf(HDFS_CONF)
   else:
     host, port = pytest.config.option.namenode_http_address.split(":")
     hdfs_client = get_hdfs_client(host, port)
   return hdfs_client

Example #2

    def setup_class(cls):
        """Setup section that runs before each test suite"""
        cls.hive_client, cls.client = [None, None]
        # Create a Hive Metastore Client (used for executing some test SETUP steps
        metastore_host, metastore_port = pytest.config.option.metastore_server.split(
            ':')
        trans_type = 'buffered'
        if pytest.config.option.use_kerberos:
            trans_type = 'kerberos'
        cls.hive_transport = create_transport(
            host=metastore_host,
            port=metastore_port,
            service=pytest.config.option.hive_service_name,
            transport_type=trans_type)
        protocol = TBinaryProtocol.TBinaryProtocol(cls.hive_transport)
        cls.hive_client = ThriftHiveMetastore.Client(protocol)
        cls.hive_transport.open()

        # Create a connection to Impala.
        cls.client = cls.create_impala_client(IMPALAD)

        cls.impalad_test_service = ImpaladService(IMPALAD.split(':')[0])
        if pytest.config.option.namenode_http_address is None:
            cls.hdfs_client = get_hdfs_client_from_conf(HDFS_CONF)
        else:
            host, port = pytest.config.option.namenode_http_address.split(":")
            cls.hdfs_client = get_hdfs_client()

Example #3

 def create_hdfs_client(cls):
     if pytest.config.option.namenode_http_address is None:
         hdfs_client = get_hdfs_client_from_conf(HDFS_CONF)
     else:
         host, port = pytest.config.option.namenode_http_address.split(":")
         hdfs_client = get_hdfs_client(host, port)
     return hdfs_client

Example #4

File: test_insert_behaviour.py Project: 1ack/Impala

 def setup_class(cls):
   if pytest.config.option.namenode_http_address is None:
     hdfs_conf = HdfsConfig(pytest.config.option.minicluster_xml_conf)
     cls.hdfs_client = get_hdfs_client_from_conf(hdfs_conf)
   else:
     host, port = pytest.config.option.namenode_http_address.split(":")
     cls.hdfs_client = get_hdfs_client()

Example #5

File: test_insert_behaviour.py Project: dralves/impala-kudu

 def setup_class(cls):
     if pytest.config.option.namenode_http_address is None:
         hdfs_conf = HdfsConfig(pytest.config.option.minicluster_xml_conf)
         cls.hdfs_client = get_hdfs_client_from_conf(hdfs_conf)
     else:
         host, port = pytest.config.option.namenode_http_address.split(":")
         cls.hdfs_client = get_hdfs_client(host, port)

Example #6

File: impala_test_suite.py Project: nkhuyu/Impala

    def setup_class(cls):
        """Setup section that runs before each test suite"""
        cls.hive_client, cls.client = [None, None]
        # Create a Hive Metastore Client (used for executing some test SETUP steps
        metastore_host, metastore_port = pytest.config.option.metastore_server.split(":")
        trans_type = "buffered"
        if pytest.config.option.use_kerberos:
            trans_type = "kerberos"
        cls.hive_transport = create_transport(
            host=metastore_host,
            port=metastore_port,
            service=pytest.config.option.hive_service_name,
            transport_type=trans_type,
        )
        protocol = TBinaryProtocol.TBinaryProtocol(cls.hive_transport)
        cls.hive_client = ThriftHiveMetastore.Client(protocol)
        cls.hive_transport.open()

        # Create a connection to Impala.
        cls.client = cls.create_impala_client(IMPALAD)

        cls.impalad_test_service = ImpaladService(IMPALAD.split(":")[0])
        if pytest.config.option.namenode_http_address is None:
            cls.hdfs_client = get_hdfs_client_from_conf(HDFS_CONF)
        else:
            host, port = pytest.config.option.namenode_http_address.split(":")
            cls.hdfs_client = get_hdfs_client()

Example #7

File: test_insert_behaviour.py Project: Accagain2014/incubator-impala

 def setup_class(cls):
     super(TestInsertBehaviourCustomCluster, cls).setup_class()
     if pytest.config.option.namenode_http_address is None:
         hdfs_conf = HdfsConfig(pytest.config.option.minicluster_xml_conf)
         cls.hdfs_client = get_hdfs_client_from_conf(hdfs_conf)
     else:
         host, port = pytest.config.option.namenode_http_address.split(":")
         cls.hdfs_client = get_hdfs_client(host, port)

Example #8

File: test_insert_behaviour.py Project: mbrukman/apache-impala

 def setup_class(cls):
   super(TestInsertBehaviourCustomCluster, cls).setup_class()
   if pytest.config.option.namenode_http_address is None:
     hdfs_conf = HdfsConfig(pytest.config.option.minicluster_xml_conf)
     cls.hdfs_client = get_hdfs_client_from_conf(hdfs_conf)
   else:
     host, port = pytest.config.option.namenode_http_address.split(":")
     cls.hdfs_client = get_hdfs_client(host, port)

Example #9

File: db_connector.py Project: plusplusjiajia/recordservice

 def end_bulk_load_table(self, create_tables):
   DbConnection.end_bulk_load_table(self, create_tables)
   if self.hdfs_host is None:
     hdfs = create_default_hdfs_client()
   else:
     hdfs = get_hdfs_client(self.hdfs_host, self.hdfs_port, user_name='hdfs')
   pywebhdfs_dirname = dirname(self.hdfs_file_path).lstrip('/')
   hdfs.make_dir(pywebhdfs_dirname)
   pywebhdfs_file_path = pywebhdfs_dirname + '/' + basename(self.hdfs_file_path)
   try:
     # TODO: Only delete the file if it exists
     hdfs.delete_file_dir(pywebhdfs_file_path)
   except Exception as e:
     LOG.debug(e)
   with open(self._bulk_load_data_file.name) as readable_file:
     hdfs.create_file(pywebhdfs_file_path, readable_file)
   self._bulk_load_data_file.close()
   if self._bulk_load_non_text_table:
     if create_tables:
       self.create_table(self._bulk_load_non_text_table)
     self.execute('INSERT INTO TABLE %s SELECT * FROM %s'
         % (self._bulk_load_non_text_table.name, self._bulk_load_table.name))
     self.drop_table(self._bulk_load_table.name)
   self._bulk_load_data_file = None