Пример #1
0
def get_autoconfig_client(show_warnings=True):
    """
    Creates the client as specified in the `client.cfg` configuration.
    """
    configured_client = hdfs_config.get_configured_hdfs_client(show_warnings=show_warnings)
    if configured_client == "snakebite":
        return hdfs_snakebite_client.SnakebiteHdfsClient()
    if configured_client == "snakebite_with_hadoopcli_fallback":
        return luigi.contrib.target.CascadingClient([hdfs_snakebite_client.SnakebiteHdfsClient(),
                                                     hdfs_hadoopcli_clients.create_hadoopcli_client()])
    if configured_client == "hadoopcli":
        return hdfs_hadoopcli_clients.create_hadoopcli_client()
    raise Exception("Unknown hdfs client " + hdfs_config.get_configured_hdfs_client())
Пример #2
0
def get_autoconfig_client():
    """
    Creates the client as specified in the `luigi.cfg` configuration.
    """
    configured_client = hdfs_config.get_configured_hdfs_client()
    if configured_client == "snakebite":
        return hdfs_snakebite_client.SnakebiteHdfsClient()
    if configured_client == "snakebite_with_hadoopcli_fallback":
        return luigi.contrib.target.CascadingClient([
            hdfs_snakebite_client.SnakebiteHdfsClient(),
            hdfs_hadoopcli_clients.create_hadoopcli_client()
        ])
    if configured_client == "hadoopcli":
        return hdfs_hadoopcli_clients.create_hadoopcli_client()
    raise Exception("Unknown hdfs client " +
                    hdfs_config.get_configured_hdfs_client())
Пример #3
0
def get_autoconfig_client(client_cache=_AUTOCONFIG_CLIENT):
    """
    Creates the client as specified in the `luigi.cfg` configuration.
    """
    try:
        return client_cache.client
    except AttributeError:
        configured_client = hdfs_config.get_configured_hdfs_client()
        if configured_client == "webhdfs":
            client_cache.client = hdfs_webhdfs_client.WebHdfsClient()
        elif configured_client == "hadoopcli":
            client_cache.client = hdfs_hadoopcli_clients.create_hadoopcli_client(
            )
        else:
            raise Exception("Unknown hdfs client " + configured_client)
        return client_cache.client
Пример #4
0
def get_autoconfig_client(client_cache=_AUTOCONFIG_CLIENT):
    """
    Creates the client as specified in the `luigi.cfg` configuration.
    """
    try:
        return client_cache.client
    except AttributeError:
        configured_client = hdfs_config.get_configured_hdfs_client()
        if configured_client == "webhdfs":
            client_cache.client = hdfs_webhdfs_client.WebHdfsClient()
        elif configured_client == "snakebite":
            client_cache.client = hdfs_snakebite_client.SnakebiteHdfsClient()
        elif configured_client == "snakebite_with_hadoopcli_fallback":
            client_cache.client = luigi.contrib.target.CascadingClient([
                hdfs_snakebite_client.SnakebiteHdfsClient(),
                hdfs_hadoopcli_clients.create_hadoopcli_client(),
            ])
        elif configured_client == "hadoopcli":
            client_cache.client = hdfs_hadoopcli_clients.create_hadoopcli_client()
        else:
            raise Exception("Unknown hdfs client " + configured_client)
        return client_cache.client
Пример #5
0
def get_autoconfig_client(client_cache=_AUTOCONFIG_CLIENT):
    """
    Creates the client as specified in the `luigi.cfg` configuration.
    """
    try:
        return client_cache.client
    except AttributeError:
        configured_client = hdfs_config.get_configured_hdfs_client()
        if configured_client == "webhdfs":
            client_cache.client = hdfs_webhdfs_client.WebHdfsClient()
        elif configured_client == "snakebite":
            client_cache.client = hdfs_snakebite_client.SnakebiteHdfsClient()
        elif configured_client == "snakebite_with_hadoopcli_fallback":
            client_cache.client = luigi.contrib.target.CascadingClient([
                hdfs_snakebite_client.SnakebiteHdfsClient(),
                hdfs_hadoopcli_clients.create_hadoopcli_client(),
            ])
        elif configured_client == "hadoopcli":
            client_cache.client = hdfs_hadoopcli_clients.create_hadoopcli_client()
        else:
            raise Exception("Unknown hdfs client " + configured_client)
        return client_cache.client