def get_autoconfig_client(): """ Creates the client as specified in the `luigi.cfg` configuration. """ configured_client = hdfs_config.get_configured_hdfs_client() if configured_client == "snakebite": return hdfs_snakebite_client.SnakebiteHdfsClient() if configured_client == "snakebite_with_hadoopcli_fallback": return luigi.contrib.target.CascadingClient([ hdfs_snakebite_client.SnakebiteHdfsClient(), hdfs_hadoopcli_clients.create_hadoopcli_client() ]) if configured_client == "hadoopcli": return hdfs_hadoopcli_clients.create_hadoopcli_client() raise Exception("Unknown hdfs client " + hdfs_config.get_configured_hdfs_client())
def get_autoconfig_client(client_cache=_AUTOCONFIG_CLIENT): """ Creates the client as specified in the `luigi.cfg` configuration. """ try: return client_cache.client except AttributeError: configured_client = hdfs_config.get_configured_hdfs_client() if configured_client == "webhdfs": client_cache.client = hdfs_webhdfs_client.WebHdfsClient() elif configured_client == "snakebite": client_cache.client = hdfs_snakebite_client.SnakebiteHdfsClient() elif configured_client == "snakebite_with_hadoopcli_fallback": client_cache.client = luigi.contrib.target.CascadingClient([ hdfs_snakebite_client.SnakebiteHdfsClient(), hdfs_hadoopcli_clients.create_hadoopcli_client(), ]) elif configured_client == "hadoopcli": client_cache.client = hdfs_hadoopcli_clients.create_hadoopcli_client() else: raise Exception("Unknown hdfs client " + configured_client) return client_cache.client