def from_current(cls, asynchronous=False, loop=None):
"""Connect to an existing ``YarnCluster`` from inside the cluster.
Parameters
----------
asynchronous : bool, optional
If true, starts the cluster in asynchronous mode, where it can be
used in other async code.
loop : IOLoop, optional
The IOLoop instance to use. Defaults to the current loop in
asynchronous mode, otherwise a background loop is started.
Returns
-------
YarnCluster
"""
self = super(YarnCluster, cls).__new__(cls)
app_id = os.environ.get("DASK_APPLICATION_ID", None)
app_address = os.environ.get("DASK_APPMASTER_ADDRESS", None)
security_dir = os.environ.get("DASK_SECURITY_CREDENTIALS", None)
if app_id is not None and app_address is not None:
security = (None if security_dir is None else
skein.Security.from_directory(security_dir))
app = skein.ApplicationClient(app_address,
app_id,
security=security)
else:
app = skein.ApplicationClient.from_current()
self._init_common(application_client=app,
asynchronous=asynchronous,
loop=loop)
return self
def _get_app_client(self, cluster_info, cluster_state):
# TODO: maybe keep an LRU cache of these?
return skein.ApplicationClient(
cluster_state["app_address"],
cluster_state["app_id"],
security=self._get_security(cluster_info),
)
async def _get_app_client(self, cluster_state):
out = self._app_client_cache.get(self.cluster_name)
if out is None:
app_id = cluster_state["app_id"]
security = self._get_security()
if not self.app_address:
# Lookup and cache the application address
skein_client = await self._get_skein_client()
report = skein_client.application_report(app_id)
if report.state != "RUNNING": # pragma: nocover
raise ValueError("Application %s is not running" % app_id)
self.app_address = "%s:%d" % (report.host, report.port)
out = skein.ApplicationClient(self.app_address, app_id, security=security)
self._app_client_cache.put(self.cluster_name, out)
return out
def from_current(cls):
"""Connect to an existing ``YarnCluster`` from inside the cluster.
Returns
-------
YarnCluster
"""
self = super(YarnCluster, cls).__new__(cls)
app_id = os.environ.get('DASK_APPLICATION_ID', None)
app_address = os.environ.get('DASK_APPMASTER_ADDRESS', None)
if app_id is not None and app_address is not None:
app = skein.ApplicationClient(app_address, app_id)
else:
app = skein.ApplicationClient.from_current()
self._connect_existing(app)
return self
async def _get_app_client(self, cluster):
out = self.app_client_cache.get(cluster.name)
if out is None:
app_id = cluster.state["app_id"]
security = self._get_security(cluster)
if cluster.name not in self.app_address_cache:
# Lookup and cache the application address
report = self.skein_client.application_report(app_id)
if report.state != "RUNNING": # pragma: nocover
raise ValueError("Application %s is not running" % app_id)
app_address = "%s:%d" % (report.host, report.port)
self.app_address_cache[cluster.name] = app_address
app_address = self.app_address_cache[cluster.name]
out = skein.ApplicationClient(app_address, app_id, security=security)
self.app_client_cache.put(cluster.name, out)
return out
def from_current(cls):
"""Connect to an existing ``YarnCluster`` from inside the cluster.
Returns
-------
YarnCluster
"""
self = super(YarnCluster, cls).__new__(cls)
app_id = os.environ.get('DASK_APPLICATION_ID', None)
app_address = os.environ.get('DASK_APPMASTER_ADDRESS', None)
security_dir = os.environ.get('DASK_SECURITY_CREDENTIALS', None)
if app_id is not None and app_address is not None:
security = (None if security_dir is None
else skein.Security.from_directory(security_dir))
app = skein.ApplicationClient(app_address, app_id,
security=security)
else:
app = skein.ApplicationClient.from_current()
self._connect_existing(app)
return self