async def async_executor(local_cluster_url):
pool = AsyncAdapter.make_pool()
sync_executor = await sync_to_async(partial(DaskJobExecutor.connect, scheduler_uri=local_cluster_url), pool=pool)
executor = AsyncAdapter(wrapped=sync_executor, pool=pool)
yield executor
await executor.close()
async def async_executor():
spec = cluster_spec(cpus=[0, 1], cudas=[], has_cupy=False)
pool = AsyncAdapter.make_pool()
sync_executor = await sync_to_async(partial(DaskJobExecutor.make_local, spec=spec), pool=pool)
executor = AsyncAdapter(wrapped=sync_executor, pool=pool)
yield executor
await executor.close()
async def test_prime_cache(
shared_state, default_raw, base_url, http_client, server_port, local_cluster_url,
default_token,
):
# first, connect to get the state
await create_connection(base_url, http_client, local_cluster_url, default_token)
executor = InlineJobExecutor()
pool = AsyncAdapter.make_pool()
executor = AsyncAdapter(wrapped=executor, pool=pool)
conn_details = {
'connection': {
'type': 'local',
'numWorkers': 1,
'cudas': [],
}
}
await shared_state.executor_state.set_executor(executor, conn_details)
raw_path = default_raw._path
uuid = "ae5d23bd-1f2a-4c57-bab2-dfc59a1219f3"
ds_url = "{}/api/datasets/{}/?token={}".format(
base_url, uuid, default_token,
)
ds_data = _get_raw_params(raw_path)
# connect to ws endpoint:
ws_url = f"ws://127.0.0.1:{server_port}/api/events/?token={default_token}"
async with websockets.connect(ws_url) as ws:
initial_msg = json.loads(await ws.recv())
assert_msg(initial_msg, 'INITIAL_STATE')
async with http_client.put(ds_url, json=ds_data) as resp:
assert resp.status == 200
resp_json = await resp.json()
assert_msg(resp_json, 'CREATE_DATASET')
async with websockets.connect(ws_url) as ws:
initial_msg = json.loads(await ws.recv())
assert_msg(initial_msg, 'INITIAL_STATE')
assert initial_msg["jobs"] == []
assert len(initial_msg["datasets"]) == 1
assert initial_msg["datasets"][0]["id"] == uuid
assert initial_msg["datasets"][0]["params"] == {
'sig_shape': [128, 128],
"enable_direct": False,
'dtype': 'float32',
'path': raw_path,
'nav_shape': [16, 16],
'shape': [16, 16, 128, 128],
'type': 'RAW',
'sync_offset': 0
}
assert len(initial_msg["datasets"][0]["diagnostics"]) == 6
async def put(self):
# TODO: extract json request data stuff into mixin?
request_data = tornado.escape.json_decode(self.request.body)
connection = request_data['connection']
pool = AsyncAdapter.make_pool()
if connection["type"].lower() == "tcp":
try:
sync_executor = await sync_to_async(partial(
DaskJobExecutor.connect,
scheduler_uri=connection['address'],
),
pool=pool)
except Exception as e:
msg = Message(self.state).cluster_conn_error(msg=str(e))
log_message(msg)
self.write(msg)
return None
elif connection["type"].lower() == "local":
devices = detect()
options = {"local_directory": self.state.get_local_directory()}
if "numWorkers" in connection:
devices["cpus"] = range(connection["numWorkers"])
devices["cudas"] = connection.get("cudas", [])
sync_executor = await sync_to_async(partial(
DaskJobExecutor.make_local,
spec=cluster_spec(**devices,
options=options,
preload=self.state.get_preload())),
pool=pool)
else:
raise ValueError("unknown connection type")
executor = AsyncAdapter(wrapped=sync_executor, pool=pool)
await self.state.executor_state.set_executor(executor, request_data)
await self.state.dataset_state.verify()
datasets = await self.state.dataset_state.serialize_all()
msg = Message(self.state).initial_state(
jobs=self.state.job_state.serialize_all(),
datasets=datasets,
analyses=self.state.analysis_state.serialize_all(),
compound_analyses=self.state.compound_analysis_state.serialize_all(
),
)
log_message(msg)
# FIXME: don't broadcast, only send to the websocket that matches this HTTP connection
# (is this even possible?)
self.event_registry.broadcast_event(msg)
await self.engine.send_existing_job_results()
self.write({
"status": "ok",
"connection": connection,
})
async def put(self):
# TODO: extract json request data stuff into mixin?
request_data = tornado.escape.json_decode(self.request.body)
connection = request_data['connection']
if connection["type"].lower() == "tcp":
sync_executor = await sync_to_async(
partial(
DaskJobExecutor.connect,
scheduler_uri=connection['address'],
))
elif connection["type"].lower() == "local":
cluster_kwargs = {
"threads_per_worker": 1,
}
if "numWorkers" in connection:
cluster_kwargs.update({"n_workers": connection["numWorkers"]})
sync_executor = await sync_to_async(
partial(DaskJobExecutor.make_local,
cluster_kwargs=cluster_kwargs))
else:
raise ValueError("unknown connection type")
executor = AsyncAdapter(wrapped=sync_executor)
await self.data.set_executor(executor, request_data)
await self.data.verify_datasets()
datasets = await self.data.serialize_datasets()
msg = Message(self.data).initial_state(
jobs=self.data.serialize_jobs(),
datasets=datasets,
)
log_message(msg)
self.event_registry.broadcast_event(msg)
self.write({
"status": "ok",
"connection": connection,
})
async def test_clust_default(hdf5_ds_2, tmpdir_factory, inline_executor):
datadir = tmpdir_factory.mktemp('template_tests')
conn = {'connection': {'type': 'local'}}
path = hdf5_ds_2.path
dataset = _get_hdf5_params(path)
analysis = [{
"analysisType": 'CLUST',
"parameters": {
'n_peaks': 42,
'n_clust': 7,
'cy': 3,
'cx': 3,
'ri': 1,
'ro': 5,
'delta': 0.05,
'min_dist': 1,
}
}]
notebook = notebook_generator(conn, dataset, analysis, save=True)
notebook = io.StringIO(notebook.getvalue())
nb = nbformat.read(notebook, as_version=4)
ep = ExecutePreprocessor(timeout=600)
out = ep.preprocess(nb, {"metadata": {"path": datadir}})
data_path = os.path.join(datadir, 'cluster_result.npy')
results = np.load(data_path)
executor = AsyncAdapter(wrapped=inline_executor)
analysis = ClusterAnalysis(dataset=hdf5_ds_2,
parameters={
'n_peaks': 42,
'n_clust': 7,
'cy': 3,
'cx': 3,
'ri': 1,
'ro': 5,
'delta': 0.05,
'min_dist': 1,
})
uuid = 'bd3b39fb-0b34-4a45-9955-339da6501bbb'
res_container = ResultContainer()
async def send_results(results, finished):
pass
await analysis.controller(
cancel_id=uuid,
executor=executor,
job_is_cancelled=lambda: False,
send_results=res_container,
)
expected = res_container.results
assert np.allclose(results, expected['intensity'].raw_data)
def get_libertem_executor(self):
cores = psutil.cpu_count(logical=False)
if cores is None:
cores = 2
executor = DaskJobExecutor.make_local(cluster_kwargs={
"threads_per_worker": 1,
"n_workers": cores
})
return AsyncAdapter(wrapped=executor)
async def async_executor():
cluster_kwargs = {
"threads_per_worker": 1,
"n_workers": 2,
}
sync_executor = await sync_to_async(
functools.partial(DaskJobExecutor.make_local,
cluster_kwargs=cluster_kwargs))
executor = AsyncAdapter(wrapped=sync_executor)
yield executor
await executor.close()
async def put(self):
# TODO: extract json request data stuff into mixin?
request_data = tornado.escape.json_decode(self.request.body)
connection = request_data['connection']
if connection["type"].lower() == "tcp":
try:
sync_executor = await sync_to_async(partial(DaskJobExecutor.connect,
scheduler_uri=connection['address'],
))
except Exception as e:
msg = Message(self.state).cluster_conn_error(msg=str(e))
log_message(msg)
self.write(msg)
return None
elif connection["type"].lower() == "local":
devices = detect()
options = {
"local_directory": self.state.get_local_directory()
}
if "numWorkers" in connection:
devices["cpus"] = range(connection["numWorkers"])
# Deactivate GPU support in local cluster until GUI allows deactivation
# to not interfere with other applications using the GPU
# FIXME implement GUI interface https://github.com/LiberTEM/LiberTEM/issues/803
devices["cudas"] = []
sync_executor = await sync_to_async(
partial(
DaskJobExecutor.make_local,
spec=cluster_spec(**devices, options=options),
)
)
else:
raise ValueError("unknown connection type")
executor = AsyncAdapter(wrapped=sync_executor)
await self.state.executor_state.set_executor(executor, request_data)
await self.state.dataset_state.verify()
datasets = await self.state.dataset_state.serialize_all()
msg = Message(self.state).initial_state(
jobs=self.state.job_state.serialize_all(),
datasets=datasets, analyses=self.state.analysis_state.serialize_all(),
compound_analyses=self.state.compound_analysis_state.serialize_all(),
)
log_message(msg)
# FIXME: don't broadcast, only send to the websocket that matches this HTTP connection
# (is this even possible?)
self.event_registry.broadcast_event(msg)
await self.send_existing_job_results()
self.write({
"status": "ok",
"connection": connection,
})
async def put(self):
# TODO: extract json request data stuff into mixin?
request_data = tornado.escape.json_decode(self.request.body)
connection = request_data['connection']
if connection["type"].lower() == "tcp":
try:
sync_executor = await sync_to_async(
partial(
DaskJobExecutor.connect,
scheduler_uri=connection['address'],
))
except Exception as e:
msg = Message(self.state).cluster_conn_error(msg=str(e))
log_message(msg)
self.write(msg)
return None
elif connection["type"].lower() == "local":
cluster_kwargs = {
"threads_per_worker": 1,
"local_directory": self.state.get_local_directory()
}
client_kwargs = {'set_as_default': False}
if "numWorkers" in connection:
cluster_kwargs.update({"n_workers": connection["numWorkers"]})
sync_executor = await sync_to_async(
partial(DaskJobExecutor.make_local,
cluster_kwargs=cluster_kwargs,
client_kwargs=client_kwargs))
else:
raise ValueError("unknown connection type")
executor = AsyncAdapter(wrapped=sync_executor)
await self.state.executor_state.set_executor(executor, request_data)
await self.state.dataset_state.verify()
datasets = await self.state.dataset_state.serialize_all()
msg = Message(self.state).initial_state(
jobs=self.state.job_state.serialize_all(),
datasets=datasets,
analyses=self.state.analysis_state.serialize_all(),
compound_analyses=self.state.compound_analysis_state.serialize_all(
),
)
log_message(msg)
# FIXME: don't broadcast, only send to the websocket that matches this HTTP connection
# (is this even possible?)
self.event_registry.broadcast_event(msg)
await self.send_existing_job_results()
self.write({
"status": "ok",
"connection": connection,
})
async def test_cluster_analysis(inline_executor):
data = np.zeros([16, 16, 8, 8]).astype(np.float32)
data[:, 2, 2] = 7
# adding strong non-zero order diffraction peaks for 0:3 frames
data[0:3, 0, 0] = 2
data[0:3, 4, 4] = 2
# adding weak non-zero order diffraction peaks for 0:3 frames
data[3:6, 2, 0] = 1
data[3:6, 2, 4] = 1
dataset = MemoryDataSet(data=data,
tileshape=(1, 8, 8),
num_partitions=1,
sig_dims=2)
executor = AsyncAdapter(wrapped=inline_executor)
analysis = ClusterAnalysis(dataset=dataset,
parameters={
'n_peaks': 23,
'n_clust': 7,
'cy': 3,
'cx': 3,
'ri': 1,
'ro': 5,
'delta': 0.05,
'min_dist': 1,
})
uuid = 'bd3b39fb-0b34-4a45-9955-339da6501bbb'
async def send_results(results, finished):
pass
await analysis.controller(
cancel_id=uuid,
executor=executor,
job_is_cancelled=lambda: False,
send_results=send_results,
)
async def aexecutor():
sync_executor = DaskJobExecutor.make_local()
executor = AsyncAdapter(wrapped=sync_executor)
yield executor
await executor.close()
def get_libertem_executor(self):
executor = DaskJobExecutor.make_local()
return AsyncAdapter(wrapped=executor)
