async def send_results(self, results, uuid, finished=False):
if self.data.job_is_cancelled(uuid):
raise JobCancelledError()
images = await result_images(results)
if self.data.job_is_cancelled(uuid):
raise JobCancelledError()
if finished:
msg = Message(self.data).finish_job(
job_id=uuid,
num_images=len(results),
image_descriptions=[{
"title": result.title,
"desc": result.desc
} for result in results],
)
else:
msg = Message(self.data).task_result(
job_id=uuid,
num_images=len(results),
image_descriptions=[{
"title": result.title,
"desc": result.desc
} for result in results],
)
log_message(msg)
# NOTE: make sure the following broadcast_event messages are sent atomically!
# (that is: keep the code below synchronous, and only send the messages
# once the images have finished encoding, and then send all at once)
futures = []
futures.append(self.event_registry.broadcast_event(msg))
for image in images:
raw_bytes = image.read()
futures.append(
self.event_registry.broadcast_event(raw_bytes, binary=True))
await asyncio.gather(*futures)
async def send_results(self,
results: AnalysisResultSet,
job_id,
analysis_id,
details,
finished=False):
if self.state.job_state.is_cancelled(job_id):
raise JobCancelledError()
images = await result_images(results)
if self.state.job_state.is_cancelled(job_id):
raise JobCancelledError()
if finished:
msg = Message(self.state).finish_job(
job_id=job_id,
num_images=len(results),
image_descriptions=[{
"title":
result.title,
"desc":
result.desc,
"includeInDownload":
result.include_in_download
} for result in results],
)
self.state.analysis_state.set_results(analysis_id, details,
results, job_id)
else:
msg = Message(self.state).task_result(
job_id=job_id,
num_images=len(results),
image_descriptions=[{
"title":
result.title,
"desc":
result.desc,
"includeInDownload":
result.include_in_download
} for result in results],
)
log_message(msg)
# NOTE: make sure the following broadcast_event messages are sent atomically!
# (that is: keep the code below synchronous, and only send the messages
# once the images have finished encoding, and then send all at once)
futures = []
futures.append(self.event_registry.broadcast_event(msg))
for image in images:
raw_bytes = image.read()
futures.append(
self.event_registry.broadcast_event(raw_bytes, binary=True))
await asyncio.gather(*futures)
async def run_sd_udf(self, roi, stddev_udf, executor, cancel_id, job_is_cancelled):
result_iter = UDFRunner([stddev_udf]).run_for_dataset_async(
self.dataset, executor, roi=roi, cancel_id=cancel_id
)
async for (sd_udf_results,) in result_iter:
pass
if job_is_cancelled():
raise JobCancelledError()
return roi, consolidate_result(sd_udf_results)
async def run_udf(self, job_id, dataset, dataset_id, analysis, analysis_id, details):
udf = analysis.get_udf()
roi = analysis.get_roi()
executor = self.state.executor_state.get_executor()
msg = Message(self.state).start_job(
job_id=job_id, analysis_id=analysis_id,
)
log_message(msg)
self.write(msg)
self.finish()
self.event_registry.broadcast_event(msg)
if hasattr(analysis, 'controller'):
return await analysis.controller(
cancel_id=job_id, executor=executor,
job_is_cancelled=lambda: self.state.job_state.is_cancelled(job_id),
send_results=lambda results, finished: self.send_results(
results, job_id, finished=finished,
details=details, analysis_id=analysis_id,
)
)
t = time.time()
post_t = time.time()
window = 0.3
# FIXME: allow to set correction data for a dataset via upload and local loading
corrections = dataset.get_correction_data()
result_iter = UDFRunner([udf]).run_for_dataset_async(
dataset, executor, roi=roi, cancel_id=job_id, corrections=corrections,
)
async for udf_results in result_iter:
window = min(max(window, 2*(t - post_t)), 5)
if time.time() - t < window:
continue
results = await sync_to_async(
analysis.get_udf_results,
udf_results=udf_results.buffers[0],
roi=roi,
damage=udf_results.damage
)
post_t = time.time()
await self.send_results(results, job_id, analysis_id, details)
# The broadcast might have taken quite some time due to
# backpressure from the network
t = time.time()
if self.state.job_state.is_cancelled(job_id):
raise JobCancelledError()
results = await sync_to_async(
analysis.get_udf_results,
udf_results=udf_results.buffers[0],
roi=roi,
damage=udf_results.damage
)
await self.send_results(results, job_id, analysis_id, details, finished=True)
async def run_udf(self, uuid, ds, analysis):
udf = analysis.get_udf()
roi = analysis.get_roi()
# FIXME: register_job for UDFs?
self.data.register_job(uuid=uuid, job=udf, dataset=ds)
# FIXME: code duplication
executor = self.data.get_executor()
msg = Message(self.data).start_job(
job_id=uuid,
)
log_message(msg)
self.write(msg)
self.finish()
self.event_registry.broadcast_event(msg)
if hasattr(analysis, 'controller'):
return await analysis.controller(
cancel_id=uuid, executor=executor,
job_is_cancelled=lambda: self.data.job_is_cancelled(uuid),
send_results=lambda results, finished: self.send_results(results, uuid,
finished=finished)
)
t = time.time()
post_t = time.time()
window = 0.3
result_iter = UDFRunner(udf).run_for_dataset_async(
ds, executor, roi=roi, cancel_id=uuid
)
async for udf_results in result_iter:
window = min(max(window, 2*(t - post_t)), 5)
if time.time() - t < window:
continue
results = await run_blocking(
analysis.get_udf_results,
udf_results=udf_results,
roi=roi,
)
post_t = time.time()
await self.send_results(results, uuid)
# The broadcast might have taken quite some time due to
# backpressure from the network
t = time.time()
if self.data.job_is_cancelled(uuid):
raise JobCancelledError()
results = await run_blocking(
analysis.get_udf_results,
udf_results=udf_results,
roi=roi,
)
await self.send_results(results, uuid, finished=True)
async def run_job(self, uuid, ds, analysis):
job = analysis.get_job()
full_result = job.get_result_buffer()
self.data.register_job(uuid=uuid, job=job, dataset=job.dataset)
executor = self.data.get_executor()
msg = Message(self.data).start_job(
job_id=uuid,
)
log_message(msg)
self.write(msg)
self.finish()
self.event_registry.broadcast_event(msg)
t = time.time()
post_t = time.time()
window = 0.3
async for result in executor.run_job(job, cancel_id=uuid):
for tile in result:
tile.reduce_into_result(full_result)
window = min(max(window, 2*(t - post_t)), 5)
if time.time() - t < window:
continue
post_t = time.time()
results = await run_blocking(
analysis.get_results,
job_results=full_result,
)
await self.send_results(results, uuid)
# The broadcast might have taken quite some time due to
# backpressure from the network
t = time.time()
if self.data.job_is_cancelled(uuid):
raise JobCancelledError()
results = await run_blocking(
analysis.get_results,
job_results=full_result,
)
await self.send_results(results, uuid, finished=True)
async def controller(self, cancel_id, executor, job_is_cancelled, send_results):
roi, sd_udf_results = await self.get_sd_results(executor, cancel_id, job_is_cancelled)
udf = self.get_cluster_udf(sd_udf_results)
result_iter = UDFRunner([udf]).run_for_dataset_async(
self.dataset, executor, cancel_id=cancel_id
)
async for (udf_results,) in result_iter:
pass
if job_is_cancelled():
raise JobCancelledError()
results = await run_blocking(
self.get_udf_results,
udf_results=udf_results,
roi=roi,
)
await send_results(results, True)
async def controller(self, cancel_id, executor, job_is_cancelled,
send_results):
stddev_udf = StdDevUDF()
roi = self.get_sd_roi()
result_iter = UDFRunner(stddev_udf).run_for_dataset_async(
self.dataset, executor, roi=roi, cancel_id=cancel_id)
async for sd_udf_results in result_iter:
pass
if job_is_cancelled():
raise JobCancelledError()
sd_udf_results['var'].data
sd_udf_results['num_frame'].data
sd_udf_results = dict(sd_udf_results.items())
sd_udf_results['var'] = sd_udf_results['var'].data / sd_udf_results[
'num_frame'].data
sd_udf_results['std'] = np.sqrt(sd_udf_results['var'].data)
sd_udf_results['mean'] = sd_udf_results[
'sum_frame'].data / sd_udf_results['num_frame'].data
sd_udf_results['num_frame'] = sd_udf_results['num_frame'].data
sd_udf_results['sum_frame'] = sd_udf_results['sum_frame'].data
center = (self.parameters["cy"], self.parameters["cx"])
rad_in = self.parameters["ri"]
rad_out = self.parameters["ro"]
delta = self.parameters["delta"]
n_peaks = self.parameters["n_peaks"]
min_dist = self.parameters["min_dist"]
savg = sd_udf_results['mean']
sstd = sd_udf_results['std']
sshape = sstd.shape
if not (center is None or rad_in is None or rad_out is None):
mask_out = 1 * _make_circular_mask(center[1], center[0], sshape[1],
sshape[0], rad_out)
mask_in = 1 * _make_circular_mask(center[1], center[0], sshape[1],
sshape[0], rad_in)
mask = mask_out - mask_in
masked_sstd = sstd * mask
else:
masked_sstd = sstd
coordinates = peak_local_max(masked_sstd,
num_peaks=n_peaks,
min_distance=min_dist)
udf = feature.FeatureVecMakerUDF(delta=delta,
savg=savg,
coordinates=coordinates)
result_iter = UDFRunner(udf).run_for_dataset_async(self.dataset,
executor,
cancel_id=cancel_id)
async for udf_results in result_iter:
pass
if job_is_cancelled():
raise JobCancelledError()
results = await run_blocking(
self.get_udf_results,
udf_results=udf_results,
roi=roi,
)
await send_results(results, True)
async def controller(self, cancel_id, executor, job_is_cancelled,
send_results):
stddev_udf = StdDevUDF()
roi = self.get_sd_roi()
result_iter = UDFRunner(stddev_udf).run_for_dataset_async(
self.dataset, executor, roi=roi, cancel_id=cancel_id)
async for sd_udf_results in result_iter:
pass
if job_is_cancelled():
raise JobCancelledError()
sd_udf_results = consolidate_result(sd_udf_results)
center = (self.parameters["cy"], self.parameters["cx"])
rad_in = self.parameters["ri"]
rad_out = self.parameters["ro"]
n_peaks = self.parameters["n_peaks"]
min_dist = self.parameters["min_dist"]
sstd = sd_udf_results['std']
sshape = sstd.shape
if not (center is None or rad_in is None or rad_out is None):
mask_out = 1 * _make_circular_mask(center[1], center[0], sshape[1],
sshape[0], rad_out)
mask_in = 1 * _make_circular_mask(center[1], center[0], sshape[1],
sshape[0], rad_in)
mask = mask_out - mask_in
masked_sstd = sstd * mask
else:
masked_sstd = sstd
coordinates = peak_local_max(masked_sstd,
num_peaks=n_peaks,
min_distance=min_dist)
y = coordinates[..., 0]
x = coordinates[..., 1]
z = range(len(y))
mask = sparse.COO(shape=(len(y), ) + tuple(self.dataset.shape.sig),
coords=(z, y, x),
data=1)
udf = ApplyMasksUDF(mask_factories=lambda: mask,
mask_count=len(y),
mask_dtype=np.uint8,
use_sparse=True)
result_iter = UDFRunner(udf).run_for_dataset_async(self.dataset,
executor,
cancel_id=cancel_id)
async for udf_results in result_iter:
pass
if job_is_cancelled():
raise JobCancelledError()
results = await run_blocking(
self.get_udf_results,
udf_results=udf_results,
roi=roi,
)
await send_results(results, True)