def run_sub_move(future, component, sub_move):
"""
Perform the sub moveAbs using the given component and axis->pos dict
:param future: cancellable future of the whole move
:param component: Either the stage or the align component
:param sub_move: the sub_move axis->pos dict
:raises TimeoutError: if the sub move timed out
:raises CancelledError: if the sub move is cancelled
"""
try:
with future._task_lock:
if future._task_state == CANCELLED:
logging.info(
"Move procedure is cancelled before moving {} -> {}.".
format(component.name, sub_move))
raise CancelledError()
logging.debug("Performing sub move {} -> {}".format(
component.name, sub_move))
future._running_subf = component.moveAbs(sub_move)
future._running_subf.result(timeout=MAX_SUBMOVE_DURATION)
except TimeoutError:
future._running_subf.cancel()
logging.exception("Timed out during moving {} -> {}.".format(
component.name, sub_move))
raise
if future._task_state == CANCELLED:
logging.info(
"Move procedure is cancelled after moving {} -> {}.".format(
component.name, sub_move))
raise CancelledError()
def _wait_for_done(self, timeout):
"""
Will not return until either timeout expires or future becomes "done".
There is one potential deadlock situation here:
The deadlock occurs if we await_result while at the same
time, this future needs to await_result from another future
---> To be safe, don't use await_result() in a Qt slot...
"""
if self.cancelled():
raise CancelledError("Future was cancelled") # pragma: no-cover
if not self.done():
self.timer_timeout = None
if (timeout is not None) and timeout > 0:
self._timer_timeout = MainThreadTimer(timeout * 1000)
self._timer_timeout.timeout.connect(self._exit_loop)
self._timer_timeout.start()
self.loop = QtCore.QEventLoop()
self.add_done_callback(self._exit_loop)
self.loop.exec_()
if self._timer_timeout is not None:
if not self._timer_timeout.isActive():
return TimeoutError("Timeout occured") # pragma: no-cover
else:
self._timer_timeout.stop()
def cancel_pending_requests(self):
"""Cancel all pending requests."""
exception = CancelledError()
for request, event in self._requests.values():
event.result = exception
event.set()
self._requests.clear()
async def handle_request(self, request, write_callback, stream_callback):
"""Take a request from the HTTP Server and return a response object
to be sent back The HTTP Server only expects a response object, so
exception handling must be done here
:param request: HTTP Request object
:param write_callback: Synchronous response function to be
called with the response as the only argument
:param stream_callback: Coroutine that handles streaming a
StreamingHTTPResponse if produced by the handler.
:return: Nothing
"""
# Define `response` var here to remove warnings about
# allocation before assignment below.
cancelled = False
try:
request.app = self
# -------------------------------------------- #
# Execute Handler
# -------------------------------------------- #
# Fetch handler from router
handler, uri = self.router.get(request.path, request.method)
request.uri_template = uri
response = handler(request)
logger.info("got response from function")
res = await response
body = res.body
headers = res.headers
status = res.status
response = HTTPResponse(
body=body,
status=status,
headers=headers,
)
except CancelledError:
response = None
cancelled = True
except Exception as e:
if isinstance(e, AsyncHTTPException):
response = self.error_handler.default(request=request,
exception=e)
elif self.debug:
response = HTTPResponse(
"Error while handling error: {}\nStack: {}".format(
e, format_exc()),
status=502,
)
else:
response = HTTPResponse(
"An error occurred while handling an error", status=502)
finally:
if cancelled:
raise CancelledError()
if isinstance(response, StreamingHTTPResponse):
await stream_callback(response)
else:
write_callback(response)
def read(self, n=-1):
if self._cancel_event and self._cancel_event.is_set():
raise CancelledError('The upload was cancelled.')
chunk = io.BytesIO.read(self, n)
self._progress += int(len(chunk))
if self._callback:
self._callback(self._progress/self._len)
return chunk
async def cancel_after(sleep):
await asyncio.sleep(sleep)
pid = os.getpid()
try:
os.kill(pid, SIGINT)
except KeyboardInterrupt:
# As an effect of the SIGINT killing the process might receive
# here a KeyboardInterrupt exception. This is Ok.
pass
return CancelledError()
def _doCryoTiltSample(future, stage, focus, rx, rz):
"""
Do the actual switching procedure for the Cryo sample stage between imaging and tilting
:param future: cancellable future of the move
:param stage: sample stage that's being controlled
:param focus: focus for optical lens
:param rx: (float) rotation movement in x axis
:param rz: (float) rotation movement in z axis
"""
try:
stage_md = stage.getMetadata()
focus_md = focus.getMetadata()
stage_active = stage_md[model.MD_FAV_POS_ACTIVE]
stage_active_range = stage_md[model.MD_POS_ACTIVE_RANGE]
focus_deactive = focus_md[model.MD_FAV_POS_DEACTIVE]
current_pos = stage.position.value
# Check that the stage X,Y,Z are within the limits
if not _isInRange(current_pos, stage_active_range, {'x', 'y', 'z'}):
raise ValueError("Current position is out of active range.")
# To hold the ordered sub moves list to perform the tilting/imaging move
sub_moves = []
# Park focus only if stage rx and rz are equal to 0
# Otherwise stop if it's not already parked
if not _isNearPosition(focus.position.value, focus_deactive, {'z'}):
if _isNearPosition(current_pos, {'rx': 0, 'rz': 0}, {'rx', 'rz'}):
sub_moves.append((focus, focus_deactive))
else:
raise ValueError(
"Cannot proceed with tilting while focus is not near FAV_POS_DEACTIVE position."
)
if rx == 0 and rz == 0: # Imaging
# Get the actual Imaging position (which should be ~ 0 as well)
rx = stage_active['rx']
rz = stage_active['rz']
sub_moves.append((stage, {'rz': rz}))
sub_moves.append((stage, {'rx': rx}))
else:
sub_moves.append((stage, {'rx': rx}))
if rz is not None:
sub_moves.append((stage, {'rz': rz}))
for component, sub_move in sub_moves:
run_sub_move(future, component, sub_move)
except CancelledError:
logging.info("_doCryoTiltSample cancelled.")
except Exception as exp:
logging.exception("Failure to move to position rx={}, rz={}.".format(
rx, rz))
raise
finally:
with future._task_lock:
if future._task_state == CANCELLED:
raise CancelledError()
future._task_state = FINISHED
def run_reference(future, component):
"""
Perform the stage reference procedure
:param future: cancellable future of the reference procedure
:param component: Either the stage or the align component
:raises CancelledError: if the reference is cancelled
"""
try:
with future._task_lock:
if future._task_state == CANCELLED:
logging.info("Reference procedure is cancelled.")
raise CancelledError()
logging.debug("Performing stage referencing.")
future._running_subf = component.reference(set(component.axes.keys()))
future._running_subf.result()
except Exception as error:
logging.exception(error)
if future._task_state == CANCELLED:
logging.info("Reference procedure is cancelled.")
raise CancelledError()
def future_result_exc(future):
'''Return a two elements tuple containing the future result and exception.
The :class:`.Future` must be ``done``
'''
if future.cancelled():
return None, CancelledError()
elif future._exception:
return None, future.exception()
else:
return future.result(), None
def cancel(self, msg: typing.Optional[str] = None) -> None:
"""Cancel a Task's further execution.
When a Task is cancelled, a :exc:`asyncio.CancelledError` is thrown into the Task.
"""
self._cancelled = CancelledError(msg)
warnings.warn(
"Calling this method will cause a CancelledError to be thrown in the "
"Task sometime in the future.",
FutureWarning,
stacklevel=2,
)
self.kill()
def result(self):
"""Return the result this future represents.
If the future has been cancelled, raises CancelledError. If the
future's result isn't yet available, raises InvalidStateError. If
the future is done and has an exception set, this exception is raised.
"""
if self._state == _CANCELLED:
raise CancelledError()
if self._state != _FINISHED:
raise InvalidStateError('Result is not ready.')
if self._exception is not None:
raise self._exception
return self._result
def cancel(self, exception=True):
if exception:
exc = CancelledError()
else:
exc = None
def _set_result():
if not self._stop_future.done():
self._stop_future.set_result(exc)
# Ensure __next__ ends. Sometimes the loop is already closing, so the exit result may not be written
# to the queue
self._q.put((exc, True))
if not self._loop.is_closed():
self._loop.call_soon_threadsafe(_set_result)
def _doCryoTiltSample(future, stage, align, rx, rz):
"""
Do the actual tilting procedure for the Cryo sample stage
:param future: cancellable future of the move
:param stage: sample stage that's being controlled
:param align: aligner for optical lens
:param rx: (float) rotation movement in x axis
:param rz: (float) rotation movement in z axis
"""
try:
stage_md = stage.getMetadata()
align_md = align.getMetadata()
stage_active_range = stage_md[model.MD_POS_ACTIVE_RANGE]
align_deactive = align_md[model.MD_FAV_POS_DEACTIVE]
current_pos = stage.position.value
# Check that the stage X,Y,Z are within the limits
if not _isInRange(current_pos, stage_active_range, {'x', 'y', 'z'}):
raise ValueError("Current position is out of active range.")
# To hold the ordered sub moves list to perform the tilting/imaging move
sub_moves = []
# Park aligner to safe position before any movement
if not _isNearPosition(align.position.value, align_deactive,
align.axes):
cryoSwitchAlignPosition(LOADING).result()
sub_moves.append((stage, {'rz': rz}))
sub_moves.append((stage, {'rx': rx}))
for component, sub_move in sub_moves:
run_sub_move(future, component, sub_move)
except CancelledError:
logging.info("_doCryoTiltSample cancelled.")
except Exception as exp:
logging.exception("Failure to move to position rx={}, rz={}.".format(
rx, rz))
raise
finally:
with future._task_lock:
if future._task_state == CANCELLED:
raise CancelledError()
future._task_state = FINISHED
async def test_delete_pod_on_cancelled_error_during_waiting_pod_ready(
future, run_test_server, aiohttp_test_client, k8s_pod):
app, server = await run_test_server()
client = aiohttp_test_client(server)
webdriver_request = {"desiredCapabilities": {"browserName": "chrome"}}
app[consts.SESSION_USE_CASE_KEY].k8s_service.create_pod = mock.Mock(
return_value=future(k8s_pod()))
app[consts.
SESSION_USE_CASE_KEY].k8s_service.wait_until_pod_is_ready = mock.Mock(
side_effect=CancelledError())
app[consts.SESSION_USE_CASE_KEY].k8s_service.delete_pod = mock.Mock(
return_value=future())
resp = await client.post('/api/v1/session', json=webdriver_request)
assert resp.status == web.HTTPInternalServerError.status_code
assert len(app[consts.SESSION_USE_CASE_KEY].state_service.sessions) == 0
app[consts.SESSION_USE_CASE_KEY].k8s_service.delete_pod.assert_called_once(
)
async def test_base_exception(self) -> None:
tasks = tuple(
map(
ensure_future,
[
sleep_then_return(0.1, 1),
sleep_then_return(0.2, 2),
sleep_then_return(0.3, 3),
sleep_then_return(0.4, 4),
],
))
with self.assertRaisesRegex(BaseException, "###test###"):
await safe_gather(sleep_then_fail(0, BaseException("###test###")),
*tasks)
with self.assertRaises(CancelledError):
await safe_gather(sleep_then_fail(0, CancelledError()), *tasks)
self.assertTrue(all(map(lambda fut: not fut.done(), tasks)))
self.assertListEqual(await safe_gather(*tasks), [1, 2, 3, 4])
def onTaskDone(task):
"""Task done handler to publish complete (success) & critical (fail) events"""
try:
err = task.exception()
if (err):
self.emit("critical", err)
else:
result = task.result()
self.emit("complete", result)
except CancelledError as err:
self.emit(
"warning",
CancelledError("onTaskDone called after task cancelled").
with_traceback(err.__traceback__))
LOGGER.warning("Shouldn't see onTaskDone when cancelled??")
except InvalidStateError as err:
self.emit(
"error",
InvalidStateError(
"onTaskDone called before task finished: Risk of zombie task"
).with_traceback(err.__traceback__))
def _error_back(fut):
if fut._exception:
callback(fut.exception())
elif fut.cancelled():
callback(CancelledError())
async def process_response(request, response):
raise CancelledError("CancelledError at response middleware")
def _doCryoLoadSample(future, stage, focus, target):
"""
Do the actual switching procedure for the Cryo sample stage between loading and imaging
:param future: cancellable future of the move
:param stage: sample stage that's being controlled
:param focus: focus for optical lens
:param target: target position either one of the constants LOADING or IMAGING
"""
try:
stage_md = stage.getMetadata()
focus_md = focus.getMetadata()
stage_active = stage_md[model.MD_FAV_POS_ACTIVE]
stage_deactive = stage_md[model.MD_FAV_POS_DEACTIVE]
stage_coating = stage_md[model.MD_FAV_POS_COATING]
stage_active_range = stage_md[model.MD_POS_ACTIVE_RANGE]
focus_deactive = focus_md[model.MD_FAV_POS_DEACTIVE]
current_pos = stage.position.value
# To hold the ordered sub moves list
sub_moves = []
# Create axis->pos dict from target position given smaller number of axes
filter_dict = lambda keys, d: {key: d[key] for key in keys}
# Initial submove for all procedures is to park focus if it's not already parked
if not _isNearPosition(focus.position.value, focus_deactive, {'z'}):
sub_moves.append((focus, focus_deactive))
if target == LOADING:
if getCurrentPositionLabel(current_pos, stage) is UNKNOWN:
logging.warning(
"Moving stage to loading while current position is unknown."
)
if abs(stage_deactive['rx']) > ATOL_ROTATION_POS:
raise ValueError(
"Absolute value of rx for FAV_POS_DEACTIVE is greater than {}"
.format(ATOL_ROTATION_POS))
# Add the sub moves to perform the loading move
sub_moves.append((stage, filter_dict({'rx', 'rz'},
stage_deactive)))
sub_moves.append((stage, filter_dict({'x', 'y'}, stage_deactive)))
sub_moves.append((stage, filter_dict({'z'}, stage_deactive)))
elif target in (IMAGING, COATING):
if not _isInRange(current_pos, stage_active_range,
{'x', 'y', 'z'}) and not _isNearPosition(
current_pos, stage_deactive, stage.axes):
raise ValueError(
"Current position is out of active range and not near FAV_POS_DEACTIVE position."
)
focus_active = focus_md[model.MD_FAV_POS_ACTIVE]
target_pos = stage_active if target is IMAGING else stage_coating
# Add the sub moves to perform the imaging/coating move
sub_moves.append((stage, filter_dict({'z'}, target_pos)))
sub_moves.append((stage, filter_dict({'x', 'y'}, target_pos)))
sub_moves.append((stage, filter_dict({'rx', 'rz'}, target_pos)))
# TODO: check if the following movement is necessary as it could be done later, only when the user start
# the FM stream (in which case it’d be handled by the optical path manager)
if target == IMAGING:
sub_moves.append((focus, focus_active))
else:
raise ValueError("Unknown target value %s." % target)
for component, sub_move in sub_moves:
run_sub_move(future, component, sub_move)
except CancelledError:
logging.info("_doCryoLoadSample cancelled.")
except Exception as exp:
target_str = {
LOADING: "loading",
IMAGING: "imaging",
COATING: "coating"
}
logging.exception("Failure to move to {} position.".format(
target_str.get(target, lambda: "unknown")))
raise
finally:
with future._task_lock:
if future._task_state == CANCELLED:
raise CancelledError()
future._task_state = FINISHED
def as_exception(future):
if future._exception:
return future.exception()
elif future.cancelled():
return CancelledError()
def check_cancelled(self):
if self.cancelled():
raise CancelledError()
async def try_point(self, job):
try:
encounter_only = HARDCORE_HYPERDRIVE
if job in ENCOUNTER_CACHE:
return
point = (job['lat'], job['lon'])
encounter_id = job['encounter_id']
spawn_id = job['spawn_id']
expire_timestamp = job['expire_timestamp']
if time() >= expire_timestamp - 30.0:
raise LateEncounterSkippedError(
"Insufficient time, {}s left".format(
int(expire_timestamp - time())))
spawn_time = spawns.spawn_timestamps.get(spawn_id, 0)
point = randomize_point(point,
amount=0.00003) # jitter around 3 meters
skip_time = monotonic() + conf.GIVE_UP_KNOWN
worker = await self.best_worker(point, skip_time)
if job in ENCOUNTER_CACHE:
return
if not worker:
raise RetriableEncounterSkippedError("Unavailable worker")
async with worker.busy:
if spawn_time:
worker.after_spawn = time() - spawn_time
if conf.LV30_MAX_SPEED and not HARDCORE_HYPERDRIVE:
if await worker.sleep_travel_time(
point, max_speed=conf.LV30_MAX_SPEED):
if job in ENCOUNTER_CACHE:
return
ENCOUNTER_CACHE.remove(job['encounter_id'], job['spawn_id'])
visit_result = await worker.visit(
point,
encounter_id=encounter_id,
spawn_id=spawn_id,
encounter_only=encounter_only,
sighting=job)
if visit_result == -1:
self.hash_burn += 1
await sleep(1.0, loop=LOOP)
point = randomize_point(
point, amount=0.00001) # jitter around 3 meters
ENCOUNTER_CACHE.remove(job['encounter_id'],
job['spawn_id'])
visit_result = await worker.visit(
point,
encounter_id=encounter_id,
spawn_id=spawn_id,
encounter_only=encounter_only,
sighting=job)
if visit_result:
if visit_result == -1:
if sb_detector:
sb_detector.add_encounter_miss(worker.account)
raise RetriableEncounterSkippedError(
"Pokemon not seen. Possibly shadow banned.")
else:
if not encounter_only:
self.visits += 1
else:
raise RetriableEncounterSkippedError(
"Encounter request failed due to unknown error.")
except CancelledError:
raise
except RetriableEncounterSkippedError as e:
try:
if 'retry' in job:
if job['retry'] > 1:
raise EncounterSkippedError('{} retry failures'.format(
job['retry']))
else:
job['retry'] += 1
else:
job['retry'] = 1
if self.overseer.running:
Worker30.add_job(job)
if worker:
log.info(
"Requeued encounter {} by {} for retry attemp #{}",
encounter_id, worker.username, job['retry'])
else:
log.info("Requeued encounter {} for retry attemp #{}",
encounter_id, job['retry'])
else:
raise CancelledError("Overseer is no longer running")
except Exception as e:
self.skipped += 1
ENCOUNTER_CACHE.add(job)
job['check_duplicate'] = True
db_proc.add(job)
if worker:
username = worker.username
else:
username = "******"
log.info("Skipping encounter {} by {} due to error: {}",
encounter_id, username, e)
except (LateEncounterSkippedError, EncounterSkippedError) as e:
if isinstance(e, LateEncounterSkippedError):
self.lates += 1
self.skipped += 1
ENCOUNTER_CACHE.add(job)
job['check_duplicate'] = True
db_proc.add(job)
log.info("Skipping encounter {} due to error: {}", encounter_id, e)
except Exception as e:
log.error('An exception occurred in try_point: {}', e)
finally:
self.coroutine_semaphore.release()
def test_job():
task = Task()
task_context = Mock()
task_context.dependencies = Mock()
task_context.pkg = Mock()
task_context.pkg.name = 'name'
job = Job(identifier='id',
dependencies=set(),
task=task,
task_context=task_context)
assert str(job) == 'id'
events = []
event_queue = Mock()
event_queue.put = lambda event: events.append(event)
job.set_event_queue(event_queue)
assert len(events) == 1
assert isinstance(events[-1][0], JobQueued)
assert events[-1][0].identifier == 'name'
assert events[-1][0].dependencies == task_context.dependencies
assert events[-1][1] == job
# successful task
rc = run_until_complete(job())
assert rc == 0
assert len(events) == 3
assert isinstance(events[-2][0], JobStarted)
assert events[-2][0].identifier == 'name'
assert events[-2][1] == job
assert isinstance(events[-1][0], JobEnded)
assert events[-1][0].identifier == 'name'
assert events[-1][0].rc == 0
assert events[-1][1] == job
# canceled task
job.returncode = None
task.return_value = CancelledError()
rc = run_until_complete(job())
assert rc is SIGINT_RESULT
assert len(events) == 5
assert isinstance(events[-2][0], JobStarted)
assert events[-2][0].identifier == 'name'
assert events[-2][1] == job
assert isinstance(events[-1][0], JobEnded)
assert events[-1][0].identifier == 'name'
assert events[-1][0].rc is SIGINT_RESULT
assert events[-1][1] == job
# task raising exception
job.returncode = None
task.return_value = RuntimeError('custom exception')
with pytest.raises(RuntimeError):
run_until_complete(job())
assert len(events) == 8
assert isinstance(events[-3][0], JobStarted)
assert events[-3][0].identifier == 'name'
assert events[-3][1] == job
assert isinstance(events[-2][0], StderrLine)
assert events[-2][0].line.endswith(b'\nRuntimeError: custom exception\n')
assert events[-2][1] == job
assert isinstance(events[-1][0], JobEnded)
assert events[-1][0].identifier == 'name'
assert events[-1][0].rc == 1
assert events[-1][1] == job
# override task return code
job.returncode = 2
task.return_value = 0
rc = run_until_complete(job())
assert rc == 2
assert len(events) == 10
assert isinstance(events[-2][0], JobStarted)
assert events[-2][0].identifier == 'name'
assert events[-2][1] == job
assert isinstance(events[-1][0], JobEnded)
assert events[-1][0].identifier == 'name'
assert events[-1][0].rc == 2
assert events[-1][1] == job
def _doCryoSwitchSamplePosition(future, target):
"""
Do the actual switching procedure for cryoSwitchSamplePosition
:param future: cancellable future of the move
:param target: (int) target position either one of the constants: LOADING, IMAGING,
ALIGNMENT, COATING, LOADING_PATH, MILLING, SEM_IMAGING, FM_IMAGING.
"""
role = model.getMicroscope().role
try:
target_name = POSITION_NAMES[target]
except KeyError:
raise ValueError(f"Unknown target '{target}'")
# Create axis->pos dict from target position given smaller number of axes
filter_dict = lambda keys, d: {key: d[key] for key in keys}
try:
if role == "enzel":
# get the stage and aligner objects
stage = model.getComponent(role='stage')
align = model.getComponent(role='align')
stage_md = stage.getMetadata()
align_md = align.getMetadata()
target_pos = {
LOADING:
stage_md[model.MD_FAV_POS_DEACTIVE],
IMAGING:
stage_md[model.MD_FAV_POS_ACTIVE],
COATING:
stage_md[model.MD_FAV_POS_COATING],
ALIGNMENT:
stage_md[model.MD_FAV_POS_ALIGN],
SEM_IMAGING:
stage_md[model.MD_FAV_POS_SEM_IMAGING],
THREE_BEAMS:
get3beamsSafePos(stage_md[model.MD_FAV_POS_ACTIVE],
SAFETY_MARGIN_5DOF)
}
align_deactive = align_md[model.MD_FAV_POS_DEACTIVE]
stage_referenced = all(stage.referenced.value.values())
# Fail early when required axes are not found on the positions metadata
required_axes = {'x', 'y', 'z', 'rx', 'rz'}
for stage_position in target_pos.values():
if not required_axes.issubset(stage_position.keys()):
raise ValueError(
"Stage %s metadata does not have all required axes %s."
% (list(stage_md.keys())[list(
stage_md.values()).index(stage_position)],
required_axes))
current_pos = stage.position.value
# To hold the sub moves to run if normal ordering failed
fallback_submoves = [{'x', 'y', 'z'}, {'rx', 'rz'}]
current_label = getCurrentPositionLabel(current_pos, stage)
current_name = POSITION_NAMES[current_label]
if target == LOADING:
if current_label is UNKNOWN and stage_referenced:
logging.warning(
"Moving stage to loading while current position is unknown."
)
if abs(target_pos[LOADING]['rx']) > ATOL_ROTATION_POS:
raise ValueError(
"Absolute value of rx for FAV_POS_DEACTIVE is greater than {}"
.format(ATOL_ROTATION_POS))
# Check if stage is not referenced:
# park aligner (move it to loading position) then reference the stage
if not stage_referenced:
future._running_subf = cryoSwitchAlignPosition(LOADING)
try:
future._running_subf.result(timeout=60)
except TimeoutError:
future._running_subf.cancel()
if future._task_state == CANCELLED:
logging.info("Cancelling aligner movement...")
raise CancelledError()
run_reference(future, stage)
# Add the sub moves to perform the loading move
if current_label is UNKNOWN and not stage_referenced:
# After referencing the stage could move near the maximum axes range,
# and moving single axes may result in an invalid/reachable position error,
# so all linear axes will be moved together for this special case.
sub_moves = [{'x', 'y', 'z'}, {'rx', 'rz'}]
else:
# Notes on the movement on the typical case:
# - Moving each linear axis separately to be easily trackable by the user from the chamber cam.
# - Moving X first is a way to move it to a safe position, as it's not affected by the rx
# (and rz is typically always 0). Moreover, X is the largest move, and so it'll be
# "around" the loading position.
# - The X/Y/Z movement is in the Rx referential. So if the rx is tilted (eg, we are in IMAGING),
# and Y/Z are far from the pivot point, we have a good chance of hitting something.
# Moving along X should always be safe (as Rx is not affected by this axis position).
sub_moves = [{'x'}, {'y'}, {'z'}, {'rx', 'rz'}]
elif target in (ALIGNMENT, IMAGING, SEM_IMAGING, COATING,
THREE_BEAMS):
if current_label is LOADING:
# Automatically run the referencing procedure as part of the
# first step of the movement loading → imaging/coating position
run_reference(future, stage)
elif current_label is UNKNOWN:
raise ValueError(
f"Unable to move to {target_name} while current position is unknown."
)
# Add the sub moves to perform the imaging/coating/alignment/sem_imaging moves
# Essentially the same/reverse as for going to LOADING: do the small movements first near
# the loading position, and end with the large x move to get close to the pole-piece.
# TODO: test if coating position needs a different ordering
if current_label == LOADING:
# As moving from loading position requires re-referencing the stage, move linked axes (y & z)
# together to prevent invalid/reachable position error
sub_moves = [{'y', 'z'}, {'rx', 'rz'}, {'x'}]
else:
sub_moves = [{'z'}, {'y'}, {'rx', 'rz'}, {'x'}]
else:
raise ValueError(f"Unsupported move to target {target_name}")
try:
logging.info(
"Starting sample movement from {} -> {}...".format(
current_name, target_name))
# Park aligner to safe position before any movement
if not _isNearPosition(align.position.value, align_deactive,
align.axes):
future._running_subf = cryoSwitchAlignPosition(LOADING)
try:
future._running_subf.result(timeout=60)
except TimeoutError:
future._running_subf.cancel()
if future._task_state == CANCELLED:
logging.info("Cancelling aligner movement...")
raise CancelledError()
for sub_move in sub_moves:
sub_move_dict = filter_dict(sub_move, target_pos[target])
logging.debug("Moving %s to %s.", stage.name,
sub_move_dict)
run_sub_move(future, stage, sub_move_dict)
if target in (IMAGING, ALIGNMENT, THREE_BEAMS):
future._running_subf = cryoSwitchAlignPosition(target)
try:
future._running_subf.result(timeout=60)
except TimeoutError:
future._running_subf.cancel()
if future._task_state == CANCELLED:
logging.info("Cancelling aligner movement...")
raise CancelledError()
except IndexError:
# In case the required movement is invalid/unreachable with the smaract 5dof stage
# Move all linear axes first then rotational ones using the fallback_submoves
logging.debug(
"This move {} is unreachable, trying to move all axes at once..."
.format(sub_move))
for sub_move in fallback_submoves:
sub_move_dict = filter_dict(sub_move, target_pos[target])
logging.debug("Moving %s to %s.", stage.name, sub_move)
run_sub_move(future, stage, sub_move_dict)
elif role == "meteor":
# get the focus and stage components
focus = model.getComponent(role='focus')
stage = model.getComponent(role='stage-bare')
# get the meta data
focus_md = focus.getMetadata()
focus_deactive = focus_md[model.MD_FAV_POS_DEACTIVE]
focus_active = focus_md[model.MD_FAV_POS_ACTIVE]
# To hold the ordered sub moves list
sub_moves = []
# get the current label
current_label = getCurrentPositionLabel(stage.position.value,
stage)
current_name = POSITION_NAMES[current_label]
if current_label == target:
logging.warning(
f"Requested move to the same position as current: {target_name}"
)
# get the set point position
target_pos = getTargetPosition(target, stage)
# If at some "weird" position, it's quite unsafe. We consider the targets
# LOADING and SEM_IMAGING safe to go. So if not going there, first pass
# by SEM_IMAGING and then go to the actual requested position.
if current_label == UNKNOWN:
logging.warning(
"Moving stage while current position is unknown.")
if target not in (LOADING, SEM_IMAGING):
logging.debug("Moving first to SEM_IMAGING position")
target_pos_sem = getTargetPosition(SEM_IMAGING, stage)
if not _isNearPosition(focus.position.value,
focus_deactive, focus.axes):
sub_moves.append((focus, focus_deactive))
sub_moves.append(
(stage, filter_dict({'x', 'y', 'z'}, target_pos_sem)))
sub_moves.append(
(stage, filter_dict({'rx', 'rz'}, target_pos_sem)))
if target in (GRID_1, GRID_2):
# The current mode doesn't change. Only X/Y/Z should move (typically
# only X/Y).
sub_moves.append(
(stage, filter_dict({'x', 'y', 'z'}, target_pos)))
elif target in (LOADING, SEM_IMAGING, FM_IMAGING):
# Park the focuser for safety
if not _isNearPosition(focus.position.value, focus_deactive,
focus.axes):
sub_moves.append((focus, focus_deactive))
# Move translation axes, then rotational ones
sub_moves.append(
(stage, filter_dict({'x', 'y', 'z'}, target_pos)))
sub_moves.append((stage, filter_dict({'rx', 'rz'},
target_pos)))
if target == FM_IMAGING:
# Engage the focuser
sub_moves.append((focus, focus_active))
else:
raise ValueError(f"Unsupported move to target {target_name}")
# run the moves
logging.info("Moving from position {} to position {}.".format(
current_name, target_name))
for component, sub_move in sub_moves:
logging.debug("Moving %s to %s.", component.name, sub_move)
run_sub_move(future, component, sub_move)
except CancelledError:
logging.info("CryoSwitchSamplePosition cancelled.")
except Exception:
logging.exception(
"Failure to move to {} position.".format(target_name))
raise
finally:
with future._task_lock:
if future._task_state == CANCELLED:
raise CancelledError()
future._task_state = FINISHED
def _doCryoSwitchAlignPosition(future, align, target):
"""
Do the actual switching procedure for the Cryo lens stage (align) between loading, imaging and alignment positions
:param future: cancellable future of the move
:param align: wrapper for optical objective (lens aligner)
:param target: target position either one of the constants LOADING, IMAGING and ALIGNMENT
"""
try:
target_name = POSITION_NAMES[target]
except KeyError:
raise ValueError(f"Unknown target '{target}'")
try:
align_md = align.getMetadata()
target_pos = {
LOADING:
align_md[model.MD_FAV_POS_DEACTIVE],
IMAGING:
align_md[model.MD_FAV_POS_ACTIVE],
ALIGNMENT:
align_md[model.MD_FAV_POS_ALIGN],
THREE_BEAMS:
get3beamsSafePos(align_md[model.MD_FAV_POS_ACTIVE],
SAFETY_MARGIN_3DOF)
}
align_referenced = all(align.referenced.value.values())
# Fail early when required axes are not found on the positions metadata
required_axes = {'x', 'y', 'z'}
for align_position in target_pos.values():
if not required_axes.issubset(align_position.keys()):
raise ValueError(
"Aligner %s metadata does not have all required axes %s." %
(list(align_md.keys())[list(
align_md.values()).index(align_position)],
required_axes))
current_pos = align.position.value
# To hold the ordered sub moves list
sub_moves = []
# Create axis->pos dict from target position given smaller number of axes
filter_dict = lambda keys, d: {key: d[key] for key in keys}
current_label = getCurrentAlignerPositionLabel(current_pos, align)
current_name = POSITION_NAMES[current_label]
if target == LOADING:
if current_label is UNKNOWN:
logging.warning(
"Parking aligner while current position is unknown.")
# reference align if not already referenced
if not align_referenced:
run_reference(future, align)
# Add the sub moves to perform the loading move
# NB: moving Z axis downward first so when aligner Y move (
# compensating 3DOF Y&Z) upwards it doesn't hit the 5DOF
sub_moves = [{'x'}, {'z'}, {'y'}]
elif target in (ALIGNMENT, IMAGING, THREE_BEAMS):
if current_label is UNKNOWN:
raise ValueError(
"Unable to move aligner to {} while current position is unknown."
.format(target_name))
# Add the sub moves to perform the imaging/alignment move
# Moving Y axis first downwards so Z move upwards it doesn't hit the 5DOF stage
sub_moves = [{'y'}, {'z'}, {'x'}]
else:
raise ValueError("Unknown target value %s." % target)
logging.info("Starting aligner movement from {} -> {}...".format(
current_name, target_name))
for sub_move in sub_moves:
run_sub_move(future, align,
filter_dict(sub_move, target_pos[target]))
except CancelledError:
logging.info("_doCryoSwitchAlignPosition cancelled.")
except Exception:
logging.exception(
"Failure to move to {} position.".format(target_name))
raise
finally:
with future._task_lock:
if future._task_state == CANCELLED:
raise CancelledError()
future._task_state = FINISHED
def _doCryoSwitchAlignPosition(future, align, target):
"""
Do the actual switching procedure for the Cryo lens stage (align) between loading, imaging and alignment positions
:param future: cancellable future of the move
:param align: wrapper for optical objective (lens aligner)
:param target: target position either one of the constants LOADING, IMAGING and ALIGNMENT
"""
try:
align_md = align.getMetadata()
target_pos = {
LOADING: align_md[model.MD_FAV_POS_DEACTIVE],
IMAGING: align_md[model.MD_FAV_POS_ACTIVE],
ALIGNMENT: align_md[model.MD_FAV_POS_ALIGN]
}
align_referenced = all(align.referenced.value.values())
# Fail early when required axes are not found on the positions metadata
required_axes = {'x', 'y', 'z'}
for align_position in target_pos.values():
if not required_axes.issubset(align_position.keys()):
raise ValueError(
"Aligner %s metadata does not have all required axes %s." %
(list(align_md.keys())[list(
align_md.values()).index(align_position)],
required_axes))
current_pos = align.position.value
# To hold the ordered sub moves list
sub_moves = []
# Create axis->pos dict from target position given smaller number of axes
filter_dict = lambda keys, d: {key: d[key] for key in keys}
current_label = getCurrentAlignerPositionLabel(current_pos, align)
if target == LOADING:
if current_label is UNKNOWN:
logging.warning(
"Parking aligner while current position is unknown.")
# reference align if not already referenced
if not align_referenced:
run_reference(future, align)
# Add the sub moves to perform the loading move
sub_moves.append((align, filter_dict({'y'}, target_pos[LOADING])))
sub_moves.append(
(align, filter_dict({'x', 'z'}, target_pos[LOADING])))
elif target in (ALIGNMENT, IMAGING):
if current_label is UNKNOWN:
raise ValueError(
"Unable to move aligner to {} while current position is unknown."
.format(target_pos_str.get(target, lambda: "unknown")))
# Add the sub moves to perform the imaging/alignment move
sub_moves.append((align, filter_dict({'x', 'z'},
target_pos[target])))
sub_moves.append((align, filter_dict({'y'}, target_pos[target])))
else:
raise ValueError("Unknown target value %s." % target)
for component, sub_move in sub_moves:
logging.info("Starting aligner movement from {} -> {}...".format(
target_pos_str[current_label], target_pos_str[target]))
run_sub_move(future, component, sub_move)
except CancelledError:
logging.info("_doCryoSwitchAlignPosition cancelled.")
except Exception as exp:
logging.exception("Failure to move to {} position.".format(
target_pos_str.get(target, lambda: "unknown")))
raise
finally:
with future._task_lock:
if future._task_state == CANCELLED:
raise CancelledError()
future._task_state = FINISHED
async def very_slow_func():
await asyncio.sleep(2)
raise CancelledError()
def cancel(self):
"""
Cancel any tasks waiting on this wait.
"""
self._result.set_exception(CancelledError("Wait was cancelled"))
async def handle_request(self, request, write_callback, stream_callback):
"""Take a request from the HTTP Server and return a response object
to be sent back The HTTP Server only expects a response object, so
exception handling must be done here
:param request: HTTP Request object
:param write_callback: Synchronous response function to be
called with the response as the only argument
:param stream_callback: Coroutine that handles streaming a
StreamingHTTPResponse if produced by the handler.
:return: Nothing
"""
# Define `response` var here to remove warnings about
# allocation before assignment below.
response = None
cancelled = False
name = None
try:
# Fetch handler from router
handler, args, kwargs, uri, name, endpoint = self.router.get(
request)
# -------------------------------------------- #
# Request Middleware
# -------------------------------------------- #
response = await self._run_request_middleware(request,
request_name=name)
# No middleware results
if not response:
# -------------------------------------------- #
# Execute Handler
# -------------------------------------------- #
request.uri_template = uri
if handler is None:
raise ServerError(
("'None' was returned while requesting a "
"handler from the router"))
request.endpoint = endpoint
# Run response handler
response = handler(request, *args, **kwargs)
if isawaitable(response):
response = await response
except CancelledError:
# If response handler times out, the server handles the error
# and cancels the handle_request job.
# In this case, the transport is already closed and we cannot
# issue a response.
response = None
cancelled = True
except Exception as e:
# -------------------------------------------- #
# Response Generation Failed
# -------------------------------------------- #
try:
response = self.error_handler.response(request, e)
if isawaitable(response):
response = await response
except Exception as e:
if isinstance(e, SanicException):
response = self.error_handler.default(request=request,
exception=e)
elif self.debug:
response = HTTPResponse(
f"Error while "
f"handling error: {e}\nStack: {format_exc()}",
status=500,
)
else:
response = HTTPResponse(
"An error occurred while handling an error",
status=500)
finally:
# -------------------------------------------- #
# Response Middleware
# -------------------------------------------- #
# Don't run response middleware if response is None
if response is not None:
try:
response = await self._run_response_middleware(
request, response, request_name=name)
except CancelledError:
# Response middleware can timeout too, as above.
response = None
cancelled = True
except BaseException:
error_logger.exception(
"Exception occurred in one of response "
"middleware handlers")
if cancelled:
raise CancelledError()
# pass the response to the correct callback
if write_callback is None or isinstance(response,
StreamingHTTPResponse):
if stream_callback:
await stream_callback(response)
else:
# Should only end here IF it is an ASGI websocket.
# TODO:
# - Add exception handling
pass
else:
write_callback(response)
def _doCryoSwitchSamplePosition(future, stage, align, target):
"""
Do the actual switching procedure for the Cryo sample stage between loading, imaging and coating positions
:param future: cancellable future of the move
:param stage: sample stage that's being controlled
:param align: focus for optical lens
:param target: target position either one of the constants LOADING, IMAGING, ALIGNMENT and COATING
"""
try:
stage_md = stage.getMetadata()
align_md = align.getMetadata()
target_pos = {
LOADING: stage_md[model.MD_FAV_POS_DEACTIVE],
IMAGING: stage_md[model.MD_FAV_POS_ACTIVE],
COATING: stage_md[model.MD_FAV_POS_COATING],
ALIGNMENT: stage_md[model.MD_FAV_POS_ALIGN],
SEM_IMAGING: stage_md[model.MD_FAV_POS_SEM_IMAGING]
}
align_deactive = align_md[model.MD_FAV_POS_DEACTIVE]
stage_referenced = all(stage.referenced.value.values())
# Fail early when required axes are not found on the positions metadata
required_axes = {'x', 'y', 'z', 'rx', 'rz'}
for stage_position in target_pos.values():
if not required_axes.issubset(stage_position.keys()):
raise ValueError(
"Stage %s metadata does not have all required axes %s." %
(list(stage_md.keys())[list(
stage_md.values()).index(stage_position)],
required_axes))
current_pos = stage.position.value
# To hold the ordered sub moves list
sub_moves = []
# To hold the sub moves to run if the ordered sub moves failed
fallback_submoves = []
# Create axis->pos dict from target position given smaller number of axes
filter_dict = lambda keys, d: {key: d[key] for key in keys}
current_label = getCurrentPositionLabel(current_pos, stage)
if target == LOADING:
if current_label is UNKNOWN and stage_referenced:
logging.warning(
"Moving stage to loading while current position is unknown."
)
if abs(target_pos[LOADING]['rx']) > ATOL_ROTATION_POS:
raise ValueError(
"Absolute value of rx for FAV_POS_DEACTIVE is greater than {}"
.format(ATOL_ROTATION_POS))
# Check if stage is not referenced:
# park aligner (move it to loading position) then reference the stage
if not stage_referenced:
cryoSwitchAlignPosition(LOADING).result()
run_reference(future, stage)
fallback_submoves.append(
(stage, filter_dict({'x', 'y', 'z'}, target_pos[LOADING])))
fallback_submoves.append(
(stage, filter_dict({'rx', 'rz'}, target_pos[LOADING])))
# Add the sub moves to perform the loading move
sub_moves.append(
(stage, filter_dict({'rx', 'rz'}, target_pos[LOADING])))
if current_label is UNKNOWN and not stage_referenced:
# After referencing the stage could move near the maximum axes range,
# and moving single axes may result in an invalid/reachable position error,
# so all axes will moved together for this special case.
sub_moves.append(
(stage, filter_dict({'x', 'y', 'z'}, target_pos[LOADING])))
else:
sub_moves.append(
(stage, filter_dict({'x', 'y'}, target_pos[LOADING])))
sub_moves.append((stage, filter_dict({'z'},
target_pos[LOADING])))
elif target in (ALIGNMENT, IMAGING, SEM_IMAGING, COATING):
if current_label is LOADING:
# Automatically run the referencing procedure as part of the
# first step of the movement loading → imaging/coating position
run_reference(future, stage)
elif current_label is UNKNOWN:
raise ValueError(
"Unable to move to {} while current position is unknown.".
format(target_pos_str.get(target, lambda: "unknown")))
fallback_submoves.append(
(stage, filter_dict({'x', 'y', 'z'}, target_pos[target])))
fallback_submoves.append(
(stage, filter_dict({'rx', 'rz'}, target_pos[target])))
# Add the sub moves to perform the imaging/coating move
if current_label == LOADING:
# As moving from loading position requires re-referencing the stage, move all axes together to
# prevent invalid/reachable position error
sub_moves.append(
(stage, filter_dict({'x', 'y', 'z'}, target_pos[target])))
else:
sub_moves.append((stage, filter_dict({'z'},
target_pos[target])))
sub_moves.append(
(stage, filter_dict({'x', 'y'}, target_pos[target])))
sub_moves.append(
(stage, filter_dict({'rx', 'rz'}, target_pos[target])))
else:
raise ValueError("Unknown target value %s." % target)
try:
logging.info("Starting sample movement from {} -> {}...".format(
target_pos_str[current_label], target_pos_str[target]))
# Park aligner to safe position before any movement
if not _isNearPosition(align.position.value, align_deactive,
align.axes):
cryoSwitchAlignPosition(LOADING).result()
for component, sub_move in sub_moves:
run_sub_move(future, component, sub_move)
if target in (IMAGING, ALIGNMENT):
cryoSwitchAlignPosition(target).result()
except IndexError:
# In case the required movement is invalid/unreachable with the smaract 5dof stage
# Move all linear axes first then rotational ones using the fallback_submoves
logging.debug(
"This move {} is unreachable, trying to move all axes at once..."
.format(sub_move))
for component, sub_move in fallback_submoves:
run_sub_move(future, component, sub_move)
except CancelledError:
logging.info("_doCryoSwitchSamplePosition cancelled.")
except Exception as exp:
logging.exception("Failure to move to {} position.".format(
target_pos_str.get(target, lambda: "unknown")))
raise
finally:
with future._task_lock:
if future._task_state == CANCELLED:
raise CancelledError()
future._task_state = FINISHED