def get_nested_dict_from_shot(filepath):
row = runmanager.get_shot_globals(filepath)
with h5py.File(filepath, 'r') as h5_file:
# if 'data' in h5_file:
# for groupname in h5_file['data']:
# resultsgroup = h5_file['data'][groupname]
# if 'camera' in dict(resultsgroup.attrs).keys():
# row[groupname] = h5_file['data'][groupname]['Raw'][:]
if 'results' in h5_file:
for groupname in h5_file['results']:
resultsgroup = h5_file['results'][groupname]
row[groupname] = dict(resultsgroup.attrs)
if 'images' in h5_file:
for orientation in h5_file['images'].keys():
if isinstance(h5_file['images'][orientation], h5py.Group):
row[orientation] = dict(
h5_file['images'][orientation].attrs)
for label in h5_file['images'][orientation]:
row[orientation][label] = {}
group = h5_file['images'][orientation][label]
for image in group:
row[orientation][label][image] = {}
for key, val in group[image].attrs.items():
if not isinstance(val, h5py.Reference):
row[orientation][label][image][key] = val
row['filepath'] = filepath
row['agnostic_path'] = labscript_utils.shared_drive.path_to_agnostic(
filepath)
row['sequence'] = asdatetime(
h5_file.attrs['sequence_id'].split('_')[0])
try:
row['sequence_index'] = h5_file.attrs['sequence_index']
except:
row['sequence_index'] = float('nan')
if 'script' in h5_file:
row['labscript'] = h5_file['script'].attrs['name']
try:
row['run time'] = asdatetime(h5_file.attrs['run time'])
except KeyError:
row['run time'] = float('nan')
try:
row['run number'] = h5_file.attrs['run number']
except KeyError:
# ignore:
pass
try:
row['individual id'] = h5_file.attrs['individual id']
row['generation'] = h5_file.attrs['generation']
except KeyError:
pass
return row
def get_nested_dict_from_shot(filepath):
row = runmanager.get_shot_globals(filepath)
with h5py.File(filepath, 'r') as h5_file:
if 'results' in h5_file:
for groupname in h5_file['results']:
resultsgroup = h5_file['results'][groupname]
row[groupname] = get_attributes(resultsgroup)
if 'images' in h5_file:
for orientation in h5_file['images'].keys():
if isinstance(h5_file['images'][orientation], h5py.Group):
row[orientation] = get_attributes(
h5_file['images'][orientation])
for label in h5_file['images'][orientation]:
row[orientation][label] = {}
group = h5_file['images'][orientation][label]
for image in group:
row[orientation][label][image] = {}
for key, val in get_attributes(
group[image]).items():
if not isinstance(val, h5py.Reference):
row[orientation][label][image][key] = val
row['filepath'] = _ensure_str(filepath)
row['agnostic_path'] = labscript_utils.shared_drive.path_to_agnostic(
filepath)
seq_id = _ensure_str(h5_file.attrs['sequence_id'])
row['sequence'] = asdatetime(seq_id.split('_')[0])
try:
row['sequence_index'] = h5_file.attrs['sequence_index']
except KeyError:
row['sequence_index'] = None
if 'script' in h5_file:
row['labscript'] = _ensure_str(h5_file['script'].attrs['name'])
try:
row['run time'] = asdatetime(_ensure_str(
h5_file.attrs['run time']))
except KeyError:
row['run time'] = float('nan')
try:
row['run number'] = h5_file.attrs['run number']
except KeyError:
row['run number'] = float('nan')
try:
row['run repeat'] = h5_file.attrs['run repeat']
except KeyError:
row['run repeat'] = 0
try:
row['n_runs'] = h5_file.attrs['n_runs']
except KeyError:
row['n_runs'] = float('nan')
return row
def get_nested_dict_from_shot(filepath):
row = runmanager.get_shot_globals(filepath)
with h5py.File(filepath,'r') as h5_file:
# if 'data' in h5_file:
# for groupname in h5_file['data']:
# resultsgroup = h5_file['data'][groupname]
# if 'camera' in dict(resultsgroup.attrs).keys():
# row[groupname] = h5_file['data'][groupname]['Raw'][:]
if 'results' in h5_file:
for groupname in h5_file['results']:
resultsgroup = h5_file['results'][groupname]
row[groupname] = dict(resultsgroup.attrs)
if 'images' in h5_file:
for orientation in h5_file['images'].keys():
if isinstance(h5_file['images'][orientation], h5py.Group):
row[orientation] = dict(h5_file['images'][orientation].attrs)
for label in h5_file['images'][orientation]:
row[orientation][label] = {}
group = h5_file['images'][orientation][label]
for image in group:
row[orientation][label][image] = {}
for key, val in group[image].attrs.items():
if not isinstance(val, h5py.Reference):
row[orientation][label][image][key] = val
row['filepath'] = filepath
row['agnostic_path'] = labscript_utils.shared_drive.path_to_agnostic(filepath)
row['sequence'] = asdatetime(h5_file.attrs['sequence_id'].split('_')[0])
try:
row['sequence_index'] = h5_file.attrs['sequence_index']
except:
row['sequence_index'] = float('nan')
if 'script' in h5_file:
row['labscript'] = h5_file['script'].attrs['name']
try:
row['run time'] = asdatetime(h5_file.attrs['run time'])
except KeyError:
row['run time'] = float('nan')
try:
row['run number'] = h5_file.attrs['run number']
except KeyError:
# ignore:
pass
try:
row['individual id'] = h5_file.attrs['individual id']
row['generation'] = h5_file.attrs['generation']
except KeyError:
pass
return row
def __init__(self, h5_file, device_name):
self.h5_file = h5_file
self.device_name = device_name
self.globals = runmanager.get_shot_globals(h5_file)
def manage(self):
logger = logging.getLogger('BLACS.queue_manager.thread')
# While the program is running!
logger.info('starting')
# HDF5 prints lots of errors by default, for things that aren't
# actually errors. These are silenced on a per thread basis,
# and automatically silenced in the main thread when h5py is
# imported. So we'll silence them in this thread too:
h5py._errors.silence_errors()
# This name stores the queue currently being used to
# communicate with tabs, so that abort signals can be put
# to it when those tabs never respond and are restarted by
# the user.
self.current_queue = Queue.Queue()
#TODO: put in general configuration
timeout_limit = 300 #seconds
self.set_status("Idle")
while self.manager_running:
# If the pause button is pushed in, sleep
if self.manager_paused:
if self.get_status() == "Idle":
logger.info('Paused')
self.set_status("Queue Paused")
time.sleep(1)
continue
# Get the top file
try:
path = self.get_next_file()
now_running_text = 'Now running: <b>%s</b>'%os.path.basename(path)
self.set_status(now_running_text)
logger.info('Got a file: %s'%path)
except:
# If no files, sleep for 1s,
self.set_status("Idle")
time.sleep(1)
continue
devices_in_use = {}
transition_list = {}
start_time = time.time()
self.current_queue = Queue.Queue()
# Function to be run when abort button is clicked
def abort_function():
try:
# Set device name to "Queue Manager" which will never be a labscript device name
# as it is not a valid python variable name (has a space in it!)
self.current_queue.put(['Queue Manager', 'abort'])
except Exception:
logger.exception('Could not send abort message to the queue manager')
def restart_function(device_name):
try:
self.current_queue.put([device_name, 'restart'])
except Exception:
logger.exception('Could not send restart message to the queue manager for device %s'%device_name)
##########################################################################################################################################
# transition to buffered #
##########################################################################################################################################
try:
# A Queue for event-based notification when the tabs have
# completed transitioning to buffered:
timed_out = False
error_condition = False
abort = False
restarted = False
self.set_status(now_running_text+"<br>Transitioning to Buffered")
# Enable abort button, and link in current_queue:
inmain(self._ui.queue_abort_button.clicked.connect,abort_function)
inmain(self._ui.queue_abort_button.setEnabled,True)
# Ready to run file: assume that the file has _not_ been compiled and compile it
# Extract script globals, and update them from the blacs mantained dictionary of globals.
shot_globals = get_shot_globals(path)
shot_globals.update(self.DynamicGlobals)
with h5py.File(path, "a") as hdf5_file:
set_shot_globals(hdf5_file, shot_globals)
# Compile file
compile_h5(path)
# Run file
with h5py.File(path, "r+") as hdf5_file:
min_time = hdf5_file.attrs['min_time']
h5_file_devices = hdf5_file['devices/'].keys()
for name in h5_file_devices:
try:
# Connect restart signal from tabs to current_queue and transition the device to buffered mode
success = self.transition_device_to_buffered(name,transition_list,path,restart_function)
if not success:
logger.error('%s has an error condition, aborting run' % name)
error_condition = True
break
except Exception as e:
logger.error('Exception while transitioning %s to buffered mode. Exception was: %s'%(name,str(e)))
error_condition = True
break
devices_in_use = transition_list.copy()
while transition_list and not error_condition:
try:
# Wait for a device to transtition_to_buffered:
logger.debug('Waiting for the following devices to finish transitioning to buffered mode: %s'%str(transition_list))
device_name, result = self.current_queue.get(timeout=2)
#Handle abort button signal
if device_name == 'Queue Manager' and result == 'abort':
# we should abort the run
logger.info('abort signal received from GUI')
abort = True
break
if result == 'fail':
logger.info('abort signal received during transition to buffered of %s' % device_name)
error_condition = True
break
elif result == 'restart':
logger.info('Device %s was restarted, aborting shot.'%device_name)
restarted = True
break
logger.debug('%s finished transitioning to buffered mode' % device_name)
# The tab says it's done, but does it have an error condition?
if self.get_device_error_state(device_name,transition_list):
logger.error('%s has an error condition, aborting run' % device_name)
error_condition = True
break
del transition_list[device_name]
except Queue.Empty:
# It's been 2 seconds without a device finishing
# transitioning to buffered. Is there an error?
for name in transition_list:
if self.get_device_error_state(name,transition_list):
error_condition = True
break
if error_condition:
break
# Has programming timed out?
if time.time() - start_time > timeout_limit:
logger.error('Transitioning to buffered mode timed out')
timed_out = True
break
# Handle if we broke out of loop due to timeout or error:
if timed_out or error_condition or abort or restarted:
# Pause the queue, re add the path to the top of the queue, and set a status message!
# only if we aren't responding to an abort click
if not abort:
self.manager_paused = True
self.prepend(path)
if timed_out:
self.set_status("Device programming timed out. Queue Paused...")
elif abort:
self.set_status("Shot aborted")
elif restarted:
self.set_status('A device was restarted during transition_to_buffered. Shot aborted')
else:
self.set_status("One or more devices is in an error state. Queue Paused...")
# Abort the run for all devices in use:
# need to recreate the queue here because we don't want to hear from devices that are still transitioning to buffered mode
self.current_queue = Queue.Queue()
for tab in devices_in_use.values():
# We call abort buffered here, because if each tab is either in mode=BUFFERED or transition_to_buffered failed in which case
# it should have called abort_transition_to_buffered itself and returned to manual mode
# Since abort buffered will only run in mode=BUFFERED, and the state is not queued indefinitely (aka it is deleted if we are not in mode=BUFFERED)
# this is the correct method call to make for either case
tab.abort_buffered(self.current_queue)
# We don't need to check the results of this function call because it will either be successful, or raise a visible error in the tab.
# disconnect restart signal from tabs
inmain(tab.disconnect_restart_receiver,restart_function)
# disconnect abort button and disable
inmain(self._ui.queue_abort_button.clicked.disconnect,abort_function)
inmain(self._ui.queue_abort_button.setEnabled,False)
# Start a new iteration
continue
##########################################################################################################################################
# SCIENCE! #
##########################################################################################################################################
# Get front panel data, but don't save it to the h5 file until the experiment ends:
states,tab_positions,window_data,plugin_data = self.BLACS.front_panel_settings.get_save_data()
self.set_status(now_running_text+"<br>Running...(program time: %.3fs)"%(time.time() - start_time))
# A Queue for event-based notification of when the experiment has finished.
experiment_finished_queue = Queue.Queue()
logger.debug('About to start the master pseudoclock')
# Do not start until delay time specificed by last sequence has expired
self._timer.wait()
# Start the timer to block until the next run starts
self._timer.start(
min_time,
countdown_queue=self.BLACS._countdown_queue,
countdown_mode='precent_done')
run_time = time.localtime()
#TODO: fix potential race condition if BLACS is closing when this line executes?
self.BLACS.tablist[self.master_pseudoclock].start_run(experiment_finished_queue)
# Wait for notification of the end of run:
abort = False
restarted = False
done = False
while not (abort or restarted or done):
try:
done = experiment_finished_queue.get(timeout=0.5) == 'done'
except Queue.Empty:
pass
try:
# Poll self.current_queue for abort signal from button or device restart
device_name, result = self.current_queue.get_nowait()
if (device_name == 'Queue Manager' and result == 'abort'):
abort = True
if result == 'restart':
restarted = True
# Check for error states in tabs
for device_name, tab in devices_in_use.items():
if self.get_device_error_state(device_name,devices_in_use):
restarted = True
except Queue.Empty:
pass
if abort or restarted:
for devicename, tab in devices_in_use.items():
if tab.mode == MODE_BUFFERED:
tab.abort_buffered(self.current_queue)
# disconnect restart signal from tabs
inmain(tab.disconnect_restart_receiver,restart_function)
# Disable abort button
inmain(self._ui.queue_abort_button.clicked.disconnect,abort_function)
inmain(self._ui.queue_abort_button.setEnabled,False)
if restarted:
self.manager_paused = True
self.prepend(path)
self.set_status("Device restarted mid-shot. Shot aborted, Queue paused.")
elif abort:
self.set_status("Shot aborted")
if abort or restarted:
# after disabling the abort button, we now start a new iteration
continue
logger.info('Run complete')
self.set_status(now_running_text+"<br>Sequence done, saving data...")
# End try/except block here
except Exception:
logger.exception("Error in queue manager execution. Queue paused.")
# clean up the h5 file
self.manager_paused = True
# clean the h5 file:
self.clean_h5_file(path, 'temp.h5')
try:
os.remove(path)
os.rename('temp.h5', path)
except WindowsError if platform.system() == 'Windows' else None:
logger.warning('Couldn\'t delete failed run file %s, another process may be using it. Using alternate filename for second attempt.'%path)
os.rename('temp.h5', path.replace('.h5','_retry.h5'))
path = path.replace('.h5','_retry.h5')
# Put it back at the start of the queue:
self.prepend(path)
# Need to put devices back in manual mode
self.current_queue = Queue.Queue()
for devicename, tab in devices_in_use.items():
if tab.mode == MODE_BUFFERED or tab.mode == MODE_TRANSITION_TO_BUFFERED:
tab.abort_buffered(self.current_queue)
# disconnect restart signal from tabs
inmain(tab.disconnect_restart_receiver,restart_function)
self.set_status("Error occured in Queue Manager. Queue Paused. \nPlease make sure all devices are back in manual mode before unpausing the queue")
# disconnect and disable abort button
inmain(self._ui.queue_abort_button.clicked.disconnect,abort_function)
inmain(self._ui.queue_abort_button.setEnabled,False)
# Start a new iteration
continue
##########################################################################################################################################
# SCIENCE OVER! #
##########################################################################################################################################
##########################################################################################################################################
# Transition to manual #
##########################################################################################################################################
# start new try/except block here
try:
with h5py.File(path,'r+') as hdf5_file:
self.BLACS.front_panel_settings.store_front_panel_in_h5(hdf5_file,states,tab_positions,window_data,plugin_data,save_conn_table = False)
with h5py.File(path,'r+') as hdf5_file:
data_group = hdf5_file['/'].create_group('data')
# stamp with the run time of the experiment
hdf5_file.attrs['run time'] = time.strftime('%Y%m%dT%H%M%S',run_time)
for devicename, tab in devices_in_use.items():
if tab.mode == MODE_BUFFERED:
tab.transition_to_manual(self.current_queue)
error_condition = False
transition_list = devices_in_use.copy()
while transition_list and not error_condition:
try:
# Wait for a device to transition_to_manual:
logger.debug('Waiting for the following devices to finish transitioning to manual mode: %s'%str(transition_list))
device_name, result = self.current_queue.get(timeout=2)
if (device_name == 'Queue Manager' and result == 'abort'):
# Ignore any abort signals left in the queue, it
# is too late to abort in any case
continue
logger.debug('%s finished transitioning to manual mode' % device_name)
if result == 'fail':
error_condition = True
if result == 'restart':
error_condition = True
if self.get_device_error_state(device_name, transition_list):
error_condition = True
del transition_list[device_name]
# disconnect restart signal from tab
inmain(tab.disconnect_restart_receiver,restart_function)
except queue.Empty:
# It's been 2 seconds without a device finishing
# transitioning to manual. Is there an error?
for name in transition_list:
if self.get_device_error_state(name,transition_list):
error_condition = True
if error_condition:
break
if error_condition:
self.set_status("Error during transtion to manual. Queue Paused.")
# TODO: Kind of dodgy raising an exception here...
raise Exception('A device failed during transition to manual')
# All data written, now run all PostProcessing functions
SavedFunctions = labscript_utils.h5_scripting.get_all_saved_functions(path)
with h5py.File(path, 'r+') as hdf5_file:
for SavedFunction in SavedFunctions:
try:
result = SavedFunction.custom_call(hdf5_file, **shot_globals)
except Exception:
import zprocess
zprocess.raise_exception_in_thread(sys.exc_info())
result = {}
logger.error('Post Processing function did not execute correctly')
try:
self.DynamicGlobals.update(result)
except:
logger.error('Post Processing function did not return a dict type')
inmain(self._ui.Globals_tableWidget.setRowCount, len(self.DynamicGlobals))
for i, key in enumerate(self.DynamicGlobals):
inmain(self._ui.Globals_tableWidget.setItem, i, 0, QTableWidgetItem(key))
inmain(self._ui.Globals_tableWidget.setItem, i, 1, QTableWidgetItem( str(self.DynamicGlobals[key]) ))
except Exception as e:
logger.exception("Error in queue manager execution. Queue paused.")
# clean up the h5 file
self.manager_paused = True
# clean the h5 file:
self.clean_h5_file(path, 'temp.h5')
# self.SavedFunctions = labscript_utils.h5_scripting.get_all_saved_functions(path(path, 'temp.h5'))
try:
os.remove(path)
os.rename('temp.h5', path)
except WindowsError if platform.system() == 'Windows' else None:
logger.warning('Couldn\'t delete failed run file %s, another process may be using it. Using alternate filename for second attempt.'%path)
os.rename('temp.h5', path.replace('.h5','_retry.h5'))
path = path.replace('.h5','_retry.h5')
# Put it back at the start of the queue:
self.prepend(path)
# Need to put devices back in manual mode. Since the experiment is over before this try/except block begins, we can
# safely call transition_to_manual() on each device tab
self.current_queue = Queue.Queue()
for devicename, tab in devices_in_use.items():
if tab.mode == MODE_BUFFERED:
tab.transition_to_manual(self.current_queue)
# disconnect restart signal from tabs
inmain(tab.disconnect_restart_receiver,restart_function)
self.set_status("Error occured in Queue Manager. Queue Paused. \nPlease make sure all devices are back in manual mode before unpausing the queue")
continue
##########################################################################################################################################
# Analysis Submission #
##########################################################################################################################################
logger.info('All devices are back in static mode.')
# Submit to the analysis server
self.BLACS.analysis_submission.get_queue().put(['file', path])
##########################################################################################################################################
# Plugin callbacks #
##########################################################################################################################################
for plugin in self.BLACS.plugins.values():
callbacks = plugin.get_callbacks()
if isinstance(callbacks, dict) and 'shot_complete' in callbacks:
try:
callbacks['shot_complete'](path)
except Exception:
logger.exception("Plugin callback raised an exception")
##########################################################################################################################################
# Repeat Experiment? #
##########################################################################################################################################
if (self.manager_repeat == 1) or (self.manager_repeat == 2 and self.get_num_files() == 0):
# Resubmit job to the bottom of the queue:
try:
message = self.process_request(path)
logger.info(message)
except:
# TODO: make this error popup for the user
logger.error('Failed to copy h5_file (%s) for repeat run'%path)
self.set_status("Idle")
logger.info('Stopping')
def manage(self):
logger = logging.getLogger('BLACS.queue_manager.thread')
# While the program is running!
logger.info('starting')
# HDF5 prints lots of errors by default, for things that aren't
# actually errors. These are silenced on a per thread basis,
# and automatically silenced in the main thread when h5py is
# imported. So we'll silence them in this thread too:
h5py._errors.silence_errors()
# This name stores the queue currently being used to
# communicate with tabs, so that abort signals can be put
# to it when those tabs never respond and are restarted by
# the user.
self.current_queue = Queue.Queue()
#TODO: put in general configuration
timeout_limit = 300 #seconds
self.set_status("Idle")
while self.manager_running:
# If the pause button is pushed in, sleep
if self.manager_paused:
if self.get_status() == "Idle":
logger.info('Paused')
self.set_status("Queue Paused")
time.sleep(1)
continue
# Get the top file
try:
path = self.get_next_file()
now_running_text = 'Now running: <b>%s</b>'%os.path.basename(path)
self.set_status(now_running_text)
logger.info('Got a file: %s'%path)
except:
# If no files, sleep for 1s,
self.set_status("Idle")
time.sleep(1)
continue
devices_in_use = {}
transition_list = {}
start_time = time.time()
self.current_queue = Queue.Queue()
# Function to be run when abort button is clicked
def abort_function():
try:
# Set device name to "Queue Manager" which will never be a labscript device name
# as it is not a valid python variable name (has a space in it!)
self.current_queue.put(['Queue Manager', 'abort'])
except Exception:
logger.exception('Could not send abort message to the queue manager')
def restart_function(device_name):
try:
self.current_queue.put([device_name, 'restart'])
except Exception:
logger.exception('Could not send restart message to the queue manager for device %s'%device_name)
##########################################################################################################################################
# transition to buffered #
##########################################################################################################################################
try:
# A Queue for event-based notification when the tabs have
# completed transitioning to buffered:
timed_out = False
error_condition = False
abort = False
restarted = False
self.set_status(now_running_text+"<br>Transitioning to Buffered")
# Enable abort button, and link in current_queue:
inmain(self._ui.queue_abort_button.clicked.connect,abort_function)
inmain(self._ui.queue_abort_button.setEnabled,True)
# Ready to run file: assume that the file has _not_ been compiled and compile it
# Extract script globals, and update them from the blacs mantained dictionary of globals.
shot_globals = get_shot_globals(path)
shot_globals.update(self.DynamicGlobals)
with h5py.File(path, "a") as hdf5_file:
set_shot_globals(hdf5_file, shot_globals)
# Compile file
compile_h5(path)
# Run file
with h5py.File(path, "r+") as hdf5_file:
min_time = hdf5_file.attrs['min_time']
h5_file_devices = hdf5_file['devices/'].keys()
for name in h5_file_devices:
try:
# Connect restart signal from tabs to current_queue and transition the device to buffered mode
success = self.transition_device_to_buffered(name,transition_list,path,restart_function)
if not success:
logger.error('%s has an error condition, aborting run' % name)
error_condition = True
break
except Exception as e:
logger.error('Exception while transitioning %s to buffered mode. Exception was: %s'%(name,str(e)))
error_condition = True
break
devices_in_use = transition_list.copy()
while transition_list and not error_condition:
try:
# Wait for a device to transtition_to_buffered:
logger.debug('Waiting for the following devices to finish transitioning to buffered mode: %s'%str(transition_list))
device_name, result = self.current_queue.get(timeout=2)
#Handle abort button signal
if device_name == 'Queue Manager' and result == 'abort':
# we should abort the run
logger.info('abort signal received from GUI')
abort = True
break
if result == 'fail':
logger.info('abort signal received during transition to buffered of %s' % device_name)
error_condition = True
break
elif result == 'restart':
logger.info('Device %s was restarted, aborting shot.'%device_name)
restarted = True
break
logger.debug('%s finished transitioning to buffered mode' % device_name)
# The tab says it's done, but does it have an error condition?
if self.get_device_error_state(device_name,transition_list):
logger.error('%s has an error condition, aborting run' % device_name)
error_condition = True
break
del transition_list[device_name]
except Queue.Empty:
# It's been 2 seconds without a device finishing
# transitioning to buffered. Is there an error?
for name in transition_list:
if self.get_device_error_state(name,transition_list):
error_condition = True
break
if error_condition:
break
# Has programming timed out?
if time.time() - start_time > timeout_limit:
logger.error('Transitioning to buffered mode timed out')
timed_out = True
break
# Handle if we broke out of loop due to timeout or error:
if timed_out or error_condition or abort or restarted:
# Pause the queue, re add the path to the top of the queue, and set a status message!
# only if we aren't responding to an abort click
if not abort:
self.manager_paused = True
self.prepend(path)
if timed_out:
self.set_status("Device programming timed out. Queue Paused...")
elif abort:
self.set_status("Shot aborted")
elif restarted:
self.set_status('A device was restarted during transition_to_buffered. Shot aborted')
else:
self.set_status("One or more devices is in an error state. Queue Paused...")
# Abort the run for all devices in use:
# need to recreate the queue here because we don't want to hear from devices that are still transitioning to buffered mode
self.current_queue = Queue.Queue()
for tab in devices_in_use.values():
# We call abort buffered here, because if each tab is either in mode=BUFFERED or transition_to_buffered failed in which case
# it should have called abort_transition_to_buffered itself and returned to manual mode
# Since abort buffered will only run in mode=BUFFERED, and the state is not queued indefinitely (aka it is deleted if we are not in mode=BUFFERED)
# this is the correct method call to make for either case
tab.abort_buffered(self.current_queue)
# We don't need to check the results of this function call because it will either be successful, or raise a visible error in the tab.
# disconnect restart signal from tabs
inmain(tab.disconnect_restart_receiver,restart_function)
# disconnect abort button and disable
inmain(self._ui.queue_abort_button.clicked.disconnect,abort_function)
inmain(self._ui.queue_abort_button.setEnabled,False)
# Start a new iteration
continue
##########################################################################################################################################
# SCIENCE! #
##########################################################################################################################################
# Get front panel data, but don't save it to the h5 file until the experiment ends:
states,tab_positions,window_data,plugin_data = self.BLACS.front_panel_settings.get_save_data()
self.set_status(now_running_text+"<br>Running...(program time: %.3fs)"%(time.time() - start_time))
# A Queue for event-based notification of when the experiment has finished.
experiment_finished_queue = Queue.Queue()
logger.debug('About to start the master pseudoclock')
# Do not start until delay time specificed by last sequence has expired
self._timer.wait()
# Start the timer to block until the next run starts
self._timer.start(
min_time,
countdown_queue=self.BLACS._countdown_queue,
countdown_mode='precent_done')
run_time = time.localtime()
#TODO: fix potential race condition if BLACS is closing when this line executes?
self.BLACS.tablist[self.master_pseudoclock].start_run(experiment_finished_queue)
# Wait for notification of the end of run:
abort = False
restarted = False
while result != 'done':
try:
result = experiment_finished_queue.get(timeout=0.5)
except Queue.Empty:
pass
try:
# Poll self.current_queue for abort signal from button or device restart
device_name, result = self.current_queue.get(timeout=0.5)
if (device_name == 'Queue Manager' and result == 'abort'):
abort = True
break
elif result == 'restart':
restarted = True
break
# Check for error states in tabs
for device_name, tab in devices_in_use.items():
if self.get_device_error_state(device_name,devices_in_use):
restarted = True
break
if restarted:
break
except Queue.Empty:
pass
if abort or restarted:
for devicename, tab in devices_in_use.items():
if tab.mode == MODE_BUFFERED:
tab.abort_buffered(self.current_queue)
# disconnect restart signal from tabs
inmain(tab.disconnect_restart_receiver,restart_function)
# Disable abort button
inmain(self._ui.queue_abort_button.clicked.disconnect,abort_function)
inmain(self._ui.queue_abort_button.setEnabled,False)
if restarted:
self.manager_paused = True
self.prepend(path)
self.set_status("Device restarted mid-shot. Shot aborted, Queue paused.")
elif abort:
self.set_status("Shot aborted")
if abort or restarted:
# after disabling the abort button, we now start a new iteration
continue
logger.info('Run complete')
self.set_status(now_running_text+"<br>Sequence done, saving data...")
# End try/except block here
except Exception:
logger.exception("Error in queue manager execution. Queue paused.")
# clean up the h5 file
self.manager_paused = True
# clean the h5 file:
self.clean_h5_file(path, 'temp.h5')
try:
os.remove(path)
os.rename('temp.h5', path)
except WindowsError if platform.system() == 'Windows' else None:
logger.warning('Couldn\'t delete failed run file %s, another process may be using it. Using alternate filename for second attempt.'%path)
os.rename('temp.h5', path.replace('.h5','_retry.h5'))
path = path.replace('.h5','_retry.h5')
# Put it back at the start of the queue:
self.prepend(path)
# Need to put devices back in manual mode
self.current_queue = Queue.Queue()
for devicename, tab in devices_in_use.items():
if tab.mode == MODE_BUFFERED or tab.mode == MODE_TRANSITION_TO_BUFFERED:
tab.abort_buffered(self.current_queue)
# disconnect restart signal from tabs
inmain(tab.disconnect_restart_receiver,restart_function)
self.set_status("Error occured in Queue Manager. Queue Paused. \nPlease make sure all devices are back in manual mode before unpausing the queue")
# disconnect and disable abort button
inmain(self._ui.queue_abort_button.clicked.disconnect,abort_function)
inmain(self._ui.queue_abort_button.setEnabled,False)
# Start a new iteration
continue
##########################################################################################################################################
# SCIENCE OVER! #
##########################################################################################################################################
##########################################################################################################################################
# Transition to manual #
##########################################################################################################################################
# start new try/except block here
try:
with h5py.File(path,'r+') as hdf5_file:
self.BLACS.front_panel_settings.store_front_panel_in_h5(hdf5_file,states,tab_positions,window_data,plugin_data,save_conn_table = False)
with h5py.File(path,'r+') as hdf5_file:
data_group = hdf5_file['/'].create_group('data')
# stamp with the run time of the experiment
hdf5_file.attrs['run time'] = time.strftime('%Y%m%dT%H%M%S',run_time)
# A Queue for event-based notification of when the devices have transitioned to static mode:
# Shouldn't need to recreate the queue: self.current_queue = Queue.Queue()
# TODO: unserialise this if everything is using zprocess.locking
# only transition one device to static at a time,
# since writing data to the h5 file can potentially
# happen at this stage:
error_condition = False
# This is far more complicated than it needs to be once transition_to_manual is unserialised!
response_list = {}
for device_name, tab in devices_in_use.items():
if device_name not in response_list:
tab.transition_to_manual(self.current_queue)
while True:
# TODO: make the call to current_queue.get() timeout
# and periodically check for error condition on the tab
got_device_name, result = self.current_queue.get()
# if the response is not for this device, then save it for later!
if device_name != got_device_name:
response_list[got_device_name] = result
else:
break
else:
result = response_list[device_name]
# Check for abort signal from device restart
if result == 'fail':
error_condition = True
if result == 'restart':
error_condition = True
if self.get_device_error_state(device_name,devices_in_use):
error_condition = True
# Once device has transitioned_to_manual, disconnect restart signal
inmain(tab.disconnect_restart_receiver,restart_function)
if error_condition:
self.set_status("Error during transtion to manual. Queue Paused.")
# TODO: Kind of dodgy raising an exception here...
raise Exception('A device failed during transition to manual')
# All data written, now run all PostProcessing functions
SavedFunctions = labscript_utils.h5_scripting.get_all_saved_functions(path)
with h5py.File(path, 'r+') as hdf5_file:
for SavedFunction in SavedFunctions:
try:
result = SavedFunction(hdf5_file, **shot_globals)
except:
result = {}
logger.error('Post Processing function did not execute correctly')
try:
self.DynamicGlobals.update(result)
except:
logger.error('Post Processing function did not return a dict type')
inmain(self._ui.Globals_tableWidget.setRowCount, len(self.DynamicGlobals))
for i, key in enumerate(self.DynamicGlobals):
inmain(self._ui.Globals_tableWidget.setItem, i, 0, QTableWidgetItem(key))
inmain(self._ui.Globals_tableWidget.setItem, i, 1, QTableWidgetItem( str(self.DynamicGlobals[key]) ))
except Exception as e:
logger.exception("Error in queue manager execution. Queue paused.")
# clean up the h5 file
self.manager_paused = True
# clean the h5 file:
self.clean_h5_file(path, 'temp.h5')
try:
os.remove(path)
os.rename('temp.h5', path)
except WindowsError if platform.system() == 'Windows' else None:
logger.warning('Couldn\'t delete failed run file %s, another process may be using it. Using alternate filename for second attempt.'%path)
os.rename('temp.h5', path.replace('.h5','_retry.h5'))
path = path.replace('.h5','_retry.h5')
# Put it back at the start of the queue:
self.prepend(path)
# Need to put devices back in manual mode. Since the experiment is over before this try/except block begins, we can
# safely call transition_to_manual() on each device tab
# TODO: Not serialised...could be bad with older BIAS versions :(
self.current_queue = Queue.Queue()
for devicename, tab in devices_in_use.items():
if tab.mode == MODE_BUFFERED:
tab.transition_to_manual(self.current_queue)
# disconnect restart signal from tabs
inmain(tab.disconnect_restart_receiver,restart_function)
self.set_status("Error occured in Queue Manager. Queue Paused. \nPlease make sure all devices are back in manual mode before unpausing the queue")
continue
##########################################################################################################################################
# Analysis Submission #
##########################################################################################################################################
logger.info('All devices are back in static mode.')
# Submit to the analysis server
self.BLACS.analysis_submission.get_queue().put(['file', path])
##########################################################################################################################################
# Repeat Experiment? #
##########################################################################################################################################
if (self.manager_repeat == 1) or (self.manager_repeat == 2 and self.get_num_files() == 0):
# Resubmit job to the bottom of the queue:
try:
message = self.process_request(path)
logger.info(message)
except:
# TODO: make this error popup for the user
logger.error('Failed to copy h5_file (%s) for repeat run'%path)
self.set_status("Idle")
logger.info('Stopping')