def get_nested_dict_from_shot(filepath): row = runmanager.get_shot_globals(filepath) with h5py.File(filepath, 'r') as h5_file: # if 'data' in h5_file: # for groupname in h5_file['data']: # resultsgroup = h5_file['data'][groupname] # if 'camera' in dict(resultsgroup.attrs).keys(): # row[groupname] = h5_file['data'][groupname]['Raw'][:] if 'results' in h5_file: for groupname in h5_file['results']: resultsgroup = h5_file['results'][groupname] row[groupname] = dict(resultsgroup.attrs) if 'images' in h5_file: for orientation in h5_file['images'].keys(): if isinstance(h5_file['images'][orientation], h5py.Group): row[orientation] = dict( h5_file['images'][orientation].attrs) for label in h5_file['images'][orientation]: row[orientation][label] = {} group = h5_file['images'][orientation][label] for image in group: row[orientation][label][image] = {} for key, val in group[image].attrs.items(): if not isinstance(val, h5py.Reference): row[orientation][label][image][key] = val row['filepath'] = filepath row['agnostic_path'] = labscript_utils.shared_drive.path_to_agnostic( filepath) row['sequence'] = asdatetime( h5_file.attrs['sequence_id'].split('_')[0]) try: row['sequence_index'] = h5_file.attrs['sequence_index'] except: row['sequence_index'] = float('nan') if 'script' in h5_file: row['labscript'] = h5_file['script'].attrs['name'] try: row['run time'] = asdatetime(h5_file.attrs['run time']) except KeyError: row['run time'] = float('nan') try: row['run number'] = h5_file.attrs['run number'] except KeyError: # ignore: pass try: row['individual id'] = h5_file.attrs['individual id'] row['generation'] = h5_file.attrs['generation'] except KeyError: pass return row
def get_nested_dict_from_shot(filepath): row = runmanager.get_shot_globals(filepath) with h5py.File(filepath, 'r') as h5_file: if 'results' in h5_file: for groupname in h5_file['results']: resultsgroup = h5_file['results'][groupname] row[groupname] = get_attributes(resultsgroup) if 'images' in h5_file: for orientation in h5_file['images'].keys(): if isinstance(h5_file['images'][orientation], h5py.Group): row[orientation] = get_attributes( h5_file['images'][orientation]) for label in h5_file['images'][orientation]: row[orientation][label] = {} group = h5_file['images'][orientation][label] for image in group: row[orientation][label][image] = {} for key, val in get_attributes( group[image]).items(): if not isinstance(val, h5py.Reference): row[orientation][label][image][key] = val row['filepath'] = _ensure_str(filepath) row['agnostic_path'] = labscript_utils.shared_drive.path_to_agnostic( filepath) seq_id = _ensure_str(h5_file.attrs['sequence_id']) row['sequence'] = asdatetime(seq_id.split('_')[0]) try: row['sequence_index'] = h5_file.attrs['sequence_index'] except KeyError: row['sequence_index'] = None if 'script' in h5_file: row['labscript'] = _ensure_str(h5_file['script'].attrs['name']) try: row['run time'] = asdatetime(_ensure_str( h5_file.attrs['run time'])) except KeyError: row['run time'] = float('nan') try: row['run number'] = h5_file.attrs['run number'] except KeyError: row['run number'] = float('nan') try: row['run repeat'] = h5_file.attrs['run repeat'] except KeyError: row['run repeat'] = 0 try: row['n_runs'] = h5_file.attrs['n_runs'] except KeyError: row['n_runs'] = float('nan') return row
def get_nested_dict_from_shot(filepath): row = runmanager.get_shot_globals(filepath) with h5py.File(filepath,'r') as h5_file: # if 'data' in h5_file: # for groupname in h5_file['data']: # resultsgroup = h5_file['data'][groupname] # if 'camera' in dict(resultsgroup.attrs).keys(): # row[groupname] = h5_file['data'][groupname]['Raw'][:] if 'results' in h5_file: for groupname in h5_file['results']: resultsgroup = h5_file['results'][groupname] row[groupname] = dict(resultsgroup.attrs) if 'images' in h5_file: for orientation in h5_file['images'].keys(): if isinstance(h5_file['images'][orientation], h5py.Group): row[orientation] = dict(h5_file['images'][orientation].attrs) for label in h5_file['images'][orientation]: row[orientation][label] = {} group = h5_file['images'][orientation][label] for image in group: row[orientation][label][image] = {} for key, val in group[image].attrs.items(): if not isinstance(val, h5py.Reference): row[orientation][label][image][key] = val row['filepath'] = filepath row['agnostic_path'] = labscript_utils.shared_drive.path_to_agnostic(filepath) row['sequence'] = asdatetime(h5_file.attrs['sequence_id'].split('_')[0]) try: row['sequence_index'] = h5_file.attrs['sequence_index'] except: row['sequence_index'] = float('nan') if 'script' in h5_file: row['labscript'] = h5_file['script'].attrs['name'] try: row['run time'] = asdatetime(h5_file.attrs['run time']) except KeyError: row['run time'] = float('nan') try: row['run number'] = h5_file.attrs['run number'] except KeyError: # ignore: pass try: row['individual id'] = h5_file.attrs['individual id'] row['generation'] = h5_file.attrs['generation'] except KeyError: pass return row
def __init__(self, h5_file, device_name): self.h5_file = h5_file self.device_name = device_name self.globals = runmanager.get_shot_globals(h5_file)
def manage(self): logger = logging.getLogger('BLACS.queue_manager.thread') # While the program is running! logger.info('starting') # HDF5 prints lots of errors by default, for things that aren't # actually errors. These are silenced on a per thread basis, # and automatically silenced in the main thread when h5py is # imported. So we'll silence them in this thread too: h5py._errors.silence_errors() # This name stores the queue currently being used to # communicate with tabs, so that abort signals can be put # to it when those tabs never respond and are restarted by # the user. self.current_queue = Queue.Queue() #TODO: put in general configuration timeout_limit = 300 #seconds self.set_status("Idle") while self.manager_running: # If the pause button is pushed in, sleep if self.manager_paused: if self.get_status() == "Idle": logger.info('Paused') self.set_status("Queue Paused") time.sleep(1) continue # Get the top file try: path = self.get_next_file() now_running_text = 'Now running: <b>%s</b>'%os.path.basename(path) self.set_status(now_running_text) logger.info('Got a file: %s'%path) except: # If no files, sleep for 1s, self.set_status("Idle") time.sleep(1) continue devices_in_use = {} transition_list = {} start_time = time.time() self.current_queue = Queue.Queue() # Function to be run when abort button is clicked def abort_function(): try: # Set device name to "Queue Manager" which will never be a labscript device name # as it is not a valid python variable name (has a space in it!) self.current_queue.put(['Queue Manager', 'abort']) except Exception: logger.exception('Could not send abort message to the queue manager') def restart_function(device_name): try: self.current_queue.put([device_name, 'restart']) except Exception: logger.exception('Could not send restart message to the queue manager for device %s'%device_name) ########################################################################################################################################## # transition to buffered # ########################################################################################################################################## try: # A Queue for event-based notification when the tabs have # completed transitioning to buffered: timed_out = False error_condition = False abort = False restarted = False self.set_status(now_running_text+"<br>Transitioning to Buffered") # Enable abort button, and link in current_queue: inmain(self._ui.queue_abort_button.clicked.connect,abort_function) inmain(self._ui.queue_abort_button.setEnabled,True) # Ready to run file: assume that the file has _not_ been compiled and compile it # Extract script globals, and update them from the blacs mantained dictionary of globals. shot_globals = get_shot_globals(path) shot_globals.update(self.DynamicGlobals) with h5py.File(path, "a") as hdf5_file: set_shot_globals(hdf5_file, shot_globals) # Compile file compile_h5(path) # Run file with h5py.File(path, "r+") as hdf5_file: min_time = hdf5_file.attrs['min_time'] h5_file_devices = hdf5_file['devices/'].keys() for name in h5_file_devices: try: # Connect restart signal from tabs to current_queue and transition the device to buffered mode success = self.transition_device_to_buffered(name,transition_list,path,restart_function) if not success: logger.error('%s has an error condition, aborting run' % name) error_condition = True break except Exception as e: logger.error('Exception while transitioning %s to buffered mode. Exception was: %s'%(name,str(e))) error_condition = True break devices_in_use = transition_list.copy() while transition_list and not error_condition: try: # Wait for a device to transtition_to_buffered: logger.debug('Waiting for the following devices to finish transitioning to buffered mode: %s'%str(transition_list)) device_name, result = self.current_queue.get(timeout=2) #Handle abort button signal if device_name == 'Queue Manager' and result == 'abort': # we should abort the run logger.info('abort signal received from GUI') abort = True break if result == 'fail': logger.info('abort signal received during transition to buffered of %s' % device_name) error_condition = True break elif result == 'restart': logger.info('Device %s was restarted, aborting shot.'%device_name) restarted = True break logger.debug('%s finished transitioning to buffered mode' % device_name) # The tab says it's done, but does it have an error condition? if self.get_device_error_state(device_name,transition_list): logger.error('%s has an error condition, aborting run' % device_name) error_condition = True break del transition_list[device_name] except Queue.Empty: # It's been 2 seconds without a device finishing # transitioning to buffered. Is there an error? for name in transition_list: if self.get_device_error_state(name,transition_list): error_condition = True break if error_condition: break # Has programming timed out? if time.time() - start_time > timeout_limit: logger.error('Transitioning to buffered mode timed out') timed_out = True break # Handle if we broke out of loop due to timeout or error: if timed_out or error_condition or abort or restarted: # Pause the queue, re add the path to the top of the queue, and set a status message! # only if we aren't responding to an abort click if not abort: self.manager_paused = True self.prepend(path) if timed_out: self.set_status("Device programming timed out. Queue Paused...") elif abort: self.set_status("Shot aborted") elif restarted: self.set_status('A device was restarted during transition_to_buffered. Shot aborted') else: self.set_status("One or more devices is in an error state. Queue Paused...") # Abort the run for all devices in use: # need to recreate the queue here because we don't want to hear from devices that are still transitioning to buffered mode self.current_queue = Queue.Queue() for tab in devices_in_use.values(): # We call abort buffered here, because if each tab is either in mode=BUFFERED or transition_to_buffered failed in which case # it should have called abort_transition_to_buffered itself and returned to manual mode # Since abort buffered will only run in mode=BUFFERED, and the state is not queued indefinitely (aka it is deleted if we are not in mode=BUFFERED) # this is the correct method call to make for either case tab.abort_buffered(self.current_queue) # We don't need to check the results of this function call because it will either be successful, or raise a visible error in the tab. # disconnect restart signal from tabs inmain(tab.disconnect_restart_receiver,restart_function) # disconnect abort button and disable inmain(self._ui.queue_abort_button.clicked.disconnect,abort_function) inmain(self._ui.queue_abort_button.setEnabled,False) # Start a new iteration continue ########################################################################################################################################## # SCIENCE! # ########################################################################################################################################## # Get front panel data, but don't save it to the h5 file until the experiment ends: states,tab_positions,window_data,plugin_data = self.BLACS.front_panel_settings.get_save_data() self.set_status(now_running_text+"<br>Running...(program time: %.3fs)"%(time.time() - start_time)) # A Queue for event-based notification of when the experiment has finished. experiment_finished_queue = Queue.Queue() logger.debug('About to start the master pseudoclock') # Do not start until delay time specificed by last sequence has expired self._timer.wait() # Start the timer to block until the next run starts self._timer.start( min_time, countdown_queue=self.BLACS._countdown_queue, countdown_mode='precent_done') run_time = time.localtime() #TODO: fix potential race condition if BLACS is closing when this line executes? self.BLACS.tablist[self.master_pseudoclock].start_run(experiment_finished_queue) # Wait for notification of the end of run: abort = False restarted = False done = False while not (abort or restarted or done): try: done = experiment_finished_queue.get(timeout=0.5) == 'done' except Queue.Empty: pass try: # Poll self.current_queue for abort signal from button or device restart device_name, result = self.current_queue.get_nowait() if (device_name == 'Queue Manager' and result == 'abort'): abort = True if result == 'restart': restarted = True # Check for error states in tabs for device_name, tab in devices_in_use.items(): if self.get_device_error_state(device_name,devices_in_use): restarted = True except Queue.Empty: pass if abort or restarted: for devicename, tab in devices_in_use.items(): if tab.mode == MODE_BUFFERED: tab.abort_buffered(self.current_queue) # disconnect restart signal from tabs inmain(tab.disconnect_restart_receiver,restart_function) # Disable abort button inmain(self._ui.queue_abort_button.clicked.disconnect,abort_function) inmain(self._ui.queue_abort_button.setEnabled,False) if restarted: self.manager_paused = True self.prepend(path) self.set_status("Device restarted mid-shot. Shot aborted, Queue paused.") elif abort: self.set_status("Shot aborted") if abort or restarted: # after disabling the abort button, we now start a new iteration continue logger.info('Run complete') self.set_status(now_running_text+"<br>Sequence done, saving data...") # End try/except block here except Exception: logger.exception("Error in queue manager execution. Queue paused.") # clean up the h5 file self.manager_paused = True # clean the h5 file: self.clean_h5_file(path, 'temp.h5') try: os.remove(path) os.rename('temp.h5', path) except WindowsError if platform.system() == 'Windows' else None: logger.warning('Couldn\'t delete failed run file %s, another process may be using it. Using alternate filename for second attempt.'%path) os.rename('temp.h5', path.replace('.h5','_retry.h5')) path = path.replace('.h5','_retry.h5') # Put it back at the start of the queue: self.prepend(path) # Need to put devices back in manual mode self.current_queue = Queue.Queue() for devicename, tab in devices_in_use.items(): if tab.mode == MODE_BUFFERED or tab.mode == MODE_TRANSITION_TO_BUFFERED: tab.abort_buffered(self.current_queue) # disconnect restart signal from tabs inmain(tab.disconnect_restart_receiver,restart_function) self.set_status("Error occured in Queue Manager. Queue Paused. \nPlease make sure all devices are back in manual mode before unpausing the queue") # disconnect and disable abort button inmain(self._ui.queue_abort_button.clicked.disconnect,abort_function) inmain(self._ui.queue_abort_button.setEnabled,False) # Start a new iteration continue ########################################################################################################################################## # SCIENCE OVER! # ########################################################################################################################################## ########################################################################################################################################## # Transition to manual # ########################################################################################################################################## # start new try/except block here try: with h5py.File(path,'r+') as hdf5_file: self.BLACS.front_panel_settings.store_front_panel_in_h5(hdf5_file,states,tab_positions,window_data,plugin_data,save_conn_table = False) with h5py.File(path,'r+') as hdf5_file: data_group = hdf5_file['/'].create_group('data') # stamp with the run time of the experiment hdf5_file.attrs['run time'] = time.strftime('%Y%m%dT%H%M%S',run_time) for devicename, tab in devices_in_use.items(): if tab.mode == MODE_BUFFERED: tab.transition_to_manual(self.current_queue) error_condition = False transition_list = devices_in_use.copy() while transition_list and not error_condition: try: # Wait for a device to transition_to_manual: logger.debug('Waiting for the following devices to finish transitioning to manual mode: %s'%str(transition_list)) device_name, result = self.current_queue.get(timeout=2) if (device_name == 'Queue Manager' and result == 'abort'): # Ignore any abort signals left in the queue, it # is too late to abort in any case continue logger.debug('%s finished transitioning to manual mode' % device_name) if result == 'fail': error_condition = True if result == 'restart': error_condition = True if self.get_device_error_state(device_name, transition_list): error_condition = True del transition_list[device_name] # disconnect restart signal from tab inmain(tab.disconnect_restart_receiver,restart_function) except queue.Empty: # It's been 2 seconds without a device finishing # transitioning to manual. Is there an error? for name in transition_list: if self.get_device_error_state(name,transition_list): error_condition = True if error_condition: break if error_condition: self.set_status("Error during transtion to manual. Queue Paused.") # TODO: Kind of dodgy raising an exception here... raise Exception('A device failed during transition to manual') # All data written, now run all PostProcessing functions SavedFunctions = labscript_utils.h5_scripting.get_all_saved_functions(path) with h5py.File(path, 'r+') as hdf5_file: for SavedFunction in SavedFunctions: try: result = SavedFunction.custom_call(hdf5_file, **shot_globals) except Exception: import zprocess zprocess.raise_exception_in_thread(sys.exc_info()) result = {} logger.error('Post Processing function did not execute correctly') try: self.DynamicGlobals.update(result) except: logger.error('Post Processing function did not return a dict type') inmain(self._ui.Globals_tableWidget.setRowCount, len(self.DynamicGlobals)) for i, key in enumerate(self.DynamicGlobals): inmain(self._ui.Globals_tableWidget.setItem, i, 0, QTableWidgetItem(key)) inmain(self._ui.Globals_tableWidget.setItem, i, 1, QTableWidgetItem( str(self.DynamicGlobals[key]) )) except Exception as e: logger.exception("Error in queue manager execution. Queue paused.") # clean up the h5 file self.manager_paused = True # clean the h5 file: self.clean_h5_file(path, 'temp.h5') # self.SavedFunctions = labscript_utils.h5_scripting.get_all_saved_functions(path(path, 'temp.h5')) try: os.remove(path) os.rename('temp.h5', path) except WindowsError if platform.system() == 'Windows' else None: logger.warning('Couldn\'t delete failed run file %s, another process may be using it. Using alternate filename for second attempt.'%path) os.rename('temp.h5', path.replace('.h5','_retry.h5')) path = path.replace('.h5','_retry.h5') # Put it back at the start of the queue: self.prepend(path) # Need to put devices back in manual mode. Since the experiment is over before this try/except block begins, we can # safely call transition_to_manual() on each device tab self.current_queue = Queue.Queue() for devicename, tab in devices_in_use.items(): if tab.mode == MODE_BUFFERED: tab.transition_to_manual(self.current_queue) # disconnect restart signal from tabs inmain(tab.disconnect_restart_receiver,restart_function) self.set_status("Error occured in Queue Manager. Queue Paused. \nPlease make sure all devices are back in manual mode before unpausing the queue") continue ########################################################################################################################################## # Analysis Submission # ########################################################################################################################################## logger.info('All devices are back in static mode.') # Submit to the analysis server self.BLACS.analysis_submission.get_queue().put(['file', path]) ########################################################################################################################################## # Plugin callbacks # ########################################################################################################################################## for plugin in self.BLACS.plugins.values(): callbacks = plugin.get_callbacks() if isinstance(callbacks, dict) and 'shot_complete' in callbacks: try: callbacks['shot_complete'](path) except Exception: logger.exception("Plugin callback raised an exception") ########################################################################################################################################## # Repeat Experiment? # ########################################################################################################################################## if (self.manager_repeat == 1) or (self.manager_repeat == 2 and self.get_num_files() == 0): # Resubmit job to the bottom of the queue: try: message = self.process_request(path) logger.info(message) except: # TODO: make this error popup for the user logger.error('Failed to copy h5_file (%s) for repeat run'%path) self.set_status("Idle") logger.info('Stopping')
def manage(self): logger = logging.getLogger('BLACS.queue_manager.thread') # While the program is running! logger.info('starting') # HDF5 prints lots of errors by default, for things that aren't # actually errors. These are silenced on a per thread basis, # and automatically silenced in the main thread when h5py is # imported. So we'll silence them in this thread too: h5py._errors.silence_errors() # This name stores the queue currently being used to # communicate with tabs, so that abort signals can be put # to it when those tabs never respond and are restarted by # the user. self.current_queue = Queue.Queue() #TODO: put in general configuration timeout_limit = 300 #seconds self.set_status("Idle") while self.manager_running: # If the pause button is pushed in, sleep if self.manager_paused: if self.get_status() == "Idle": logger.info('Paused') self.set_status("Queue Paused") time.sleep(1) continue # Get the top file try: path = self.get_next_file() now_running_text = 'Now running: <b>%s</b>'%os.path.basename(path) self.set_status(now_running_text) logger.info('Got a file: %s'%path) except: # If no files, sleep for 1s, self.set_status("Idle") time.sleep(1) continue devices_in_use = {} transition_list = {} start_time = time.time() self.current_queue = Queue.Queue() # Function to be run when abort button is clicked def abort_function(): try: # Set device name to "Queue Manager" which will never be a labscript device name # as it is not a valid python variable name (has a space in it!) self.current_queue.put(['Queue Manager', 'abort']) except Exception: logger.exception('Could not send abort message to the queue manager') def restart_function(device_name): try: self.current_queue.put([device_name, 'restart']) except Exception: logger.exception('Could not send restart message to the queue manager for device %s'%device_name) ########################################################################################################################################## # transition to buffered # ########################################################################################################################################## try: # A Queue for event-based notification when the tabs have # completed transitioning to buffered: timed_out = False error_condition = False abort = False restarted = False self.set_status(now_running_text+"<br>Transitioning to Buffered") # Enable abort button, and link in current_queue: inmain(self._ui.queue_abort_button.clicked.connect,abort_function) inmain(self._ui.queue_abort_button.setEnabled,True) # Ready to run file: assume that the file has _not_ been compiled and compile it # Extract script globals, and update them from the blacs mantained dictionary of globals. shot_globals = get_shot_globals(path) shot_globals.update(self.DynamicGlobals) with h5py.File(path, "a") as hdf5_file: set_shot_globals(hdf5_file, shot_globals) # Compile file compile_h5(path) # Run file with h5py.File(path, "r+") as hdf5_file: min_time = hdf5_file.attrs['min_time'] h5_file_devices = hdf5_file['devices/'].keys() for name in h5_file_devices: try: # Connect restart signal from tabs to current_queue and transition the device to buffered mode success = self.transition_device_to_buffered(name,transition_list,path,restart_function) if not success: logger.error('%s has an error condition, aborting run' % name) error_condition = True break except Exception as e: logger.error('Exception while transitioning %s to buffered mode. Exception was: %s'%(name,str(e))) error_condition = True break devices_in_use = transition_list.copy() while transition_list and not error_condition: try: # Wait for a device to transtition_to_buffered: logger.debug('Waiting for the following devices to finish transitioning to buffered mode: %s'%str(transition_list)) device_name, result = self.current_queue.get(timeout=2) #Handle abort button signal if device_name == 'Queue Manager' and result == 'abort': # we should abort the run logger.info('abort signal received from GUI') abort = True break if result == 'fail': logger.info('abort signal received during transition to buffered of %s' % device_name) error_condition = True break elif result == 'restart': logger.info('Device %s was restarted, aborting shot.'%device_name) restarted = True break logger.debug('%s finished transitioning to buffered mode' % device_name) # The tab says it's done, but does it have an error condition? if self.get_device_error_state(device_name,transition_list): logger.error('%s has an error condition, aborting run' % device_name) error_condition = True break del transition_list[device_name] except Queue.Empty: # It's been 2 seconds without a device finishing # transitioning to buffered. Is there an error? for name in transition_list: if self.get_device_error_state(name,transition_list): error_condition = True break if error_condition: break # Has programming timed out? if time.time() - start_time > timeout_limit: logger.error('Transitioning to buffered mode timed out') timed_out = True break # Handle if we broke out of loop due to timeout or error: if timed_out or error_condition or abort or restarted: # Pause the queue, re add the path to the top of the queue, and set a status message! # only if we aren't responding to an abort click if not abort: self.manager_paused = True self.prepend(path) if timed_out: self.set_status("Device programming timed out. Queue Paused...") elif abort: self.set_status("Shot aborted") elif restarted: self.set_status('A device was restarted during transition_to_buffered. Shot aborted') else: self.set_status("One or more devices is in an error state. Queue Paused...") # Abort the run for all devices in use: # need to recreate the queue here because we don't want to hear from devices that are still transitioning to buffered mode self.current_queue = Queue.Queue() for tab in devices_in_use.values(): # We call abort buffered here, because if each tab is either in mode=BUFFERED or transition_to_buffered failed in which case # it should have called abort_transition_to_buffered itself and returned to manual mode # Since abort buffered will only run in mode=BUFFERED, and the state is not queued indefinitely (aka it is deleted if we are not in mode=BUFFERED) # this is the correct method call to make for either case tab.abort_buffered(self.current_queue) # We don't need to check the results of this function call because it will either be successful, or raise a visible error in the tab. # disconnect restart signal from tabs inmain(tab.disconnect_restart_receiver,restart_function) # disconnect abort button and disable inmain(self._ui.queue_abort_button.clicked.disconnect,abort_function) inmain(self._ui.queue_abort_button.setEnabled,False) # Start a new iteration continue ########################################################################################################################################## # SCIENCE! # ########################################################################################################################################## # Get front panel data, but don't save it to the h5 file until the experiment ends: states,tab_positions,window_data,plugin_data = self.BLACS.front_panel_settings.get_save_data() self.set_status(now_running_text+"<br>Running...(program time: %.3fs)"%(time.time() - start_time)) # A Queue for event-based notification of when the experiment has finished. experiment_finished_queue = Queue.Queue() logger.debug('About to start the master pseudoclock') # Do not start until delay time specificed by last sequence has expired self._timer.wait() # Start the timer to block until the next run starts self._timer.start( min_time, countdown_queue=self.BLACS._countdown_queue, countdown_mode='precent_done') run_time = time.localtime() #TODO: fix potential race condition if BLACS is closing when this line executes? self.BLACS.tablist[self.master_pseudoclock].start_run(experiment_finished_queue) # Wait for notification of the end of run: abort = False restarted = False while result != 'done': try: result = experiment_finished_queue.get(timeout=0.5) except Queue.Empty: pass try: # Poll self.current_queue for abort signal from button or device restart device_name, result = self.current_queue.get(timeout=0.5) if (device_name == 'Queue Manager' and result == 'abort'): abort = True break elif result == 'restart': restarted = True break # Check for error states in tabs for device_name, tab in devices_in_use.items(): if self.get_device_error_state(device_name,devices_in_use): restarted = True break if restarted: break except Queue.Empty: pass if abort or restarted: for devicename, tab in devices_in_use.items(): if tab.mode == MODE_BUFFERED: tab.abort_buffered(self.current_queue) # disconnect restart signal from tabs inmain(tab.disconnect_restart_receiver,restart_function) # Disable abort button inmain(self._ui.queue_abort_button.clicked.disconnect,abort_function) inmain(self._ui.queue_abort_button.setEnabled,False) if restarted: self.manager_paused = True self.prepend(path) self.set_status("Device restarted mid-shot. Shot aborted, Queue paused.") elif abort: self.set_status("Shot aborted") if abort or restarted: # after disabling the abort button, we now start a new iteration continue logger.info('Run complete') self.set_status(now_running_text+"<br>Sequence done, saving data...") # End try/except block here except Exception: logger.exception("Error in queue manager execution. Queue paused.") # clean up the h5 file self.manager_paused = True # clean the h5 file: self.clean_h5_file(path, 'temp.h5') try: os.remove(path) os.rename('temp.h5', path) except WindowsError if platform.system() == 'Windows' else None: logger.warning('Couldn\'t delete failed run file %s, another process may be using it. Using alternate filename for second attempt.'%path) os.rename('temp.h5', path.replace('.h5','_retry.h5')) path = path.replace('.h5','_retry.h5') # Put it back at the start of the queue: self.prepend(path) # Need to put devices back in manual mode self.current_queue = Queue.Queue() for devicename, tab in devices_in_use.items(): if tab.mode == MODE_BUFFERED or tab.mode == MODE_TRANSITION_TO_BUFFERED: tab.abort_buffered(self.current_queue) # disconnect restart signal from tabs inmain(tab.disconnect_restart_receiver,restart_function) self.set_status("Error occured in Queue Manager. Queue Paused. \nPlease make sure all devices are back in manual mode before unpausing the queue") # disconnect and disable abort button inmain(self._ui.queue_abort_button.clicked.disconnect,abort_function) inmain(self._ui.queue_abort_button.setEnabled,False) # Start a new iteration continue ########################################################################################################################################## # SCIENCE OVER! # ########################################################################################################################################## ########################################################################################################################################## # Transition to manual # ########################################################################################################################################## # start new try/except block here try: with h5py.File(path,'r+') as hdf5_file: self.BLACS.front_panel_settings.store_front_panel_in_h5(hdf5_file,states,tab_positions,window_data,plugin_data,save_conn_table = False) with h5py.File(path,'r+') as hdf5_file: data_group = hdf5_file['/'].create_group('data') # stamp with the run time of the experiment hdf5_file.attrs['run time'] = time.strftime('%Y%m%dT%H%M%S',run_time) # A Queue for event-based notification of when the devices have transitioned to static mode: # Shouldn't need to recreate the queue: self.current_queue = Queue.Queue() # TODO: unserialise this if everything is using zprocess.locking # only transition one device to static at a time, # since writing data to the h5 file can potentially # happen at this stage: error_condition = False # This is far more complicated than it needs to be once transition_to_manual is unserialised! response_list = {} for device_name, tab in devices_in_use.items(): if device_name not in response_list: tab.transition_to_manual(self.current_queue) while True: # TODO: make the call to current_queue.get() timeout # and periodically check for error condition on the tab got_device_name, result = self.current_queue.get() # if the response is not for this device, then save it for later! if device_name != got_device_name: response_list[got_device_name] = result else: break else: result = response_list[device_name] # Check for abort signal from device restart if result == 'fail': error_condition = True if result == 'restart': error_condition = True if self.get_device_error_state(device_name,devices_in_use): error_condition = True # Once device has transitioned_to_manual, disconnect restart signal inmain(tab.disconnect_restart_receiver,restart_function) if error_condition: self.set_status("Error during transtion to manual. Queue Paused.") # TODO: Kind of dodgy raising an exception here... raise Exception('A device failed during transition to manual') # All data written, now run all PostProcessing functions SavedFunctions = labscript_utils.h5_scripting.get_all_saved_functions(path) with h5py.File(path, 'r+') as hdf5_file: for SavedFunction in SavedFunctions: try: result = SavedFunction(hdf5_file, **shot_globals) except: result = {} logger.error('Post Processing function did not execute correctly') try: self.DynamicGlobals.update(result) except: logger.error('Post Processing function did not return a dict type') inmain(self._ui.Globals_tableWidget.setRowCount, len(self.DynamicGlobals)) for i, key in enumerate(self.DynamicGlobals): inmain(self._ui.Globals_tableWidget.setItem, i, 0, QTableWidgetItem(key)) inmain(self._ui.Globals_tableWidget.setItem, i, 1, QTableWidgetItem( str(self.DynamicGlobals[key]) )) except Exception as e: logger.exception("Error in queue manager execution. Queue paused.") # clean up the h5 file self.manager_paused = True # clean the h5 file: self.clean_h5_file(path, 'temp.h5') try: os.remove(path) os.rename('temp.h5', path) except WindowsError if platform.system() == 'Windows' else None: logger.warning('Couldn\'t delete failed run file %s, another process may be using it. Using alternate filename for second attempt.'%path) os.rename('temp.h5', path.replace('.h5','_retry.h5')) path = path.replace('.h5','_retry.h5') # Put it back at the start of the queue: self.prepend(path) # Need to put devices back in manual mode. Since the experiment is over before this try/except block begins, we can # safely call transition_to_manual() on each device tab # TODO: Not serialised...could be bad with older BIAS versions :( self.current_queue = Queue.Queue() for devicename, tab in devices_in_use.items(): if tab.mode == MODE_BUFFERED: tab.transition_to_manual(self.current_queue) # disconnect restart signal from tabs inmain(tab.disconnect_restart_receiver,restart_function) self.set_status("Error occured in Queue Manager. Queue Paused. \nPlease make sure all devices are back in manual mode before unpausing the queue") continue ########################################################################################################################################## # Analysis Submission # ########################################################################################################################################## logger.info('All devices are back in static mode.') # Submit to the analysis server self.BLACS.analysis_submission.get_queue().put(['file', path]) ########################################################################################################################################## # Repeat Experiment? # ########################################################################################################################################## if (self.manager_repeat == 1) or (self.manager_repeat == 2 and self.get_num_files() == 0): # Resubmit job to the bottom of the queue: try: message = self.process_request(path) logger.info(message) except: # TODO: make this error popup for the user logger.error('Failed to copy h5_file (%s) for repeat run'%path) self.set_status("Idle") logger.info('Stopping')