def update_status(self, status): """Save/register the loggers' retrieved status report with the API.""" logging.info('Got status: %s', status) now = datetime_obj() for cruise_logger, logger_report in status.items(): cruise_id, logger_id = cruise_logger.split(sep=ID_SEPARATOR, maxsplit=1) logger_config = logger_report.get('config', None) logger_errors = logger_report.get('errors', None) logger_pid = logger_report.get('pid', None) logger_failed = logger_report.get('failed', None) logger_running = logger_report.get('running', None) # Get the most recent corresponding LoggerConfigState from # datastore. If there isn't a most recent, create a dummy that # will get filled in. try: # Get the latest LoggerConfigState for this logger stored_state = LoggerConfigState.objects.filter( logger__name=logger_id, logger__cruise__id=cruise_id).latest('timestamp') except LoggerConfigState.DoesNotExist: # If no existing LoggerConfigState for logger, create one logger = Logger.objects.get(name=logger_id, cruise=cruise_id) config = LoggerConfig.objects.get(name=logger_config, cruise=cruise_id, logger=logger) stored_state = LoggerConfigState(logger=logger, config=config, running=False, failed=False, pid=0, errors='') stored_state.save() # Compare stored LoggerConfigState with the new status. If there # have been changes, reset pk, which will create a new object # when we save. if (logger_errors or not stored_state.running == logger_running or not stored_state.failed == logger_failed or not stored_state.pid == logger_pid): # Otherwise, add changes and save as a new object stored_state.pk = None stored_state.running = logger_running stored_state.failed = logger_failed stored_state.pid = logger_pid stored_state.errors = '\n'.join(logger_errors) # Update last_checked field and save, regardless of whether we # made other changes. stored_state.last_checked = now stored_state.save()
def load_configuration(self, configuration): """Add a complete cruise configuration (id, modes, configs, default) to the data store.""" with self.config_rlock: with transaction.atomic(): cruise_def = configuration.get('cruise', {}) loggers = configuration.get('loggers', None) modes = configuration.get('modes', None) default_mode = configuration.get('default_mode', None) configs = configuration.get('configs', None) if loggers is None: raise ValueError('Cruise definition has no loggers') if modes is None: raise ValueError('Cruise definition has no modes') if configs is None: raise ValueError('Cruise definition has no configs') # We're going in - a select_for_update() locks the Logger table # so no one else can mess until we're done with the transaction. loggers_lock = Logger.objects.select_for_update().all() ################ # Assemble the top-level information about the cruise # definition. If no cruise name, just call it 'Cruise'. cruise_id = cruise_def.get('id', 'Cruise') cruise_start = cruise_def.get('start', None) if cruise_start: cruise_start = datetime_obj(cruise_start, time_format=DATE_FORMAT) cruise_end = cruise_def.get('end', None) if cruise_end: cruise_end = datetime_obj(cruise_end, time_format=DATE_FORMAT) config_filename = cruise_def.get('config_filename', None) # Delete old cruise. There should be only one, but... for old_cruise in Cruise.objects.all(): logging.info('Deleting old cruise "%s"', old_cruise.id) old_cruise.delete() # Create and save the new cruise cruise = Cruise(id=cruise_id, start=cruise_start, end=cruise_end, config_filename=config_filename) cruise.save() ################ # Create the modes for mode_name in modes: logging.info(' Creating mode %s (cruise %s)', mode_name, cruise_id) mode = Mode(name=mode_name, cruise=cruise) mode.save() # Is this mode the default mode for the cruise? if mode_name == default_mode: logging.info(' Setting %s as default mode', mode_name) cruise.default_mode = mode cruise.save() ################ # Create the loggers for logger_name, logger_spec in loggers.items(): logging.info('Creating logger %s (cruise %s)', logger_name, cruise_id) logger = Logger(name=logger_name, cruise=cruise) logger.save() # Create and associate all relevant modes for the logger logger_configs = logger_spec.get('configs', None) if logger_configs is None: raise ValueError( 'Logger %s (cruise %s) has no config declaration' % (logger_name, cruise_id)) # Find the corresponding configuration for config_name in logger_configs: config_spec = configs.get(config_name, None) if config_spec is None: raise ValueError( 'Config %s (declared by logger %s) not found' % (config_name, logger_name)) logging.debug(' Associating config %s with logger %s', config_name, logger_name) logging.debug('config_spec: %s', config_spec) # A minor hack: fold the config's name into the spec if not 'name' in config_spec: config_spec['name'] = config_name config = LoggerConfig( name=config_name, cruise=cruise, logger=logger, config_json=json.dumps(config_spec)) config.save() # Is this logger config part of a mode? for mode_name, mode_dict in modes.items(): logger_config_name = mode_dict.get( logger_name, None) if logger_config_name and logger_config_name == config_name: try: logging.debug( 'modes: %s', Mode.objects.filter(name=mode_name, cruise=cruise)) mode = Mode.objects.get(name=mode_name, cruise=cruise) except Mode.DoesNotExist: raise ValueError( 'Mode %s does not exist?!?' % mode_name) logging.debug( ' Associating config %s with mode %s', config_name, mode_name) config.modes.add(mode) # Is this config in the default mode of this logger? if mode_name == default_mode: logging.debug( ' Setting logger %s to default config: %s', logger_name, config_name) logger.config = config logger.save() logging.info('Cruise loaded') #self.set_active_mode(default_mode) # we now do this outside of load # Let anyone who's interested know that we've got new configurations. self.signal_load()
def load_cruise(self, cruise_config, config_filename=None): """Add a complete cruise configuration (id, modes, configs, default) to the data store.""" cruise_def = cruise_config.get('cruise', {}) loggers = cruise_config.get('loggers', None) modes = cruise_config.get('modes', None) default_mode = cruise_config.get('default_mode', None) configs = cruise_config.get('configs', None) if loggers is None: raise ValueError('Cruise configuration has no loggers') if modes is None: raise ValueError('Cruise configuration has no modes') if configs is None: raise ValueError('Cruise configuration has no configs') ################ # Begin by creating the Cruise object. If no cruise name, use # filename. If no filename, make up a sequential name. cruise_id = cruise_def.get('id', None) if cruise_id is None: cruise_id = config_filename if cruise_id is None: cruise_id = 'cruise_%d' % len(Cruise.objects.all()) # Does this cruise already exist? If so, delete it. # Alternatively, we could throw a ValueError telling user they # have to delete it first. for old_cruise in Cruise.objects.filter(id=cruise_id): #raise ValueError('Cruise %s already exists; delete it first' % cruise_id) logging.warning('Cruise %s already exists - deleting old one', cruise_id) old_cruise.delete() if ID_SEPARATOR in cruise_id: raise ValueError('Illegal character "%s" in cruise id: "%s"' % ID_SEPARATOR, cruise_id) cruise = Cruise(id=cruise_id, config_filename=config_filename) start_time = cruise_def.get('start', None) if start_time: cruise.start = datetime_obj(start_time, time_format=DATE_FORMAT) end_time = cruise_def.get('end', None) if end_time: cruise.end = datetime_obj(end_time, time_format=DATE_FORMAT) cruise.save() ################ # Create the modes for mode_name in modes: logging.info(' Creating mode %s (cruise %s)', mode_name, cruise_id) mode = Mode(name=mode_name, cruise=cruise) mode.save() # Is this mode the default mode for the cruise? if mode_name == default_mode: logging.info(' Setting %s as default mode', mode_name) cruise.default_mode = mode cruise.save() ################ # Create the loggers for logger_name, logger_spec in loggers.items(): logging.info('Creating logger %s (cruise %s)', logger_name, cruise_id) logger = Logger(name=logger_name, cruise=cruise) logger.save() # Create and associate all relevant modes for the logger logger_configs = logger_spec.get('configs', None) if logger_configs is None: raise ValueError('Logger %s (cruise %s) has no config declaration' % (logger_name, cruise_id)) # Find the corresponding configuration for config_name in logger_configs: config_spec = configs.get(config_name, None) if config_spec is None: raise ValueError('Config %s (declared by logger %s) not found' % (config_name, logger_name)) logging.info(' Associating config %s with logger %s', config_name, logger_name) logging.info('config_spec: %s', config_spec) # A minor hack: fold the config's name into the spec if not 'name' in config_spec: config_spec['name'] = config_name config = LoggerConfig(name=config_name, cruise=cruise, logger=logger, config_json=json.dumps(config_spec)) config.save() # Is this logger config part of a mode? for mode_name, mode_dict in modes.items(): logger_config_name = mode_dict.get(logger_name, None) if logger_config_name and logger_config_name == config_name: try: logging.info('modes: %s', Mode.objects.filter(name=mode_name, cruise=cruise)) mode = Mode.objects.get(name=mode_name, cruise=cruise) except Mode.DoesNotExist: raise ValueError('Mode %s does not exist?!?' % mode_name) logging.info(' Associating config %s with mode %s', config_name, mode_name) config.modes.add(mode) # Is this config in the default mode of this logger? if mode_name == default_mode: logging.info(' Setting logger %s to default config: %s', logger_name, config_name) logger.config = config logger.save() logging.info('Cruise %s loaded - setting to default mode %s', cruise_id, default_mode) self.set_mode(cruise_id, default_mode)