def get_datastore(cls, datastore_name=None, variant=DS_BASE, config=None, container=None, profile=None, scope=None): #log.info("get_datastore(%s, variant=%s, profile=%s, scope=%s, config=%s)", datastore_name, variant, profile, scope, "") # Step 1: Get datastore server config if not config and container: config = container.CFG if config: if "container" in config: server_cfg = cls.get_server_config(config) else: server_cfg = config config = None if not server_cfg: raise BadRequest("No config available to determine datastore") # Step 2: Find type specific implementation class if config: server_types = get_safe(config, "container.datastore.server_types", None) if not server_types: # Some tests fudge the CFG - make it more lenient #raise BadRequest("Server types not configured!") variant_store = cls.get_datastore_class( server_cfg, variant=variant) else: server_type = server_cfg.get("type", "postgresql") type_cfg = server_types.get(server_type, None) if not type_cfg: raise BadRequest( "Server type '%s' not configured!" % server_type) variant_store = type_cfg.get(variant, cls.DS_BASE) else: # Fallback in case a server config was given (NOT NICE) variant_store = cls.get_datastore_class( server_cfg, variant=variant) # Step 3: Instantiate type specific implementation store_class = named_any(variant_store) profile = profile or DataStore.DS_PROFILE_MAPPING.get( datastore_name, DataStore.DS_PROFILE.BASIC) log.debug("get_datastore(%s, profile=%s, scope=%s, variant=%s) -> %s", datastore_name, profile, scope, variant, store_class.__name__) store = store_class( datastore_name=datastore_name, config=server_cfg, profile=profile, scope=scope) return store
def get_datastore(cls, datastore_name=None, variant=DS_BASE, config=None, container=None, profile=None, scope=None): #log.info("get_datastore(%s, variant=%s, profile=%s, scope=%s, config=%s)", datastore_name, variant, profile, scope, "") # Step 1: Get datastore server config if not config and container: config = container.CFG if config: if "container" in config: server_cfg = cls.get_server_config(config) else: server_cfg = config config = None if not server_cfg: raise BadRequest("No config available to determine datastore") # Step 2: Find type specific implementation class if config: server_types = get_safe(config, "container.datastore.server_types", None) if not server_types: # Some tests fudge the CFG - make it more lenient #raise BadRequest("Server types not configured!") variant_store = cls.get_datastore_class(server_cfg, variant=variant) else: server_type = server_cfg.get("type", "postgresql") type_cfg = server_types.get(server_type, None) if not type_cfg: raise BadRequest("Server type '%s' not configured!" % server_type) variant_store = type_cfg.get(variant, cls.DS_BASE) else: # Fallback in case a server config was given (NOT NICE) variant_store = cls.get_datastore_class(server_cfg, variant=variant) # Step 3: Instantiate type specific implementation store_class = named_any(variant_store) profile = profile or DataStore.DS_PROFILE_MAPPING.get( datastore_name, DataStore.DS_PROFILE.BASIC) log.debug("get_datastore(%s, profile=%s, scope=%s, variant=%s) -> %s", datastore_name, profile, scope, variant, store_class.__name__) store = store_class(datastore_name=datastore_name, config=server_cfg, profile=profile, scope=scope) return store
def stop(self): log.debug('container management stopping') with self.lock: self.receiver.stop() self.sender.close() self.running = False log.debug('container management stopped') self.container.stats_mgr.stop()
def bootstrap_pyon(logging_config_override=None, pyon_cfg=None): """ This function initializes the core elements of the Pyon framework in a controlled way. It does not initialize the ION container or the ION system. @param logging_config_override A dict to initialize the Python logging subsystem (None loads default files) @param pyon_cfg A DotDict with the fully loaded pyon configuration to merge into CFG (None loads default files) """ log.info("pyon.bootstrap (bootstrap_pyon) executing...") # Make sure Pyon is only initialized only once global pyon_initialized if pyon_initialized: log.warn("WARNING -- bootstrap_pyon() called again!") return # ENVIRONMENT. Check we are called in an expected environment (files, directories, etc) assert_environment() # LOGGING. Initialize logging from config if not logutil.is_logging_configured(): logutil.configure_logging( logutil.DEFAULT_LOGGING_PATHS, logging_config_override=logging_config_override) # YAML patch: OrderedDicts instead of dicts from pyon.util.yaml_ordered_dict import apply_yaml_patch apply_yaml_patch() # CONFIG. Initialize pyon global configuration from local files set_config(pyon_cfg) log.debug("CFG set to %s", CFG) # OBJECTS. Object and message definitions. from pyon.core.registry import IonObjectRegistry global _obj_registry _obj_registry = IonObjectRegistry() # SERVICES. Service definitions # TODO: change the following to read service definitions from directory and import selectively from pyon.ion.service import IonServiceRegistry import interface.services global _service_registry _service_registry = IonServiceRegistry() _service_registry.load_service_mods(interface.services) _service_registry.build_service_map() # RESOURCES. Load and initialize definitions from pyon.ion import resource resource.load_definitions() # Fix a weird bug on Ubuntu that resets time.sleep to un-monkey patched version on import threading from gevent import monkey monkey.patch_time() # Set initialized flag pyon_initialized = True log.debug("Pyon initialized OK")
def stop(self): log.debug("container management stopping") with self.lock: self.receiver.stop() self.sender.close() self.running = False log.debug("container management stopped") self.container_tracer.stop_tracing()
def _check_condition(self): try: value = self._condition() if value: self._callback(value) except Exception as e: log.debug('stopping poller after exception', exc_info=True) self.shutdown() if self._on_exception: self._on_exception(e)
def start(self): self.container.stats_mgr = ContainerStatsManager(self.container) self.container.stats_mgr.start() ## create queue listener and publisher self.sender = EventPublisher(event_type="ContainerManagementResult") self.receiver = EventSubscriber(event_type="ContainerManagementRequest", callback=self._receive_event) with self.lock: self.running = True self.receiver.start() log.debug('Container ready for container management requests')
def bootstrap_pyon(logging_config_override=None, pyon_cfg=None): """ This function initializes the core elements of the Pyon framework in a controlled way. It does not initialize the ION container or the ION system. @param logging_config_override A dict to initialize the Python logging subsystem (None loads default files) @param pyon_cfg A DotDict with the fully loaded pyon configuration to merge into CFG (None loads default files) """ print "pyon: pyon.bootstrap (bootstrap_pyon) executing..." # Make sure Pyon is only initialized only once global pyon_initialized if pyon_initialized: print "pyon: WARNING -- bootstrap_pyon() called again!" return # ENVIRONMENT. Check we are called in an expected environment (files, directories, etc) assert_environment() # LOGGING. Initialize logging from config if not logutil.is_logging_configured(): logutil.configure_logging(logutil.DEFAULT_LOGGING_PATHS, logging_config_override=logging_config_override) # YAML patch: OrderedDicts instead of dicts from pyon.util.yaml_ordered_dict import apply_yaml_patch apply_yaml_patch() # CONFIG. Initialize pyon global configuration from local files set_config(pyon_cfg) log.debug("pyon: CFG set to %s", CFG) # OBJECTS. Object and message definitions. from pyon.core.registry import IonObjectRegistry global _obj_registry _obj_registry = IonObjectRegistry() # SERVICES. Service definitions # TODO: change the following to read service definitions from directory and import selectively from pyon.ion.service import IonServiceRegistry import interface.services global _service_registry _service_registry = IonServiceRegistry() _service_registry.load_service_mods(interface.services) _service_registry.build_service_map() # RESOURCES. Load and initialize definitions from pyon.ion import resource resource.load_definitions() # Fix a weird bug on Ubuntu that resets time.sleep to un-monkey patched version on import threading from gevent import monkey; monkey.patch_time() # Set initialized flag pyon_initialized = True log.debug("pyon: initialized OK")
def test_logging_handler(self): """ initial log level for ion.process.event is INFO -- test we can change it to TRACE """ config.replace_configuration(os.path.join(TEST_DIR, 'logging.yml')) log.debug('this should probably not be logged') self.assertFalse(log.isEnabledFor(TRACE)) # handler = pyon.container.management.LogLevelHandler() action = IonObject(OT.ChangeLogLevel, logger='pyon.container', level='TRACE') handler.handle_request(action) # self.assertTrue(log.isEnabledFor(TRACE))
def start(self): # Install the container tracer (could be its own self.container_tracer = ContainerTracer() self.container_tracer.start_tracing() self.container.tracer = CallTracer self.container.tracer.configure(CFG.get_safe("container.tracer", {})) ## create queue listener and publisher self.sender = EventPublisher(event_type="ContainerManagementResult") self.receiver = EventSubscriber(event_type="ContainerManagementRequest", callback=self._receive_event) with self.lock: self.running = True self.receiver.start() log.debug("Container ready for container management requests")
def test_logging_clear(self): """ initial log level for ion.process.event is INFO -- test that we can clear it (root level WARN should apply) """ config.replace_configuration(os.path.join(TEST_DIR, 'logging.yml')) log.debug('this should probably not be logged') self.assertTrue(log.isEnabledFor(logging.INFO), msg=repr(log.__dict__)) # handler = pyon.container.management.LogLevelHandler() action = IonObject(OT.ChangeLogLevel, logger='pyon.container', level='WARN') handler.handle_request(action) self.assertFalse(log.isEnabledFor(logging.INFO))
def _perform_action(self, action): handlers = self._get_handlers(action) if not handlers: log.info('action accepted but no handlers found: %s', action) result = 'unhandled' self.sender.publish_event(origin=self.container.id, action=action, outcome=str(result)) log.debug('received action: %s, outcome: %s', action, result) else: for handler in handlers: try: result = handler.handle_request(action) or "completed" except Exception,e: log.error("handler %r failed to perform action: %s", handler, action, exc_info=True) result = e self.sender.publish_event(origin=self.container.id, action=action, outcome=str(result)) log.debug('performed action: %s, outcome: %s', action, result)
def _perform_action(self, action): handlers = self._get_handlers(action) if not handlers: log.info("action accepted but no handlers found: %s", action) result = "unhandled" self.sender.publish_event(origin=self.container.id, action=action, outcome=str(result)) log.debug("received action: %s, outcome: %s", action, result) else: for handler in handlers: try: result = handler.handle_request(action) or "completed" except Exception, e: log.error("handler %r failed to perform action: %s", handler, action, exc_info=True) result = e self.sender.publish_event(origin=self.container.id, action=action, outcome=str(result)) log.debug("performed action: %s, outcome: %s", action, result)
def _receive_event(self, event, headers): with self.lock: if not isinstance(event, ContainerManagementRequest): log.trace("ignoring wrong type event: %r", event) return if not self.running: log.warn("ignoring admin message received after shutdown: %s", event.action) return predicate = ContainerSelector.from_object(event.predicate) if predicate.should_handle(self.container): log.trace("handling admin message: %s", event.action) self._perform_action(event.action) else: log.trace("ignoring admin action: %s", event.action) if SEND_RESULT_IF_NOT_SELECTED: self.sender.publish_event(origin=self.container.id, action=event.action, outcome="not selected") log.debug("received action: %s, outcome: not selected", event.action)
def _receive_event(self, event, headers): with self.lock: if not isinstance(event, ContainerManagementRequest): log.trace('ignoring wrong type event: %r', event) return if not self.running: log.warn('ignoring admin message received after shutdown: %s', event.action) return predicate = ContainerSelector.from_object(event.predicate) if predicate.should_handle(self.container): log.trace('handling admin message: %s', event.action) self._perform_action(event.action) else: log.trace('ignoring admin action: %s', event.action) if SEND_RESULT_IF_NOT_SELECTED: self.sender.publish_event(origin=self.container.id, action=event.action, outcome='not selected') log.debug('received action: %s, outcome: not selected', event.action)
def zip_of_b64(b64_data, title): """ return a zipfile object based on a base64 input string. "title" is a string to provide a helpful error message function returns a (zipfile object, err_msg) tuple... zipfile object will be None on error """ log.debug("decoding base64 zipfile for %s", title) try: zip_str = base64.decodestring(b64_data) except: return None, ("could not base64 decode the supplied %s" % title) log.debug("opening zipfile for %s", title) try: zip_file = StringIO(zip_str) zip_obj = zipfile.ZipFile(zip_file) except: return None, ("could not parse zipfile contained in %s" % title) return zip_obj, ""