def get_datastore(cls,
datastore_name=None,
variant=DS_BASE,
config=None,
container=None,
profile=None,
scope=None):
#log.info("get_datastore(%s, variant=%s, profile=%s, scope=%s, config=%s)", datastore_name, variant, profile, scope, "")
# Step 1: Get datastore server config
if not config and container:
config = container.CFG
if config:
if "container" in config:
server_cfg = cls.get_server_config(config)
else:
server_cfg = config
config = None
if not server_cfg:
raise BadRequest("No config available to determine datastore")
# Step 2: Find type specific implementation class
if config:
server_types = get_safe(config, "container.datastore.server_types",
None)
if not server_types:
# Some tests fudge the CFG - make it more lenient
#raise BadRequest("Server types not configured!")
variant_store = cls.get_datastore_class(
server_cfg, variant=variant)
else:
server_type = server_cfg.get("type", "postgresql")
type_cfg = server_types.get(server_type, None)
if not type_cfg:
raise BadRequest(
"Server type '%s' not configured!" % server_type)
variant_store = type_cfg.get(variant, cls.DS_BASE)
else:
# Fallback in case a server config was given (NOT NICE)
variant_store = cls.get_datastore_class(
server_cfg, variant=variant)
# Step 3: Instantiate type specific implementation
store_class = named_any(variant_store)
profile = profile or DataStore.DS_PROFILE_MAPPING.get(
datastore_name, DataStore.DS_PROFILE.BASIC)
log.debug("get_datastore(%s, profile=%s, scope=%s, variant=%s) -> %s",
datastore_name, profile, scope, variant,
store_class.__name__)
store = store_class(
datastore_name=datastore_name,
config=server_cfg,
profile=profile,
scope=scope)
return store
def get_datastore(cls,
datastore_name=None,
variant=DS_BASE,
config=None,
container=None,
profile=None,
scope=None):
#log.info("get_datastore(%s, variant=%s, profile=%s, scope=%s, config=%s)", datastore_name, variant, profile, scope, "")
# Step 1: Get datastore server config
if not config and container:
config = container.CFG
if config:
if "container" in config:
server_cfg = cls.get_server_config(config)
else:
server_cfg = config
config = None
if not server_cfg:
raise BadRequest("No config available to determine datastore")
# Step 2: Find type specific implementation class
if config:
server_types = get_safe(config, "container.datastore.server_types",
None)
if not server_types:
# Some tests fudge the CFG - make it more lenient
#raise BadRequest("Server types not configured!")
variant_store = cls.get_datastore_class(server_cfg,
variant=variant)
else:
server_type = server_cfg.get("type", "postgresql")
type_cfg = server_types.get(server_type, None)
if not type_cfg:
raise BadRequest("Server type '%s' not configured!" %
server_type)
variant_store = type_cfg.get(variant, cls.DS_BASE)
else:
# Fallback in case a server config was given (NOT NICE)
variant_store = cls.get_datastore_class(server_cfg,
variant=variant)
# Step 3: Instantiate type specific implementation
store_class = named_any(variant_store)
profile = profile or DataStore.DS_PROFILE_MAPPING.get(
datastore_name, DataStore.DS_PROFILE.BASIC)
log.debug("get_datastore(%s, profile=%s, scope=%s, variant=%s) -> %s",
datastore_name, profile, scope, variant,
store_class.__name__)
store = store_class(datastore_name=datastore_name,
config=server_cfg,
profile=profile,
scope=scope)
return store
def stop(self):
log.debug('container management stopping')
with self.lock:
self.receiver.stop()
self.sender.close()
self.running = False
log.debug('container management stopped')
self.container.stats_mgr.stop()
def bootstrap_pyon(logging_config_override=None, pyon_cfg=None):
"""
This function initializes the core elements of the Pyon framework in a controlled way.
It does not initialize the ION container or the ION system.
@param logging_config_override A dict to initialize the Python logging subsystem (None loads default files)
@param pyon_cfg A DotDict with the fully loaded pyon configuration to merge into CFG (None loads default files)
"""
log.info("pyon.bootstrap (bootstrap_pyon) executing...")
# Make sure Pyon is only initialized only once
global pyon_initialized
if pyon_initialized:
log.warn("WARNING -- bootstrap_pyon() called again!")
return
# ENVIRONMENT. Check we are called in an expected environment (files, directories, etc)
assert_environment()
# LOGGING. Initialize logging from config
if not logutil.is_logging_configured():
logutil.configure_logging(
logutil.DEFAULT_LOGGING_PATHS,
logging_config_override=logging_config_override)
# YAML patch: OrderedDicts instead of dicts
from pyon.util.yaml_ordered_dict import apply_yaml_patch
apply_yaml_patch()
# CONFIG. Initialize pyon global configuration from local files
set_config(pyon_cfg)
log.debug("CFG set to %s", CFG)
# OBJECTS. Object and message definitions.
from pyon.core.registry import IonObjectRegistry
global _obj_registry
_obj_registry = IonObjectRegistry()
# SERVICES. Service definitions
# TODO: change the following to read service definitions from directory and import selectively
from pyon.ion.service import IonServiceRegistry
import interface.services
global _service_registry
_service_registry = IonServiceRegistry()
_service_registry.load_service_mods(interface.services)
_service_registry.build_service_map()
# RESOURCES. Load and initialize definitions
from pyon.ion import resource
resource.load_definitions()
# Fix a weird bug on Ubuntu that resets time.sleep to un-monkey patched version on import threading
from gevent import monkey
monkey.patch_time()
# Set initialized flag
pyon_initialized = True
log.debug("Pyon initialized OK")
def stop(self):
log.debug("container management stopping")
with self.lock:
self.receiver.stop()
self.sender.close()
self.running = False
log.debug("container management stopped")
self.container_tracer.stop_tracing()
def _check_condition(self):
try:
value = self._condition()
if value:
self._callback(value)
except Exception as e:
log.debug('stopping poller after exception', exc_info=True)
self.shutdown()
if self._on_exception:
self._on_exception(e)
def _check_condition(self):
try:
value = self._condition()
if value:
self._callback(value)
except Exception as e:
log.debug('stopping poller after exception', exc_info=True)
self.shutdown()
if self._on_exception:
self._on_exception(e)
def start(self):
self.container.stats_mgr = ContainerStatsManager(self.container)
self.container.stats_mgr.start()
## create queue listener and publisher
self.sender = EventPublisher(event_type="ContainerManagementResult")
self.receiver = EventSubscriber(event_type="ContainerManagementRequest", callback=self._receive_event)
with self.lock:
self.running = True
self.receiver.start()
log.debug('Container ready for container management requests')
def bootstrap_pyon(logging_config_override=None, pyon_cfg=None):
"""
This function initializes the core elements of the Pyon framework in a controlled way.
It does not initialize the ION container or the ION system.
@param logging_config_override A dict to initialize the Python logging subsystem (None loads default files)
@param pyon_cfg A DotDict with the fully loaded pyon configuration to merge into CFG (None loads default files)
"""
print "pyon: pyon.bootstrap (bootstrap_pyon) executing..."
# Make sure Pyon is only initialized only once
global pyon_initialized
if pyon_initialized:
print "pyon: WARNING -- bootstrap_pyon() called again!"
return
# ENVIRONMENT. Check we are called in an expected environment (files, directories, etc)
assert_environment()
# LOGGING. Initialize logging from config
if not logutil.is_logging_configured():
logutil.configure_logging(logutil.DEFAULT_LOGGING_PATHS, logging_config_override=logging_config_override)
# YAML patch: OrderedDicts instead of dicts
from pyon.util.yaml_ordered_dict import apply_yaml_patch
apply_yaml_patch()
# CONFIG. Initialize pyon global configuration from local files
set_config(pyon_cfg)
log.debug("pyon: CFG set to %s", CFG)
# OBJECTS. Object and message definitions.
from pyon.core.registry import IonObjectRegistry
global _obj_registry
_obj_registry = IonObjectRegistry()
# SERVICES. Service definitions
# TODO: change the following to read service definitions from directory and import selectively
from pyon.ion.service import IonServiceRegistry
import interface.services
global _service_registry
_service_registry = IonServiceRegistry()
_service_registry.load_service_mods(interface.services)
_service_registry.build_service_map()
# RESOURCES. Load and initialize definitions
from pyon.ion import resource
resource.load_definitions()
# Fix a weird bug on Ubuntu that resets time.sleep to un-monkey patched version on import threading
from gevent import monkey; monkey.patch_time()
# Set initialized flag
pyon_initialized = True
log.debug("pyon: initialized OK")
def test_logging_handler(self):
""" initial log level for ion.process.event is INFO -- test we can change it to TRACE """
config.replace_configuration(os.path.join(TEST_DIR, 'logging.yml'))
log.debug('this should probably not be logged')
self.assertFalse(log.isEnabledFor(TRACE))
#
handler = pyon.container.management.LogLevelHandler()
action = IonObject(OT.ChangeLogLevel, logger='pyon.container', level='TRACE')
handler.handle_request(action)
#
self.assertTrue(log.isEnabledFor(TRACE))
def start(self):
# Install the container tracer (could be its own
self.container_tracer = ContainerTracer()
self.container_tracer.start_tracing()
self.container.tracer = CallTracer
self.container.tracer.configure(CFG.get_safe("container.tracer", {}))
## create queue listener and publisher
self.sender = EventPublisher(event_type="ContainerManagementResult")
self.receiver = EventSubscriber(event_type="ContainerManagementRequest", callback=self._receive_event)
with self.lock:
self.running = True
self.receiver.start()
log.debug("Container ready for container management requests")
def test_logging_clear(self):
""" initial log level for ion.process.event is INFO -- test that we can clear it
(root level WARN should apply)
"""
config.replace_configuration(os.path.join(TEST_DIR, 'logging.yml'))
log.debug('this should probably not be logged')
self.assertTrue(log.isEnabledFor(logging.INFO), msg=repr(log.__dict__))
#
handler = pyon.container.management.LogLevelHandler()
action = IonObject(OT.ChangeLogLevel, logger='pyon.container', level='WARN')
handler.handle_request(action)
self.assertFalse(log.isEnabledFor(logging.INFO))
def _perform_action(self, action):
handlers = self._get_handlers(action)
if not handlers:
log.info('action accepted but no handlers found: %s', action)
result = 'unhandled'
self.sender.publish_event(origin=self.container.id, action=action, outcome=str(result))
log.debug('received action: %s, outcome: %s', action, result)
else:
for handler in handlers:
try:
result = handler.handle_request(action) or "completed"
except Exception,e:
log.error("handler %r failed to perform action: %s", handler, action, exc_info=True)
result = e
self.sender.publish_event(origin=self.container.id, action=action, outcome=str(result))
log.debug('performed action: %s, outcome: %s', action, result)
def _perform_action(self, action):
handlers = self._get_handlers(action)
if not handlers:
log.info("action accepted but no handlers found: %s", action)
result = "unhandled"
self.sender.publish_event(origin=self.container.id, action=action, outcome=str(result))
log.debug("received action: %s, outcome: %s", action, result)
else:
for handler in handlers:
try:
result = handler.handle_request(action) or "completed"
except Exception, e:
log.error("handler %r failed to perform action: %s", handler, action, exc_info=True)
result = e
self.sender.publish_event(origin=self.container.id, action=action, outcome=str(result))
log.debug("performed action: %s, outcome: %s", action, result)
def _receive_event(self, event, headers):
with self.lock:
if not isinstance(event, ContainerManagementRequest):
log.trace("ignoring wrong type event: %r", event)
return
if not self.running:
log.warn("ignoring admin message received after shutdown: %s", event.action)
return
predicate = ContainerSelector.from_object(event.predicate)
if predicate.should_handle(self.container):
log.trace("handling admin message: %s", event.action)
self._perform_action(event.action)
else:
log.trace("ignoring admin action: %s", event.action)
if SEND_RESULT_IF_NOT_SELECTED:
self.sender.publish_event(origin=self.container.id, action=event.action, outcome="not selected")
log.debug("received action: %s, outcome: not selected", event.action)
def _receive_event(self, event, headers):
with self.lock:
if not isinstance(event, ContainerManagementRequest):
log.trace('ignoring wrong type event: %r', event)
return
if not self.running:
log.warn('ignoring admin message received after shutdown: %s', event.action)
return
predicate = ContainerSelector.from_object(event.predicate)
if predicate.should_handle(self.container):
log.trace('handling admin message: %s', event.action)
self._perform_action(event.action)
else:
log.trace('ignoring admin action: %s', event.action)
if SEND_RESULT_IF_NOT_SELECTED:
self.sender.publish_event(origin=self.container.id, action=event.action, outcome='not selected')
log.debug('received action: %s, outcome: not selected', event.action)
def zip_of_b64(b64_data, title):
"""
return a zipfile object based on a base64 input string.
"title" is a string to provide a helpful error message
function returns a (zipfile object, err_msg) tuple... zipfile object will be None on error
"""
log.debug("decoding base64 zipfile for %s", title)
try:
zip_str = base64.decodestring(b64_data)
except:
return None, ("could not base64 decode the supplied %s" % title)
log.debug("opening zipfile for %s", title)
try:
zip_file = StringIO(zip_str)
zip_obj = zipfile.ZipFile(zip_file)
except:
return None, ("could not parse zipfile contained in %s" % title)
return zip_obj, ""
def zip_of_b64(b64_data, title):
"""
return a zipfile object based on a base64 input string.
"title" is a string to provide a helpful error message
function returns a (zipfile object, err_msg) tuple... zipfile object will be None on error
"""
log.debug("decoding base64 zipfile for %s", title)
try:
zip_str = base64.decodestring(b64_data)
except:
return None, ("could not base64 decode the supplied %s" % title)
log.debug("opening zipfile for %s", title)
try:
zip_file = StringIO(zip_str)
zip_obj = zipfile.ZipFile(zip_file)
except:
return None, ("could not parse zipfile contained in %s" % title)
return zip_obj, ""
