def log_scope_call(scope, log_entry, include_stack=True, stack_first_frame=4):
try:
if not trace_data["config"].get("enabled", False):
return
log_entry["scope"] = scope
if not "ts" in log_entry:
log_entry["ts"] = get_ion_ts()
trace_data["scope_seq"][scope] += 1
log_entry["seq"] = trace_data["scope_seq"][scope]
if include_stack:
stack = inspect.stack()
frame_num = stack_first_frame
context = []
while len(stack) > frame_num and frame_num < 15:
exec_line = "%s:%s:%s" % (stack[frame_num][1], stack[frame_num][2], stack[frame_num][3])
context.insert(0, exec_line)
if exec_line.endswith("_control_flow") or exec_line.endswith("load_ion") or exec_line.endswith("spawn_process")\
or exec_line.endswith(":main") or exec_line.endswith(":dispatch_request"):
break
frame_num += 1
log_entry["stack"] = context
trace_data["trace_log"].append(log_entry)
if len(trace_data["trace_log"]) > trace_data["config"].get("max_entries", DEFAULT_CONFIG["max_entries"]) + 100:
trace_data["trace_log"] = trace_data["trace_log"][-trace_data["config"].get("max_entries", DEFAULT_CONFIG["max_entries"]):]
CallTracer.log_trace(log_entry)
except Exception as ex:
log.warn("Count not log trace call: %s", log_entry)
def get_obj_geometry(doc):
"""Extracts EWKT geometry given object dict
"""
geo_obj = None
geo_wkt = None
geo_type = None
if "boundary" in doc and isinstance(doc["boundary"], dict):
geo_obj = doc["boundary"]
if geo_obj and not geo_wkt:
if "boundary" in geo_obj and isinstance(geo_obj["boundary"],
basestring):
# Object type GeospatialArea
geo_wkt = geo_obj["boundary"]
if not geo_wkt:
present, coord_list = get_obj_geospatial_bounds(doc, calculate=False)
if present:
try:
coords = ", ".join("%s %s" % (x, y) for (x, y) in coord_list)
geo_wkt = "MULTIPOLYGON(((%s)))" % coords
except ValueError as ve:
log.warn("GeospatialBounds location values not parseable: %s",
ve)
if geo_wkt:
geo_type = geo_type or geo_wkt.split("(", 1)[0]
return True, geo_type, geo_wkt
return False, None, None
def get_obj_vertical_bounds(doc, calculate=True):
"""Extracts vertical bounds (min, max) from given object dict, by looking for an
attribute with GeospatialBounds type, or by computing from a geospatial point
"""
geo_bounds = None
if "geospatial_bounds" in doc:
geo_bounds = doc["geospatial_bounds"]
if "vertical_range" in doc:
geo_bounds = doc["vertical_range"]
if geo_bounds and isinstance(geo_bounds, dict):
if "geospatial_vertical_min" in geo_bounds and "geospatial_vertical_max" in geo_bounds:
try:
z1 = float(geo_bounds["geospatial_vertical_min"])
z2 = float(geo_bounds["geospatial_vertical_max"])
if not any((z1, z2)):
return False, (0, 0)
return True, (z1, z2)
except ValueError as ve:
log.warn(
"GeospatialBounds vertical values not parseable %s: %s",
geo_bounds, ve)
if calculate:
# Set bounds from center point
present, (lat, lon, elev) = get_obj_geospatial_point(doc, False)
if present:
return True, (elev, elev) # Point range
return False, (0, 0)
def get_obj_vertical_bounds(doc, calculate=True):
"""Extracts vertical bounds (min, max) from given object dict, by looking for an
attribute with GeospatialBounds type, or by computing from a geospatial point
"""
geo_bounds = None
if "geospatial_bounds" in doc:
geo_bounds = doc["geospatial_bounds"]
if "vertical_range" in doc:
geo_bounds = doc["vertical_range"]
if geo_bounds and isinstance(geo_bounds, dict):
if "geospatial_vertical_min" in geo_bounds and "geospatial_vertical_max" in geo_bounds:
try:
z1 = float(geo_bounds["geospatial_vertical_min"])
z2 = float(geo_bounds["geospatial_vertical_max"])
if not any((z1, z2)):
return False, (0, 0)
return True, (z1, z2)
except ValueError as ve:
log.warn(
"GeospatialBounds vertical values not parseable %s: %s",
geo_bounds, ve)
if calculate:
# Set bounds from center point
present, (lat, lon, elev) = get_obj_geospatial_point(doc, False)
if present:
return True, (elev, elev) # Point range
return False, (0, 0)
def get_obj_geospatial_bounds(doc, calculate=True, return_geo_bounds=False):
"""Extracts geospatial bounds (list of x,y coordinates) from given object dict, by looking for an
attribute with GeospatialBounds type, or by computing from a geospatial point
"""
geo_bounds = None
if "geospatial_bounds" in doc:
geo_bounds = doc["geospatial_bounds"]
if "bounding_box" in doc:
geo_bounds = doc["bounding_box"]
if geo_bounds and isinstance(geo_bounds, dict):
if "geospatial_longitude_limit_west" in geo_bounds and "geospatial_latitude_limit_south" in geo_bounds and \
"geospatial_longitude_limit_east" in geo_bounds and "geospatial_latitude_limit_north" in geo_bounds:
if return_geo_bounds:
return True, geo_bounds
try:
x1 = float(geo_bounds["geospatial_longitude_limit_west"])
y1 = float(geo_bounds["geospatial_latitude_limit_south"])
x2 = float(geo_bounds["geospatial_longitude_limit_east"])
y2 = float(geo_bounds["geospatial_latitude_limit_north"])
if not any((x1, x2, y1, y2)):
return False, None
return True, ((x1, y1), (x2, y1), (x2, y2), (x1, y2), (x1, y1))
except ValueError as ve:
log.warn("GeospatialBounds values not parseable %s: %s",
geo_bounds, ve)
if calculate:
# Set bounds from center point
present, (lat, lon, elev) = get_obj_geospatial_point(doc, False)
if present:
return True, ((lat, lon), (lat, lon), (lat, lon),
(lat, lon)) # Polygon with 4 point (otherwise error)
return False, None
def get_obj_geospatial_bounds(doc, calculate=True, return_geo_bounds=False):
"""Extracts geospatial bounds (list of x,y coordinates) from given object dict, by looking for an
attribute with GeospatialBounds type, or by computing from a geospatial point
"""
geo_bounds = None
if "geospatial_bounds" in doc:
geo_bounds = doc["geospatial_bounds"]
if "bounding_box" in doc:
geo_bounds = doc["bounding_box"]
if geo_bounds and isinstance(geo_bounds, dict):
if "geospatial_longitude_limit_west" in geo_bounds and "geospatial_latitude_limit_south" in geo_bounds and \
"geospatial_longitude_limit_east" in geo_bounds and "geospatial_latitude_limit_north" in geo_bounds:
if return_geo_bounds:
return True, geo_bounds
try:
x1 = float(geo_bounds["geospatial_longitude_limit_west"])
y1 = float(geo_bounds["geospatial_latitude_limit_south"])
x2 = float(geo_bounds["geospatial_longitude_limit_east"])
y2 = float(geo_bounds["geospatial_latitude_limit_north"])
if not any((x1, x2, y1, y2)):
return False, None
return True, ((x1, y1), (x2, y1), (x2, y2), (x1, y2), (x1, y1))
except ValueError as ve:
log.warn("GeospatialBounds values not parseable %s: %s",
geo_bounds, ve)
if calculate:
# Set bounds from center point
present, (lat, lon, elev) = get_obj_geospatial_point(doc, False)
if present:
return True, ((lat, lon), (lat, lon), (lat, lon), (lat, lon)
) # Polygon with 4 point (otherwise error)
return False, None
def get_obj_geometry(doc):
"""Extracts EWKT geometry given object dict
"""
geo_obj = None
geo_wkt = None
geo_type = None
if "boundary" in doc and isinstance(doc["boundary"], dict):
geo_obj = doc["boundary"]
if geo_obj and not geo_wkt:
if "boundary" in geo_obj and isinstance(geo_obj["boundary"],
basestring):
# Object type GeospatialArea
geo_wkt = geo_obj["boundary"]
if not geo_wkt:
present, coord_list = get_obj_geospatial_bounds(doc, calculate=False)
if present:
try:
coords = ", ".join("%s %s" % (x, y) for (x, y) in coord_list)
geo_wkt = "MULTIPOLYGON(((%s)))" % coords
except ValueError as ve:
log.warn("GeospatialBounds location values not parseable: %s",
ve)
if geo_wkt:
geo_type = geo_type or geo_wkt.split("(", 1)[0]
return True, geo_type, geo_wkt
return False, None, None
def handle_request(self, action):
try:
cs = ContainerSnapshot(bootstrap.container_instance)
if action.clear_all:
cs.clear_snapshots()
log.info(
"Container %s snapshot cleared (id=%s)"
% (bootstrap.container_instance.id, bootstrap.container_instance.proc_manager.cc_id)
)
return
cs.take_snapshot(
snapshot_id=action.snapshot_id,
snapshot_kwargs=action.snapshot_kwargs,
include_list=action.include_snapshots,
exclude_list=action.exclude_snapshots,
)
if action.persist_snapshot:
cs.persist_snapshot()
log.info(
"Container %s snapshot persisted (id=%s)"
% (bootstrap.container_instance.id, bootstrap.container_instance.proc_manager.cc_id)
)
else:
cs.log_snapshot()
except Exception as ex:
log.warn("Error taking container snapshot", exc_info=True)
def bootstrap_pyon(logging_config_override=None, pyon_cfg=None):
"""
This function initializes the core elements of the Pyon framework in a controlled way.
It does not initialize the ION container or the ION system.
@param logging_config_override A dict to initialize the Python logging subsystem (None loads default files)
@param pyon_cfg A DotDict with the fully loaded pyon configuration to merge into CFG (None loads default files)
"""
log.info("pyon.bootstrap (bootstrap_pyon) executing...")
# Make sure Pyon is only initialized only once
global pyon_initialized
if pyon_initialized:
log.warn("WARNING -- bootstrap_pyon() called again!")
return
# ENVIRONMENT. Check we are called in an expected environment (files, directories, etc)
assert_environment()
# LOGGING. Initialize logging from config
if not logutil.is_logging_configured():
logutil.configure_logging(
logutil.DEFAULT_LOGGING_PATHS,
logging_config_override=logging_config_override)
# YAML patch: OrderedDicts instead of dicts
from pyon.util.yaml_ordered_dict import apply_yaml_patch
apply_yaml_patch()
# CONFIG. Initialize pyon global configuration from local files
set_config(pyon_cfg)
log.debug("CFG set to %s", CFG)
# OBJECTS. Object and message definitions.
from pyon.core.registry import IonObjectRegistry
global _obj_registry
_obj_registry = IonObjectRegistry()
# SERVICES. Service definitions
# TODO: change the following to read service definitions from directory and import selectively
from pyon.ion.service import IonServiceRegistry
import interface.services
global _service_registry
_service_registry = IonServiceRegistry()
_service_registry.load_service_mods(interface.services)
_service_registry.build_service_map()
# RESOURCES. Load and initialize definitions
from pyon.ion import resource
resource.load_definitions()
# Fix a weird bug on Ubuntu that resets time.sleep to un-monkey patched version on import threading
from gevent import monkey
monkey.patch_time()
# Set initialized flag
pyon_initialized = True
log.debug("Pyon initialized OK")
def bootstrap_pyon(logging_config_override=None, pyon_cfg=None):
"""
This function initializes the core elements of the Pyon framework in a controlled way.
It does not initialize the ION container or the ION system.
@param logging_config_override A dict to initialize the Python logging subsystem (None loads default files)
@param pyon_cfg A DotDict with the fully loaded pyon configuration to merge into CFG (None loads default files)
"""
log.info("pyon.bootstrap (bootstrap_pyon) executing...")
# Make sure Pyon is only initialized only once
global pyon_initialized
if pyon_initialized:
log.warn("WARNING -- bootstrap_pyon() called again!")
return
# ENVIRONMENT. Check we are called in an expected environment (files, directories, etc)
assert_environment()
# LOGGING. Initialize logging from config
if not logutil.is_logging_configured():
logutil.configure_logging(logutil.DEFAULT_LOGGING_PATHS, logging_config_override=logging_config_override)
# YAML patch: OrderedDicts instead of dicts
from pyon.util.yaml_ordered_dict import apply_yaml_patch
apply_yaml_patch()
# CONFIG. Initialize pyon global configuration from local files
set_config(pyon_cfg)
log.debug("CFG set to %s", CFG)
# OBJECTS. Object and message definitions.
from pyon.core.registry import IonObjectRegistry
global _obj_registry
_obj_registry = IonObjectRegistry()
# SERVICES. Service definitions
# TODO: change the following to read service definitions from directory and import selectively
from pyon.ion.service import IonServiceRegistry
import interface.services
global _service_registry
_service_registry = IonServiceRegistry()
_service_registry.load_service_mods(interface.services)
_service_registry.build_service_map()
# RESOURCES. Load and initialize definitions
from pyon.ion import resource
resource.load_definitions()
# Fix a weird bug on Ubuntu that resets time.sleep to un-monkey patched version on import threading
from gevent import monkey; monkey.patch_time()
# Set initialized flag
pyon_initialized = True
log.debug("Pyon initialized OK")
def get_obj_temporal_bounds(doc):
"""Extracts a temporal bounds from given object dict, by looking for an attribute with TemporalBounds type"""
temp_range = None
if "temporal_range" in doc:
temp_range = doc["temporal_range"]
if temp_range and isinstance(temp_range, dict):
if "start_datetime" in temp_range and "end_datetime" in temp_range:
try:
t1 = float(temp_range["start_datetime"])
t2 = float(temp_range["end_datetime"])
if not any((t1, t2)):
return False, (0, 0)
return True, (t1, t2)
except ValueError as ve:
log.warn("TemporalBounds values not parseable %s: %s", temp_range, ve)
return False, (0, 0)
def _receive_event(self, event, headers):
with self.lock:
if not isinstance(event, ContainerManagementRequest):
log.trace('ignoring wrong type event: %r', event)
return
if not self.running:
log.warn('ignoring admin message received after shutdown: %s', event.action)
return
predicate = ContainerSelector.from_object(event.predicate)
if predicate.should_handle(self.container):
log.trace('handling admin message: %s', event.action)
self._perform_action(event.action)
else:
log.trace('ignoring admin action: %s', event.action)
if SEND_RESULT_IF_NOT_SELECTED:
self.sender.publish_event(origin=self.container.id, action=event.action, outcome='not selected')
log.debug('received action: %s, outcome: not selected', event.action)
def get_obj_temporal_bounds(doc):
"""Extracts a temporal bounds from given object dict, by looking for an attribute with TemporalBounds type"""
temp_range = None
if "temporal_range" in doc:
temp_range = doc["temporal_range"]
if temp_range and isinstance(temp_range, dict):
if "start_datetime" in temp_range and "end_datetime" in temp_range:
try:
t1 = float(temp_range["start_datetime"])
t2 = float(temp_range["end_datetime"])
if not any((t1, t2)):
return False, (0, 0)
return True, (t1, t2)
except ValueError as ve:
log.warn("TemporalBounds values not parseable %s: %s",
temp_range, ve)
return False, (0, 0)
def _receive_event(self, event, headers):
with self.lock:
if not isinstance(event, ContainerManagementRequest):
log.trace("ignoring wrong type event: %r", event)
return
if not self.running:
log.warn("ignoring admin message received after shutdown: %s", event.action)
return
predicate = ContainerSelector.from_object(event.predicate)
if predicate.should_handle(self.container):
log.trace("handling admin message: %s", event.action)
self._perform_action(event.action)
else:
log.trace("ignoring admin action: %s", event.action)
if SEND_RESULT_IF_NOT_SELECTED:
self.sender.publish_event(origin=self.container.id, action=event.action, outcome="not selected")
log.debug("received action: %s, outcome: not selected", event.action)
def handle_request(self, action):
try:
cs = ContainerSnapshot(bootstrap.container_instance)
if action.clear_all:
cs.clear_snapshots()
log.info("Container %s snapshot cleared (id=%s)" % (bootstrap.container_instance.id,
bootstrap.container_instance.proc_manager.cc_id))
return
cs.take_snapshot(snapshot_id=action.snapshot_id, snapshot_kwargs=action.snapshot_kwargs,
include_list=action.include_snapshots, exclude_list=action.exclude_snapshots)
if action.persist_snapshot:
cs.persist_snapshot()
log.info("Container %s snapshot persisted (id=%s)" % (bootstrap.container_instance.id,
bootstrap.container_instance.proc_manager.cc_id))
else:
cs.log_snapshot()
except Exception as ex:
log.warn("Error taking container snapshot", exc_info=True)
def log_scope_call(scope,
log_entry,
include_stack=True,
stack_first_frame=4):
try:
if not trace_data["config"].get("enabled", False):
return
log_entry["scope"] = scope
if not "ts" in log_entry:
log_entry["ts"] = get_ion_ts()
trace_data["scope_seq"][scope] += 1
log_entry["seq"] = trace_data["scope_seq"][scope]
if include_stack:
stack = inspect.stack()
frame_num = stack_first_frame
context = []
while len(stack) > frame_num and frame_num < 15:
exec_line = "%s:%s:%s" % (stack[frame_num][1],
stack[frame_num][2],
stack[frame_num][3])
context.insert(0, exec_line)
if exec_line.endswith("_control_flow") or exec_line.endswith("load_ion") or exec_line.endswith("spawn_process")\
or exec_line.endswith(":main") or exec_line.endswith(":dispatch_request"):
break
frame_num += 1
log_entry["stack"] = context
trace_data["trace_log"].append(log_entry)
if len(trace_data["trace_log"]) > trace_data["config"].get(
"max_entries", DEFAULT_CONFIG["max_entries"]) + 100:
trace_data["trace_log"] = trace_data["trace_log"][
-trace_data["config"].
get("max_entries", DEFAULT_CONFIG["max_entries"]):]
CallTracer.log_trace(log_entry)
except Exception as ex:
log.warn("Count not log trace call: %s", log_entry)