def _publish_event(self, event_msg, origin, event_type=None):
event_type = event_type or self.event_type or event_msg._get_type()
assert origin and event_type
topic = self._topic(event_type, origin, base_types=event_msg.base_types,
sub_type=event_msg.sub_type, origin_type=event_msg.origin_type)
to_name = (self._send_name.exchange, topic)
log.debug("Publishing event message to %s", to_name)
try:
ep = self.publish(event_msg, to_name=to_name)
ep.close()
except Exception as ex:
log.exception("Failed to publish event '%s'" % (event_msg))
return False
try:
# store published event but only if we specified an event_repo
if self.event_repo:
self.event_repo.put_event(event_msg)
except Exception as ex:
log.exception("Failed to store published event '%s'" % (event_msg))
return False
return True
def start_app(self, appdef=None, config=None):
"""
@brief Start an app from an app definition.
Note: apps can come in one of 2 variants:
1 processapp: In-line defined process to be started
2 regular app: Full app definition
"""
log.debug("AppManager.start_app(appdef=%s) ..." % appdef)
appdef = DotDict(appdef)
app_config = DictModifier(CFG)
if 'config' in appdef:
# Apply config from app file
app_file_cfg = DotDict(appdef.config)
app_config.update(app_file_cfg)
if config:
# Nest dict modifier and apply config from rel file
app_config = DictModifier(app_config, config)
if 'processapp' in appdef:
# Case 1: Appdef contains definition of process to start
name, module, cls = appdef.processapp
try:
pid = self.container.spawn_process(name, module, cls, app_config)
appdef._pid = pid
self.apps.append(appdef)
except Exception, ex:
log.exception("Appl %s start from processapp failed" % appdef.name)
def get_valid_principal_commitments(principal_id=None, consumer_id=None):
"""
Returns the list of valid commitments for the specified principal (org or actor.
If optional consumer_id (actor) is supplied, then filtered by consumer_id
"""
log.debug("Finding commitments for principal: %s", principal_id)
if principal_id is None:
return None
try:
gov_controller = bootstrap.container_instance.governance_controller
commitments, _ = gov_controller.rr.find_objects(principal_id, PRED.hasCommitment, RT.Commitment, id_only=False)
if not commitments:
return None
cur_time = get_ion_ts_millis()
commitment_list = [com for com in commitments if (consumer_id == None or com.consumer == consumer_id) and \
(int(com.expiration) == 0 or (int(com.expiration) > 0 and cur_time < int(com.expiration)))]
if commitment_list:
return commitment_list
except Exception:
log.exception("Could not determine actor resource commitments")
return None
def get_valid_resource_commitments(resource_id=None, actor_id=None):
"""
Returns the list of valid commitments for the specified resource.
If optional actor_id is supplied, then filtered by actor_id
"""
log.debug("Finding commitments for resource_id: %s and actor_id: %s", resource_id, actor_id)
if resource_id is None:
return None
try:
gov_controller = bootstrap.container_instance.governance_controller
commitments, _ = gov_controller.rr.find_subjects(RT.Commitment, PRED.hasTarget, resource_id, id_only=False)
if not commitments:
return None
cur_time = get_ion_ts_millis()
commitment_list = [com for com in commitments if (actor_id == None or com.consumer == actor_id) and \
(int(com.expiration) == 0 or (int(com.expiration) > 0 and cur_time < int(com.expiration)))]
if commitment_list:
return commitment_list
except Exception:
log.exception("Could not determine actor resource commitments")
return None
def listen(self, binding=None):
log.debug("LEF.listen: binding %s", binding)
binding = binding or self._binding or self.name[1]
self._ensure_node()
self._chan = self.node.channel(self.channel_type)
self._setup_listener(self.name, binding=binding)
self._chan.start_consume()
# notify any listeners of our readiness
self._ready_event.set()
while True:
log.debug("LEF: %s blocking, waiting for a message" % str(self.name))
try:
newchan = self._chan.accept()
msg, headers, delivery_tag = newchan.recv()
log.debug("LEF %s received message %s, headers %s, delivery_tag %s", self.name, msg, headers, delivery_tag)
except ChannelClosedError as ex:
log.debug('Channel was closed during LEF.listen')
break
try:
e = self.create_endpoint(existing_channel=newchan)
e._message_received(msg, headers)
except Exception:
log.exception("Unhandled error while handling received message")
raise
finally:
# ALWAYS ACK
newchan.ack(delivery_tag)
def build_service_map(self):
"""
Adds all known service definitions to service registry.
@todo: May be a bit fragile due to using BaseService.__subclasses__
"""
for cls in BaseService.__subclasses__():
assert hasattr(cls, 'name'), 'Service class must define name value. Service class in error: %s' % cls
if cls.name:
self.services_by_name[cls.name] = cls
self.add_servicedef_entry(cls.name, "base", cls)
try:
self.add_servicedef_entry(cls.name, "schema", json.loads(cls.SCHEMA_JSON))
except Exception as ex:
log.exception("Cannot parse service schema " + cls.name)
interfaces = list(implementedBy(cls))
if interfaces:
self.add_servicedef_entry(cls.name, "interface", interfaces[0])
if cls.__name__.startswith("Base"):
try:
client = "%s.%sProcessClient" % (cls.__module__, cls.__name__[4:])
self.add_servicedef_entry(cls.name, "client", named_any(client))
sclient = "%s.%sClient" % (cls.__module__, cls.__name__[4:])
self.add_servicedef_entry(cls.name, "simple_client", named_any(sclient))
except Exception, ex:
log.warning("Cannot find client for service %s" % (cls.name))
def _call_callbacks(self, group, action, cb_data):
cbs = self._stats_callbacks[group]
for cb in cbs:
try:
cb(group, action, cb_data)
except Exception:
log.exception("Error in stats callback")
def serve_forever(self):
""" Run the container until killed. """
log.debug("In Container.serve_forever")
if not self.proc_manager.proc_sup.running:
self.start()
# serve forever short-circuits if immediate is on and children len is ok
num_procs = len(self.proc_manager.proc_sup.children)
immediate = CFG.system.get('immediate', False)
if not (immediate and num_procs == 1): # only spawned greenlet is the CC-Agent
# print a warning just in case
if immediate and num_procs != 1:
log.warn("CFG.system.immediate=True but number of spawned processes is not 1 (%d)", num_procs)
try:
# This just waits in this Greenlet for all child processes to complete,
# which is triggered somewhere else.
self.proc_manager.proc_sup.join_children()
except (KeyboardInterrupt, SystemExit) as ex:
log.info('Received a kill signal, shutting down the container.')
if hasattr(self, 'gl_parent_watch') and self.gl_parent_watch is not None:
self.gl_parent_watch.kill()
except:
log.exception('Unhandled error! Forcing container shutdown')
else:
log.debug("Container.serve_forever short-circuiting due to CFG.system.immediate")
self.proc_manager.proc_sup.shutdown(CFG.cc.timeout.shutdown)
def execute_retrieve(self):
"""
execute_retrieve Executes a retrieval and returns the result
as a value in lieu of publishing it on a stream
"""
try:
coverage = DatasetManagementService._get_coverage(self.dataset_id, mode="r")
if coverage.num_timesteps == 0:
log.info("Reading from an empty coverage")
rdt = RecordDictionaryTool(param_dictionary=coverage.parameter_dictionary)
else:
rdt = self._coverage_to_granule(
coverage=coverage,
start_time=self.start_time,
end_time=self.end_time,
stride_time=self.stride_time,
parameters=self.parameters,
tdoa=self.tdoa,
)
except:
log.exception("Problems reading from the coverage")
raise BadRequest("Problems reading from the coverage")
finally:
coverage.close(timeout=5)
return rdt.to_granule()
def outgoing(self, invocation):
#log.trace("PolicyInterceptor.outgoing: %s", invocation.get_arg_value('process', invocation))
#Check for a field with the ResourceId decorator and if found, then set resource-id
# in the header with that field's value or if the decorator specifies a field within an object,
#then use the object's field value ( ie. _id)
try:
if isinstance(invocation.message, IonObjectBase):
decorator = 'ResourceId'
field = invocation.message.find_field_for_decorator(decorator)
if field is not None and hasattr(invocation.message,field):
deco_value = invocation.message.get_decorator_value(field, decorator)
if deco_value:
#Assume that if there is a value, then it is specifying a field in the object
fld_value = getattr(invocation.message,field)
if getattr(fld_value, deco_value) is not None:
invocation.headers['resource-id'] = getattr(fld_value, deco_value)
else:
if getattr(invocation.message,field) is not None:
invocation.headers['resource-id'] = getattr(invocation.message,field)
except Exception, ex:
log.exception(ex)
def serve_forever(self):
""" Run the container until killed. """
log.debug("In Container.serve_forever")
if not self.proc_manager.proc_sup.running:
self.start()
# Exit if immediate==True and children len is ok
num_procs = len(self.proc_manager.proc_sup.children)
immediate = CFG.system.get('immediate', False)
if immediate and num_procs == 1: # only spawned greenlet is the CC-Agent
log.debug("Container.serve_forever exiting due to CFG.system.immediate")
else:
# print a warning just in case
if immediate and num_procs != 1:
log.warn("CFG.system.immediate=True but number of spawned processes is not 1 (%d)", num_procs)
try:
# This just waits in this Greenlet for all child processes to complete,
# which is triggered somewhere else.
self.proc_manager.proc_sup.join_children()
except (KeyboardInterrupt, SystemExit) as ex:
if hasattr(self, 'gl_parent_watch') and self.gl_parent_watch is not None:
# Remove the greenlet that watches the parent process
self.gl_parent_watch.kill()
# Let the caller handle this
raise
except:
log.exception('Unhandled error! Forcing container shutdown')
def _run_consumer(self, ctag, queue_name, gqueue, callback):
cnt = 0
while True:
m = gqueue.get()
if isinstance(m, self.ConsumerClosedMessage):
break
exchange, routing_key, body, props = m
# create method frame
method_frame = DotDict()
method_frame['consumer_tag'] = ctag
method_frame['redelivered'] = False # @TODO
method_frame['exchange'] = exchange
method_frame['routing_key'] = routing_key
# create header frame
header_frame = DotDict()
header_frame['headers'] = props.copy()
# make delivery tag for ack/reject later
dtag = self._generate_dtag(ctag, cnt)
cnt += 1
with self._lock_unacked:
self._unacked[dtag] = (ctag, queue_name, m)
method_frame['delivery_tag'] = dtag
# deliver to callback
try:
callback(self, method_frame, header_frame, body)
except Exception:
log.exception("delivering to consumer, ignore!")
def outgoing(self, invocation):
payload = invocation.message
# Compliance: Make sure sent message objects support DotDict as arguments.
# Although DotDict is subclass of dict, msgpack does not like it
if isinstance(payload, IonMessageObjectBase):
for k, v in payload.__dict__.iteritems():
if isinstance(v, DotDict):
setattr(payload, k, v.as_dict())
# Msgpack the content to binary str - does nested IonObject encoding
try:
invocation.message = msgpack.packb(payload, default=encode_ion)
except Exception:
log.exception("Illegal type in IonObject attributes: %s", payload)
raise BadRequest("Illegal type in IonObject attributes")
# Make sure no Nones exist in headers - this indicates a problem somewhere up the stack.
# pika will choke hard on them as well, masking the actual problem, so we catch here.
nonelist = [(k, v) for k, v in invocation.headers.iteritems() if v is None]
if nonelist:
raise BadRequest("Invalid headers containing None values: %s" % str(nonelist))
msg_size = len(invocation.message)
if msg_size > self.max_message_size:
raise BadRequest('The message size %s is larger than the max_message_size value of %s' % (
msg_size, self.max_message_size))
return invocation
def handle_dds(self, coverage, dataset, fields):
cov = coverage
try:
time_name = coverage.temporal_parameter_name
time_context = coverage.get_parameter_context(time_name)
time_attrs = self.get_attrs(cov, time_name)
time_base = BaseType(time_name, type=self.dap_type(time_context), attributes=time_attrs, dimensions=(time_name,), shape=(coverage.num_timesteps,))
dataset[time_name] = time_base
except:
log.exception('Problem reading cov %s', str(cov))
raise # Can't do much without time
for var in fields:
while var:
name, slice_ = var.pop(0)
name = urllib.unquote(name)
if name == time_name:
continue # Already added to the dataset
try:
grid = GridType(name=name)
context = coverage.get_parameter_context(name)
attrs = self.get_attrs(cov, name)
grid[name] = BaseType(name=name, type=self.dap_type(context), attributes=attrs, dimensions=(time_name,), shape=(coverage.num_timesteps,))
grid[cov.temporal_parameter_name] = time_base
dataset[name] = grid
except Exception:
log.exception('Problem reading cov %s', str(cov))
continue
return dataset
def start_app(self, appdef=None, config=None):
"""
@brief Start an app from an app definition.
Note: apps can come in one of 2 variants:
1 processapp: In-line defined process to be started
2 regular app: Full app definition
"""
log.debug("AppManager.start_app(appdef=%s) ..." % appdef)
appdef = DotDict(appdef)
if 'config' in appdef:
app_cfg = appdef.config.copy()
if config:
dict_merge(app_cfg, config, inplace=True)
config = app_cfg
if 'processapp' in appdef:
# Case 1: Appdef contains definition of process to start
name, module, cls = appdef.processapp
try:
pid = self.container.spawn_process(name, module, cls, config)
appdef._pid = pid
self.apps.append(appdef)
except Exception, ex:
log.exception("Appl %s start from processapp failed" % appdef.name)
def get_version_info():
import pkg_resources
pkg_list = ["coi-services",
"pyon",
"coverage-model",
"ion-functions",
"eeagent",
"epu",
"utilities",
"marine-integrations"]
version = {}
for package in pkg_list:
try:
version["%s-release" % package] = pkg_resources.require(package)[0].version
# @TODO git versions for each?
except pkg_resources.DistributionNotFound:
pass
try:
dir_client = DirectoryServiceProcessClient(process=service_gateway_instance)
sys_attrs = dir_client.lookup("/System")
if sys_attrs and isinstance(sys_attrs, dict):
version.update({k: v for (k, v) in sys_attrs.iteritems() if "version" in k.lower()})
except Exception as ex:
log.exception("Could not determine system directory attributes")
return gateway_json_response(version)
def publish_heartbeat(self):
try:
hb_msg = self.get_heartbeat_message()
headers = dict(expiration=60000)
self.heartbeat_pub.publish(hb_msg, headers=headers)
except Exception:
log.exception("Error publishing heatbeat")
def stop(self, do_exit=True):
log.info("=============== Container stopping... ===============")
self._status = TERMINATING
if self.has_capability(CCAP.EVENT_PUBLISHER) and self.event_pub is not None:
try:
self.event_pub.publish_event(event_type="ContainerLifecycleEvent",
origin=self.id, origin_type="CapabilityContainer",
sub_type="TERMINATE",
state=ContainerStateEnum.TERMINATE)
except Exception as ex:
log.exception("Error sending event")
while self._capabilities:
capability = self._capabilities.pop()
#log.debug("stop(): Stopping '%s'" % capability)
try:
cap_obj = self._cap_instances[capability]
cap_obj.stop()
del self._cap_instances[capability]
except Exception as ex:
log.exception("Container stop(): Error stop %s" % capability)
Container.instance = None
from pyon.core import bootstrap
bootstrap.container_instance = None
self._is_started = False
self._status = TERMINATED
log.info("Container stopped (%s).", self.id)
if do_exit:
os.kill(os.getpid(), signal.SIGTERM)
def _read_interval_time(self):
"""
Reads the hsflowd conf file to determine what time should be used.
"""
if not (self._hsflowd_addr == "localhost" or self._hsflowd_addr == "127.0.0.1"):
log.debug("Skipping reading hsflow auto file, hsflowd is not running locally")
else:
try:
mtime = os.stat(self._hsflowd_conf).st_mtime
except OSError:
# if you can't stat it, you can't read it most likely
log.info("Could not stat hsflowd.auto file")
return
if mtime != self._conf_last_mod:
self._conf_last_mod = mtime
# appears to be simple key=value, one per line
try:
with open(self._hsflowd_conf) as f:
while True:
c = f.readline()
if c == "":
break
elif c.startswith("polling="):
self._counter_interval = int(c.rstrip().split("=")[1])
log.debug("New polling interval time: %d", self._counter_interval)
break
except IOError:
log.exception("Could not open/read hsflowd.auto")
def stop(self):
log.info("=============== Container stopping... ===============")
if self.event_pub is not None:
try:
self.event_pub.publish_event(event_type="ContainerLifecycleEvent",
origin=self.id, origin_type="CapabilityContainer",
sub_type="TERMINATE",
state=ContainerStateEnum.TERMINATE)
except Exception as ex:
log.exception(ex)
while self._capabilities:
capability = self._capabilities.pop()
log.debug("stop(): Stopping '%s'" % capability)
try:
self._stop_capability(capability)
except Exception as ex:
log.exception("Container stop(): Error stop %s" % capability)
Container.instance = None
from pyon.core import bootstrap
bootstrap.container_instance = None
self._is_started = False
log.debug("Container stopped, OK.")
def message_received(self, msg, headers):
assert self._routing_obj, "How did I get created without a routing object?"
log.debug("RPCResponseEndpointUnit.message_received\n\tmsg: %s\n\theaders: %s", msg, headers)
cmd_arg_obj = msg
cmd_op = headers.get('op', None)
# transform cmd_arg_obj into a dict
if hasattr(cmd_arg_obj, '__dict__'):
cmd_arg_obj = cmd_arg_obj.__dict__
elif isinstance(cmd_arg_obj, dict):
pass
else:
raise BadRequest("Unknown message type, cannot convert into kwarg dict: %s" % str(type(cmd_arg_obj)))
# op name must exist!
if not hasattr(self._routing_obj, cmd_op):
raise BadRequest("Unknown op name: %s" % cmd_op)
ro_meth = getattr(self._routing_obj, cmd_op)
result = None
response_headers = {}
try:
result = ro_meth(**cmd_arg_obj)
response_headers = { 'status_code': 200, 'error_message': '' }
except TypeError as ex:
log.exception("TypeError while attempting to call routing object's method")
response_headers = self._create_error_response(ServerError(ex.message))
return result, response_headers
def stop_app(self, appdef):
log.debug("App '%s' stopping" % appdef.name)
try:
if '_mod_loaded' in appdef:
appdef._mod_loaded.stop(self.container, appdef._state)
except Exception, ex:
log.exception("Application %s stop failed" % appdef.name)
def _unbind(self, exchange_point, exchange_name, binding_key):
xp = self.container.ex_manager.create_xp(exchange_point)
xn = self.container.ex_manager.create_xn_queue(exchange_name)
try:
xn.unbind(binding_key, xp)
except TransportError:
log.exception('Pubsub could not unbind %s from %s', binding_key, str(xp))
def stop(self):
log.debug("Container stopping...")
try:
self.app_manager.stop()
except Exception, ex:
log.exception("Container stop(): Error stop AppManager")
def publish_event_object(self, event_object):
"""
Publishes an event of given type for the given origin. Event_type defaults to an
event_type set when initializing the EventPublisher. Other kwargs fill out the fields
of the event. This operation will fail with an exception.
@param event_object the event object to be published
@retval event_object the event object which was published
"""
assert event_object
topic = self._topic(event_object)
to_name = (self._send_name.exchange, topic)
log.trace("Publishing event message to %s", to_name)
current_time = int(get_ion_ts())
#Ensure valid created timestamp if supplied
if event_object.ts_created:
if not is_valid_ts(event_object.ts_created):
raise BadRequest("The ts_created value is not a valid timestamp: '%s'" % (event_object.ts_created))
#Reject events that are older than specified time
if int(event_object.ts_created) > ( current_time + VALID_EVENT_TIME_PERIOD ):
raise BadRequest("This ts_created value is too far in the future:'%s'" % (event_object.ts_created))
#Reject events that are older than specified time
if int(event_object.ts_created) < (current_time - VALID_EVENT_TIME_PERIOD) :
raise BadRequest("This ts_created value is too old:'%s'" % (event_object.ts_created))
else:
event_object.ts_created = str(current_time)
#Validate this object
#TODO - enable this once the resource agent issue sending a dict is figured out
#event_object._validate()
#Ensure the event object has a unique id
if '_id' in event_object:
raise BadRequest("The event object cannot contain a _id field '%s'" % (event_object))
#Generate a unique ID for this event
event_object._id = create_unique_event_id()
try:
self.publish(event_object, to_name=to_name)
except Exception as ex:
log.exception("Failed to publish event (%s): '%s'" % (ex.message, event_object))
raise
try:
# store published event but only if we specified an event_repo
if PERSIST_ON_PUBLISH and self.event_repo:
self.event_repo.put_event(event_object)
except Exception as ex:
log.exception("Failed to store published event (%s): '%s'" % (ex.message, event_object))
raise
return event_object
def register_dap_dataset(self, dataset_id, data_product_name=''):
coverage_path = DatasetManagementService._get_coverage_path(dataset_id)
try:
self.add_dataset_to_xml(coverage_path=coverage_path, product_name=data_product_name)
self.create_symlink(coverage_path, self.pydap_data_path)
except: # We don't re-raise to prevent clients from bombing out...
log.exception('Problem registering dataset')
log.error('Failed to register dataset for coverage path %s' % coverage_path)
def call_process(self, packet, headers=None):
try:
self.process(packet)
except Exception as e:
log.exception('Unhandled caught in transform process')
event_publisher = EventPublisher()
event_publisher.publish_event(origin=self._transform_id, event_type='ExceptionEvent',
exception_type=str(type(e)), exception_message=e.message)
def register_safe(self, parent, key, **kwargs):
"""
Use this method to protect caller from any form of directory register error
"""
try:
return self.register(parent, key, **kwargs)
except Exception as ex:
log.exception("Error registering key=%s/%s, args=%s" % (parent, key, kwargs))
def delete_attachment(attachment_id):
try:
rr_client = ResourceRegistryServiceProcessClient(node=Container.instance.node, process=service_gateway_instance)
ret = rr_client.delete_attachment(attachment_id)
return gateway_json_response(ret)
except Exception, e:
log.exception("Error deleting attachment")
return build_error_response(e)
def stop(self):
log.debug("DatastoreManager.stop() [%d datastores]", len(self._datastores))
for x in self._datastores.itervalues():
try:
x.close()
except Exception as ex:
log.exception("Error closing datastore")
self._datastores = {}
def serve_forever(self):
""" Run the container until killed. """
log.debug("In Container.serve_forever")
if not self.proc_manager.proc_sup.running:
self.start()
# Exit if immediate==True and children len is ok
num_procs = len(self.proc_manager.proc_sup.children)
immediate = CFG.system.get('immediate', False)
if immediate and num_procs == 1: # only spawned greenlet is the CC-Agent
log.debug(
"Container.serve_forever exiting due to CFG.system.immediate")
else:
# print a warning just in case
if immediate and num_procs != 1:
log.warn(
"CFG.system.immediate=True but number of spawned processes is not 1 (%d)",
num_procs)
try:
# This just waits in this Greenlet for all child processes to complete,
# which is triggered somewhere else.
self.proc_manager.proc_sup.join_children()
except (KeyboardInterrupt, SystemExit) as ex:
if hasattr(self, 'gl_parent_watch'
) and self.gl_parent_watch is not None:
# Remove the greenlet that watches the parent process
self.gl_parent_watch.kill()
# Let the caller handle this
raise
except:
log.exception('Unhandled error! Forcing container shutdown')
def stop(self):
log.info("=============== Container stopping... ===============")
self.event_pub.publish_event(event_type="ContainerLifecycleEvent",
origin=self.id,
origin_type="CapabilityContainer",
sub_type="TERMINATE",
state=ContainerStateEnum.TERMINATE)
while self._capabilities:
capability = self._capabilities.pop()
log.debug("stop(): Stopping '%s'" % capability)
try:
self._stop_capability(capability)
except Exception as ex:
log.exception("Container stop(): Error stop %s" % capability)
Container.instance = None
from pyon.core import bootstrap
bootstrap.container_instance = None
self._is_started = False
log.debug("Container stopped, OK.")
def on_start(self):
try:
SimpleProcess.on_start(self)
self.pydap_host = self.CFG.get_safe(
'container.pydap_gateway.web_server.host', 'localhost')
self.pydap_port = self.CFG.get_safe(
'container.pydap_gateway.web_server.port', '8001')
self.pydap_data_path = self.CFG.get_safe('server.pydap.data_path',
'RESOURCE:ext/pydap')
self.pydap_data_path = FileSystem.get_extended_url(
self.pydap_data_path)
self.app = make_app(None, self.pydap_data_path,
'ion/core/static/templates/')
self.log = getLogger('pydap')
self.log.write = self.log.info
self.server = WSGIServer((self.pydap_host, int(self.pydap_port)),
self.app,
log=self.log)
self.server.start()
except:
log.exception('Unable to start PyDAP server')
raise
def execute_retrieve(self):
'''
execute_retrieve Executes a retrieval and returns the result
as a value in lieu of publishing it on a stream
'''
try:
coverage = DatasetManagementService._get_coverage(self.dataset_id,mode='r')
if coverage.is_empty():
log.info('Reading from an empty coverage')
rdt = RecordDictionaryTool(param_dictionary=coverage.parameter_dictionary)
else:
rdt = ReplayProcess._cov2granule(coverage=coverage,
start_time=self.start_time,
end_time=self.end_time,
stride_time=self.stride_time,
parameters=self.parameters,
stream_def_id=self.delivery_format,
tdoa=self.tdoa)
except:
log.exception('Problems reading from the coverage')
raise BadRequest('Problems reading from the coverage')
finally:
coverage.close(timeout=5)
return rdt.to_granule()
def get_event_computed_attributes(event,
include_event=False,
include_special=False,
include_formatted=False):
"""
@param event any Event to compute attributes for
@retval an EventComputedAttributes object for given event
"""
evt_computed = IonObject(OT.EventComputedAttributes)
evt_computed.event_id = event._id
evt_computed.ts_computed = get_ion_ts()
evt_computed.event = event if include_event else None
try:
summary = get_event_summary(event)
evt_computed.event_summary = summary
if include_special:
spc_attrs = [
"%s:%s" % (k, str(getattr(event, k))[:50])
for k in sorted(event.__dict__.keys()) if k not in [
'_id', '_rev', 'type_', 'origin', 'origin_type',
'ts_created', 'base_types'
]
]
evt_computed.special_attributes = ", ".join(spc_attrs)
if include_formatted:
evt_computed.event_attributes_formatted = pprint.pformat(
event.__dict__)
except Exception as ex:
log.exception(
"Error computing EventComputedAttributes for event %s: %s", event,
ex)
return evt_computed
def _publish_event(self, event_msg, origin, event_type=None):
event_type = event_type or self.event_type or event_msg._get_type()
assert origin and event_type
topic = self._topic(event_type, origin, base_types=event_msg.base_types,
sub_type=event_msg.sub_type, origin_type=event_msg.origin_type)
to_name = (self._send_name.exchange, topic)
log.trace("Publishing event message to %s", to_name)
try:
self.publish(event_msg, to_name=to_name)
except Exception as ex:
log.exception("Failed to publish event '%s'" % (event_msg))
return False
try:
# store published event but only if we specified an event_repo
if PERSIST_ON_PUBLISH and self.event_repo:
self.event_repo.put_event(event_msg)
except Exception as ex:
log.exception("Failed to store published event '%s'" % (event_msg))
return False
return True
def _load_actor(self, actor):
'''
Returns callable execute method if it exists, otherwise it raises a BadRequest
'''
try:
module = __import__(actor['module'], fromlist=[''])
except ImportError:
log.exception('Actor could not be loaded')
raise
try:
cls = getattr(module, actor['class'])
except AttributeError:
log.exception('Module %s does not have class %s', repr(module),
actor['class'])
raise
try:
execute = getattr(cls, 'execute')
except AttributeError:
log.exception('Actor class does not contain execute method')
raise
return execute
def validation(self, conditional, message, lineno):
if not conditional:
log.name = self.name
log.exception('[%s] %s: %s', lineno or '?',
self.exception.__name__, message)
raise self.exception(message)
def spawn_process(self,
name=None,
module=None,
cls=None,
config=None,
process_id=None):
"""
Spawn a process within the container. Processes can be of different type.
"""
if process_id and not is_valid_identifier(process_id, ws_sub='_'):
raise BadRequest("Given process_id %s is not a valid identifier" %
process_id)
# Generate a new process id if not provided
# TODO: Ensure it is system-wide unique
process_id = process_id or "%s.%s" % (self.container.id,
self.proc_id_pool.get_id())
log.debug(
"ProcManager.spawn_process(name=%s, module.cls=%s.%s) as pid=%s",
name, module, cls, process_id)
if not config:
# Use system CFG. It has the command line args in it
config = DictModifier(CFG)
else:
# Use provided config. Must be dict or DotDict
if not isinstance(config, DotDict):
config = DotDict(config)
config = DictModifier(CFG, config)
if self.container.spawn_args:
# Override config with spawn args
dict_merge(config, self.container.spawn_args, inplace=True)
#log.debug("spawn_process() pid=%s config=%s", process_id, config)
# PROCESS TYPE. Determines basic process context (messaging, service interface)
# One of: service, stream_process, agent, simple, immediate
service_cls = named_any("%s.%s" % (module, cls))
process_type = get_safe(config, "process.type") or getattr(
service_cls, "process_type", "service")
service_instance = None
try:
# spawn service by type
if process_type == "service":
service_instance = self._spawn_service_process(
process_id, name, module, cls, config)
elif process_type == "stream_process":
service_instance = self._spawn_stream_process(
process_id, name, module, cls, config)
elif process_type == "agent":
service_instance = self._spawn_agent_process(
process_id, name, module, cls, config)
elif process_type == "standalone":
service_instance = self._spawn_standalone_process(
process_id, name, module, cls, config)
elif process_type == "immediate":
service_instance = self._spawn_immediate_process(
process_id, name, module, cls, config)
elif process_type == "simple":
service_instance = self._spawn_simple_process(
process_id, name, module, cls, config)
else:
raise BadRequest("Unknown process type: %s" % process_type)
service_instance._proc_type = process_type
self._register_process(service_instance, name)
service_instance.errcause = "OK"
log.info("AppManager.spawn_process: %s.%s -> pid=%s OK", module,
cls, process_id)
if process_type == 'immediate':
log.info('Terminating immediate process: %s',
service_instance.id)
self.terminate_process(service_instance.id)
return service_instance.id
except Exception:
errcause = service_instance.errcause if service_instance else "instantiating service"
log.exception("Error spawning %s %s process (process_id: %s): %s",
name, process_type, process_id, errcause)
raise
def _control_flow(self):
"""
Entry point for process control thread of execution.
This method is run by the control greenlet for each ION process. Listeners attached
to the process, either RPC Servers or Subscribers, synchronize calls to the process
by placing call requests into the queue by calling _routing_call.
This method blocks until there are calls to be made in the synchronized queue, and
then calls from within this greenlet. Any exception raised is caught and re-raised
in the greenlet that originally scheduled the call. If successful, the AsyncResult
created at scheduling time is set with the result of the call.
"""
svc_name = getattr(
self.service, "name",
"unnamed-service") if self.service else "unnamed-service"
proc_id = getattr(self.service, "id",
"unknown-pid") if self.service else "unknown-pid"
if self.name:
threading.current_thread().name = "%s-%s" % (svc_name, self.name)
thread_base_name = threading.current_thread().name
self._ready_control.set()
for calltuple in self._ctrl_queue:
calling_gl, ar, call, callargs, callkwargs, context = calltuple
request_id = (context or {}).get("request-id", None)
if request_id:
threading.current_thread(
).name = thread_base_name + "-" + str(request_id)
#log.debug("control_flow making call: %s %s %s (has context: %s)", call, callargs, callkwargs, context is not None)
res = None
start_proc_time = get_ion_ts_millis()
self._record_proc_time(start_proc_time)
# check context for expiration
if context is not None and 'reply-by' in context:
if start_proc_time >= int(context['reply-by']):
log.info(
"control_flow: attempting to process message already exceeding reply-by, ignore"
)
# raise a timeout in the calling thread to allow endpoints to continue processing
e = IonTimeout(
"Reply-by time has already occurred (reply-by: %s, op start time: %s)"
% (context['reply-by'], start_proc_time))
calling_gl.kill(exception=e, block=False)
continue
# If ar is set, means it is cancelled
if ar.ready():
log.info(
"control_flow: attempting to process message that has been cancelled, ignore"
)
continue
init_db_stats()
try:
# ******************************************************************
# ****** THIS IS WHERE THE RPC OPERATION/SERVICE CALL IS MADE ******
with self.service.push_context(context), \
self.service.container.context.push_context(context):
self._ctrl_current = ar
res = call(*callargs, **callkwargs)
# ****** END CALL, EXCEPTION HANDLING FOLLOWS ******
# ******************************************************************
except OperationInterruptedException:
# endpoint layer takes care of response as it's the one that caused this
log.debug("Operation interrupted")
pass
except Exception as e:
if self._log_call_exception:
log.exception("PROCESS exception: %s" % e.message)
# Raise the exception in the calling greenlet.
# Try decorating the args of the exception with the true traceback -
# this should be reported by ThreadManager._child_failed
exc = PyonThreadTraceback(
"IonProcessThread _control_flow caught an exception "
"(call: %s, *args %s, **kwargs %s, context %s)\n"
"True traceback captured by IonProcessThread' _control_flow:\n\n%s"
% (call, callargs, callkwargs, context,
traceback.format_exc()))
e.args = e.args + (exc, )
if isinstance(e, (TypeError, IonException)):
# Pass through known process exceptions, in particular IonException
calling_gl.kill(exception=e, block=False)
else:
# Otherwise, wrap unknown, forward and hopefully we can continue on our way
self._errors.append(
(call, callargs, callkwargs, context, e, exc))
log.warn(exc)
log.warn("Attempting to continue...")
# Note: Too large exception string will crash the container (when passed on as msg header).
exception_str = str(exc)
if len(exception_str) > 10000:
exception_str = (
"Exception string representation too large. "
"Begin and end of the exception:\n" +
exception_str[:2000] + "\n...\n" +
exception_str[-2000:])
calling_gl.kill(exception=ContainerError(exception_str),
block=False)
finally:
try:
# Compute statistics
self._compute_proc_stats(start_proc_time)
db_stats = get_db_stats()
if db_stats:
if self._warn_call_dbstmt_threshold > 0 and db_stats.get(
"count.all",
0) >= self._warn_call_dbstmt_threshold:
stats_str = ", ".join(
"{}={}".format(k, db_stats[k])
for k in sorted(db_stats.keys()))
log.warn(
"PROC_OP '%s.%s' EXCEEDED DB THRESHOLD. stats=%s",
svc_name, call.__name__, stats_str)
elif self._log_call_dbstats:
stats_str = ", ".join(
"{}={}".format(k, db_stats[k])
for k in sorted(db_stats.keys()))
log.info("PROC_OP '%s.%s' DB STATS: %s", svc_name,
call.__name__, stats_str)
clear_db_stats()
if stats_callback:
stats_callback(proc_id=proc_id,
proc_name=self.name,
svc=svc_name,
op=call.__name__,
request_id=request_id,
context=context,
db_stats=db_stats,
proc_stats=self.time_stats,
result=res,
exc=None)
except Exception:
log.exception("Error computing process call stats")
self._ctrl_current = None
threading.current_thread().name = thread_base_name
# Set response in AsyncEvent of caller (endpoint greenlet)
ar.set(res)
def test_stacks(self):
try:
self.throw_caused()
except:
log.exception('ion caused by python')
except Exception, ex:
log.exception("Appl %s start from processapp failed" %
appdef.name)
else:
# Case 2: Appdef contains full app start params
modpath = appdef.mod
try:
mod = named_any(modpath)
appdef._mod_loaded = mod
# Start the app
supid, state = mod.start(self.container, START_PERMANENT,
appdef, config)
appdef._supid = supid
appdef._state = state
log.debug("App '%s' started. Root sup-id=%s" %
(appdef.name, supid))
self.apps.append(appdef)
except Exception, ex:
log.exception("Appl %s start from appdef failed" % appdef.name)
def stop_app(self, appdef):
log.debug("App '%s' stopping" % appdef.name)
try:
if '_mod_loaded' in appdef:
appdef._mod_loaded.stop(self.container, appdef._state)
except Exception, ex:
log.exception("Application %s stop failed" % appdef.name)
def spawn_process(self,
name=None,
module=None,
cls=None,
config=None,
process_id=None):
"""
Spawn a process within the container. Processes can be of different type.
"""
if process_id and not is_valid_identifier(process_id, ws_sub='_'):
raise BadRequest("Given process_id %s is not a valid identifier" %
process_id)
# Generate a new process id if not provided
# TODO: Ensure it is system-wide unique
process_id = process_id or "%s.%s" % (self.container.id,
self.proc_id_pool.get_id())
log.debug(
"ProcManager.spawn_process(name=%s, module.cls=%s.%s, config=%s) as pid=%s",
name, module, cls, config, process_id)
process_cfg = deepcopy(CFG)
if config:
# Use provided config. Must be dict or DotDict
if not isinstance(config, DotDict):
config = DotDict(config)
dict_merge(process_cfg, config, inplace=True)
if self.container.spawn_args:
# Override config with spawn args
dict_merge(process_cfg,
self.container.spawn_args,
inplace=True)
#log.debug("spawn_process() pid=%s process_cfg=%s", process_id, process_cfg)
# PROCESS TYPE. Determines basic process context (messaging, service interface)
# One of: service, stream_process, agent, simple, immediate
service_cls = named_any("%s.%s" % (module, cls))
process_type = get_safe(process_cfg, "process.type") or getattr(
service_cls, "process_type", "service")
process_start_mode = get_safe(config, "process.start_mode")
process_instance = None
# alert we have a spawning process, but we don't have the instance yet, so give the class instead (more accurate than name)
self._call_proc_state_changed("%s.%s" % (module, cls),
ProcessStateEnum.PENDING)
try:
# spawn service by type
if process_type == "service":
process_instance = self._spawn_service_process(
process_id, name, module, cls, process_cfg)
elif process_type == "stream_process":
process_instance = self._spawn_stream_process(
process_id, name, module, cls, process_cfg)
elif process_type == "agent":
process_instance = self._spawn_agent_process(
process_id, name, module, cls, process_cfg)
elif process_type == "standalone":
process_instance = self._spawn_standalone_process(
process_id, name, module, cls, process_cfg)
elif process_type == "immediate":
process_instance = self._spawn_immediate_process(
process_id, name, module, cls, process_cfg)
elif process_type == "simple":
process_instance = self._spawn_simple_process(
process_id, name, module, cls, process_cfg)
else:
raise BadRequest("Unknown process type: %s" % process_type)
process_instance._proc_type = process_type
self._register_process(process_instance, name)
process_instance.errcause = "OK"
log.info("ProcManager.spawn_process: %s.%s -> pid=%s OK", module,
cls, process_id)
if process_type == 'immediate':
log.info('Terminating immediate process: %s',
process_instance.id)
self.terminate_process(process_instance.id)
# terminate process also triggers TERMINATING/TERMINATED
self._call_proc_state_changed(process_instance,
ProcessStateEnum.EXITED)
else:
#Shouldn't be any policies for immediate processes
self.update_container_policies(process_instance)
return process_instance.id
except IonProcessError:
errcause = process_instance.errcause if process_instance else "instantiating process"
log.exception("Error spawning %s %s process (process_id: %s): %s",
name, process_type, process_id, errcause)
return None
except Exception:
errcause = process_instance.errcause if process_instance else "instantiating process"
log.exception("Error spawning %s %s process (process_id: %s): %s",
name, process_type, process_id, errcause)
# trigger failed notification - catches problems in init/start
self._call_proc_state_changed(process_instance,
ProcessStateEnum.FAILED)
raise
def register_safe(self, parent, key, **kwargs):
try:
return self.register(parent, key, **kwargs)
except Exception as ex:
log.exception("Error registering key=%s/%s, args=%s" %
(parent, key, kwargs))
def unregister_safe(self, parent, key):
try:
return self.unregister(parent, key)
except Exception as ex:
log.exception("Error unregistering key=%s/%s" % (parent, key))
def publish_event_object(self, event_object):
"""
Publishes an event of given type for the given origin. Event_type defaults to an
event_type set when initializing the EventPublisher. Other kwargs fill out the fields
of the event. This operation will fail with an exception.
@param event_object the event object to be published
@retval event_object the event object which was published
"""
assert event_object
topic = self._topic(event_object)
to_name = (self._send_name.exchange, topic)
log.trace("Publishing event message to %s", to_name)
current_time = int(get_ion_ts())
#Ensure valid created timestamp if supplied
if event_object.ts_created:
if not is_valid_ts(event_object.ts_created):
raise BadRequest(
"The ts_created value is not a valid timestamp: '%s'" %
(event_object.ts_created))
#Reject events that are older than specified time
if int(event_object.ts_created) > (current_time +
VALID_EVENT_TIME_PERIOD):
raise BadRequest(
"This ts_created value is too far in the future:'%s'" %
(event_object.ts_created))
#Reject events that are older than specified time
if int(event_object.ts_created) < (current_time -
VALID_EVENT_TIME_PERIOD):
raise BadRequest("This ts_created value is too old:'%s'" %
(event_object.ts_created))
else:
event_object.ts_created = str(current_time)
#Validate this object
#TODO - enable this once the resource agent issue sending a dict is figured out
#event_object._validate()
#Ensure the event object has a unique id
if '_id' in event_object:
raise BadRequest(
"The event object cannot contain a _id field '%s'" %
(event_object))
#Generate a unique ID for this event
event_object._id = create_unique_event_id()
try:
self.publish(event_object, to_name=to_name)
except Exception as ex:
log.exception("Failed to publish event (%s): '%s'" %
(ex.message, event_object))
raise
try:
# store published event but only if we specified an event_repo
if PERSIST_ON_PUBLISH and self.event_repo:
self.event_repo.put_event(event_object)
except Exception as ex:
log.exception("Failed to store published event (%s): '%s'" %
(ex.message, event_object))
raise
return event_object
def publish_event_object(self, event_object):
"""
Publishes an event of given type for the given origin. Event_type defaults to an
event_type set when initializing the EventPublisher. Other kwargs fill out the fields
of the event. This operation will fail with an exception.
@param event_object the event object to be published
@retval event_object the event object which was published
"""
if not event_object:
raise BadRequest("Must provide event_object")
event_object.base_types = event_object._get_extends()
topic = self._topic(
event_object
) # Routing key generated using type_, base_types, origin, origin_type, sub_type
container = (hasattr(self, '_process') and hasattr(
self._process, 'container') and self._process.container
) or BaseEndpoint._get_container_instance()
if container and container.has_capability(
container.CCAP.EXCHANGE_MANAGER):
# make sure we are an xp, if not, upgrade
if not isinstance(self._send_name, XOTransport):
default_nt = NameTrio(self.get_events_exchange_point())
if isinstance(self._send_name, NameTrio) \
and self._send_name.exchange == default_nt.exchange \
and self._send_name.queue == default_nt.queue \
and self._send_name.binding == default_nt.binding:
self._send_name = container.create_xp(self._events_xp)
else:
self._send_name = container.create_xp(self._send_name)
xp = self._send_name
to_name = xp.create_route(topic)
else:
to_name = (self._send_name.exchange, topic)
current_time = get_ion_ts_millis()
# Ensure valid created timestamp if supplied
if event_object.ts_created:
if not is_valid_ts(event_object.ts_created):
raise BadRequest(
"The ts_created value is not a valid timestamp: '%s'" %
(event_object.ts_created))
# Reject events that are older than specified time
if int(event_object.ts_created) > (current_time +
VALID_EVENT_TIME_PERIOD):
raise BadRequest(
"This ts_created value is too far in the future:'%s'" %
(event_object.ts_created))
# Reject events that are older than specified time
if int(event_object.ts_created) < (current_time -
VALID_EVENT_TIME_PERIOD):
raise BadRequest("This ts_created value is too old:'%s'" %
(event_object.ts_created))
else:
event_object.ts_created = str(current_time)
# Set the actor id based on
if not event_object.actor_id:
event_object.actor_id = self._get_actor_id()
#Validate this object - ideally the validator should pass on problems, but for now just log
#any errors and keep going, since seeing invalid situations are better than skipping validation.
try:
event_object._validate()
except Exception as e:
log.exception(e)
#Ensure the event object has a unique id
if '_id' in event_object:
raise BadRequest(
"The event object cannot contain a _id field '%s'" %
(event_object))
#Generate a unique ID for this event
event_object._id = create_unique_event_id()
try:
self.publish(event_object, to_name=to_name)
except Exception as ex:
log.exception("Failed to publish event (%s): '%s'" %
(ex.message, event_object))
raise
return event_object
def get_dataset(self, cov, fields, fill_index, dataset, response):
for var in fields:
while var:
name, slice_ = var.pop(0)
name = urllib.unquote(name)
slice_ = self.update_slice_object(slice_, fill_index)
if slice_ is None:
continue
#print name
#print "slice", slice_
pc = cov.get_parameter_context(name)
try:
param = cov.get_parameter(name)
data = np.array([])
time_data = np.array([])
if response == "dods":
data = self.get_data(cov, name, slice_)
time_data = self.get_time_data(cov, slice_)
time_attrs = self.get_attrs(cov, name)
attrs = self.get_attrs(cov, name)
dims = (cov.temporal_parameter_name,)
if isinstance(pc.param_type, QuantityType) and not param.is_coordinate and cov.temporal_parameter_name != name:
dataset[name] = self.make_grid(name, data, time_data, attrs, time_attrs, dims, data.dtype.char)
if isinstance(pc.param_type, ConstantType):
dataset[name] = self.make_grid(name, data, time_data, attrs, time_attrs, dims, data.dtype.char)
if isinstance(pc.param_type, ConstantRangeType):
#start = time.time()
#convert to string
try:
#scalar case
if data.shape == (2,):
data = np.atleast_1d('_'.join([str(data[0]), str(data[1])]))
else:
for i,d in enumerate(data):
f = [str(d[0]),str(d[1])]
data[i] = '_'.join(f)
except Exception, e:
data = np.asanyarray(['None' for d in data])
#print "range end", time.time() - start
dataset[name] = self.make_grid(name, data, time_data, attrs, time_attrs, dims, 'S')
if isinstance(pc.param_type,BooleanType):
data = np.asanyarray(data, dtype='int32')
dataset[name] = self.make_grid(name, data, time_data, attrs, time_attrs, dims, data.dtype.char)
if isinstance(pc.param_type,CategoryType):
#start = time.time()
dataset[name] = self.make_grid(name, data, time_data, attrs, time_attrs, dims, self.get_numpy_type(data))
#print "category end", time.time() - start
if isinstance(pc.param_type,ArrayType):
#start = time.time()
dataset[name] = self.make_grid(name, data, time_data, attrs, time_attrs, dims, self.get_numpy_type(data))
#print "array end", time.time() - start
if isinstance(pc.param_type,RecordType):
#start = time.time()
#convert to string
try:
for i,ddict in enumerate(data):
data[i] = str(ddict)
except Exception, e:
data = np.asanyarray(['None' for d in data])
#print "record end", time.time() - start
dataset[name] = self.make_grid(name, data, time_data, attrs, time_attrs, dims, 'S')
if param.is_coordinate and cov.temporal_parameter_name == name:
dataset[name] = BaseType(name=name, data=data, type=data.dtype.char, attributes=attrs, shape=data.shape)
except Exception, e:
log.exception('Problem reading cov %s %s', cov.name, e)
continue
def acquire_lock(self, key, timeout=LOCK_EXPIRES_DEFAULT, lock_holder=None, lock_info=None):
"""
Attempts to atomically acquire a lock with the given key and namespace.
If holder is given and holder already has the lock, renew.
Checks for expired locks.
@param timeout Secs until lock expiration or 0 for no expiration
@param lock_holder Str value identifying lock holder for subsequent exclusive access
@param lock_info Dict value for additional attributes describing lock
@retval bool - could lock be acquired?
"""
if not key:
raise BadRequest("Missing argument: key")
if "/" in key:
raise BadRequest("Invalid argument value: key")
lock_attrs = {LOCK_EXPIRES_ATTR: get_ion_ts_millis() + int(1000*timeout) if timeout else 0,
LOCK_HOLDER_ATTR: lock_holder or ""}
if lock_info:
lock_attrs.update(lock_info)
expires = int(lock_attrs[LOCK_EXPIRES_ATTR]) # Check type just to be sure
if expires and get_ion_ts_millis() > expires:
raise BadRequest("Invalid lock expiration value: %s", expires)
direntry = self._create_dir_entry(LOCK_DIR_PATH, key, attributes=lock_attrs)
lock_result = False
try:
# This is an atomic operation. It relies on the unique key constraint of the directory service
self.dir_store.create(direntry, create_unique_directory_id())
lock_result = True
except BadRequest as ex:
if ex.message.startswith("DirEntry already exists"):
de_old = self.lookup(LOCK_DIR_PATH, key, return_entry=True)
if de_old:
if self._is_lock_expired(de_old):
# Lock is expired: remove, try to relock
# Note: even as holder, it's safer to reacquire in this case than renew
log.warn("Removing expired lock: %s/%s", de_old.parent, de_old.key)
try:
# This is safe, because of lock was deleted + recreated in the meantime, it has different id
self._delete_lock(de_old)
# Try recreate - may fail again due to concurrency
self.dir_store.create(direntry, create_unique_directory_id())
lock_result = True
except BadRequest as ex:
if not ex.message.startswith("DirEntry already exists"):
log.exception("Error releasing/reacquiring expired lock %s", de_old.key)
except Exception:
log.exception("Error releasing/reacquiring expired lock %s", de_old.key)
elif lock_holder and de_old.attributes[LOCK_HOLDER_ATTR] == lock_holder:
# Holder currently holds the lock: renew
log.debug("Renewing lock %s/%s for holder %s", de_old.parent, de_old.key, lock_holder)
de_old.attributes = lock_attrs
try:
self.dir_store.update(de_old)
lock_result = True
except Exception:
log.exception("Error renewing expired lock %s", de_old.key)
# We do nothing if we could not find the lock now...
else:
raise
log.debug("Directory.acquire_lock(%s): %s -> %s", key, lock_attrs, lock_result)
return lock_result
dtype)
elif isinstance(pc.param_type, SparseConstantType):
data, dtype = self.filter_data(data)
seq[name] = self.make_series(response, name, data, attrs,
dtype)
#dataset[name] = self.make_series(response, name, data, attrs, dtype)
# elif param.is_coordinate and cov.temporal_parameter_name == name:
# dataset[name] = BaseType(name=name, data=data, type=data.dtype.char, attributes=attrs, shape=data.shape)
# else:
# log.error("Unhandled parameter for parameter (%s) type: %s", name, pc.param_type.__class__.__name__)
except Exception, e:
log.exception('Problem reading cov %s %s', cov.name,
e.__class__.__name__)
continue
dataset['data'] = seq
return dataset
def value_encoding_to_dap_type(self, value_encoding):
if value_encoding is None:
return 'S'
dt = np.dtype(value_encoding).char
if dt == 'O':
return 'S'
return dt
def dap_type(self, context):
if isinstance(
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except:
log.exception("Failed handling PyDAP request")
raise
class ProcManager(object):
def __init__(self, container):
self.container = container
# Define the callables that can be added to Container public API
self.container_api = [self.spawn_process, self.terminate_process]
# Add the public callables to Container
for call in self.container_api:
setattr(self.container, call.__name__, call)
self.proc_id_pool = IDPool()
# Temporary registry of running processes
self.procs_by_name = {}
self.procs = {}
# mapping of greenlets we spawn to process_instances for error handling
self._spawned_proc_to_process = {}
# The pyon worker process supervisor
self.proc_sup = IonProcessThreadManager(
heartbeat_secs=CFG.cc.timeout.heartbeat,
failure_notify_callback=self._spawned_proc_failed)
# list of callbacks for process state changes
self._proc_state_change_callbacks = []
def start(self):
log.debug("ProcManager starting ...")
self.proc_sup.start()
# Register container as resource object
cc_obj = CapabilityContainer(name=self.container.id,
cc_agent=self.container.name)
self.cc_id, _ = self.container.resource_registry.create(cc_obj)
#Create an association to an Org object if not the rot ION org and only if found
if CFG.container.org_name and CFG.container.org_name != CFG.system.root_org:
org, _ = self.container.resource_registry.find_resources(
restype=RT.Org, name=CFG.container.org_name, id_only=True)
if org:
self.container.resource_registry.create_association(
org[0], PRED.hasResource,
self.cc_id) # TODO - replace with proper association
log.debug("ProcManager started, OK.")
def stop(self):
log.debug("ProcManager stopping ...")
from pyon.datastore.couchdb.couchdb_datastore import CouchDB_DataStore
stats1 = CouchDB_DataStore._stats.get_stats()
# Call quit on procs to give them ability to clean up
# @TODO terminate_process is not gl-safe
# gls = map(lambda k: spawn(self.terminate_process, k), self.procs.keys())
# join(gls)
procs_list = sorted(self.procs.values(),
key=lambda proc: proc._proc_start_time,
reverse=True)
for proc in procs_list:
try:
self.terminate_process(proc.id)
except Exception as ex:
log.warn("Failed to terminate process (%s): %s", proc.id, ex)
# TODO: Have a choice of shutdown behaviors for waiting on children, timeouts, etc
self.proc_sup.shutdown(CFG.cc.timeout.shutdown)
if self.procs:
log.warn("ProcManager procs not empty: %s", self.procs)
if self.procs_by_name:
log.warn("ProcManager procs_by_name not empty: %s",
self.procs_by_name)
# Remove Resource registration
try:
self.container.resource_registry.delete(self.cc_id,
del_associations=True)
except NotFound:
# already gone, this is ok
pass
# TODO: Check associations to processes
stats2 = CouchDB_DataStore._stats.get_stats()
stats3 = CouchDB_DataStore._stats.diff_stats(stats2, stats1)
log.debug("Datastore stats difference during stop(): %s", stats3)
log.debug("ProcManager stopped, OK.")
def spawn_process(self,
name=None,
module=None,
cls=None,
config=None,
process_id=None):
"""
Spawn a process within the container. Processes can be of different type.
"""
if process_id and not is_valid_identifier(process_id, ws_sub='_'):
raise BadRequest("Given process_id %s is not a valid identifier" %
process_id)
# Generate a new process id if not provided
# TODO: Ensure it is system-wide unique
process_id = process_id or "%s.%s" % (self.container.id,
self.proc_id_pool.get_id())
log.debug(
"ProcManager.spawn_process(name=%s, module.cls=%s.%s, config=%s) as pid=%s",
name, module, cls, config, process_id)
process_cfg = deepcopy(CFG)
if config:
# Use provided config. Must be dict or DotDict
if not isinstance(config, DotDict):
config = DotDict(config)
dict_merge(process_cfg, config, inplace=True)
if self.container.spawn_args:
# Override config with spawn args
dict_merge(process_cfg,
self.container.spawn_args,
inplace=True)
#log.debug("spawn_process() pid=%s process_cfg=%s", process_id, process_cfg)
# PROCESS TYPE. Determines basic process context (messaging, service interface)
# One of: service, stream_process, agent, simple, immediate
service_cls = named_any("%s.%s" % (module, cls))
process_type = get_safe(process_cfg, "process.type") or getattr(
service_cls, "process_type", "service")
process_start_mode = get_safe(config, "process.start_mode")
process_instance = None
# alert we have a spawning process, but we don't have the instance yet, so give the class instead (more accurate than name)
self._call_proc_state_changed("%s.%s" % (module, cls),
ProcessStateEnum.PENDING)
try:
# spawn service by type
if process_type == "service":
process_instance = self._spawn_service_process(
process_id, name, module, cls, process_cfg)
elif process_type == "stream_process":
process_instance = self._spawn_stream_process(
process_id, name, module, cls, process_cfg)
elif process_type == "agent":
process_instance = self._spawn_agent_process(
process_id, name, module, cls, process_cfg)
elif process_type == "standalone":
process_instance = self._spawn_standalone_process(
process_id, name, module, cls, process_cfg)
elif process_type == "immediate":
process_instance = self._spawn_immediate_process(
process_id, name, module, cls, process_cfg)
elif process_type == "simple":
process_instance = self._spawn_simple_process(
process_id, name, module, cls, process_cfg)
else:
raise BadRequest("Unknown process type: %s" % process_type)
process_instance._proc_type = process_type
self._register_process(process_instance, name)
process_instance.errcause = "OK"
log.info("ProcManager.spawn_process: %s.%s -> pid=%s OK", module,
cls, process_id)
if process_type == 'immediate':
log.info('Terminating immediate process: %s',
process_instance.id)
self.terminate_process(process_instance.id)
# terminate process also triggers TERMINATING/TERMINATED
self._call_proc_state_changed(process_instance,
ProcessStateEnum.EXITED)
else:
#Shouldn't be any policies for immediate processes
self.update_container_policies(process_instance)
return process_instance.id
except IonProcessError:
errcause = process_instance.errcause if process_instance else "instantiating process"
log.exception("Error spawning %s %s process (process_id: %s): %s",
name, process_type, process_id, errcause)
return None
except Exception:
errcause = process_instance.errcause if process_instance else "instantiating process"
log.exception("Error spawning %s %s process (process_id: %s): %s",
name, process_type, process_id, errcause)
# trigger failed notification - catches problems in init/start
self._call_proc_state_changed(process_instance,
ProcessStateEnum.FAILED)
raise
#This must be called after registering the process
def update_container_policies(self, process_instance):
if not self.container.governance_controller:
return
if process_instance._proc_type == "service":
# look to load any existing policies for this service
self.container.governance_controller.safe_update_service_access_policy(
process_instance._proc_listen_name)
if process_instance._proc_type == "agent":
# look to load any existing policies for this agent service
if process_instance.resource_type is None:
self.container.governance_controller.safe_update_service_access_policy(
process_instance.name)
else:
self.container.governance_controller.safe_update_service_access_policy(
process_instance.resource_type)
if process_instance.resource_id:
# look to load any existing policies for this resource
self.container.governance_controller.safe_update_resource_access_policy(
process_instance.resource_id)
def list_local_processes(self, process_type=''):
"""
Returns a list of the running ION processes in the container or filtered by the process_type
"""
if not process_type:
return self.procs.values()
return [
p for p in self.procs.itervalues()
if p.process_type == process_type
]
def get_a_local_process(self, proc_name=''):
"""
Returns a running ION processes in the container for the specified name
"""
for p in self.procs.itervalues():
if p.name == proc_name:
return p
if p.process_type == 'agent' and p.resource_type == proc_name:
return p
return None
def is_local_service_process(self, service_name):
local_services = self.list_local_processes('service')
for p in local_services:
if p.name == service_name:
return True
return False
def is_local_agent_process(self, resource_type):
local_agents = self.list_local_processes('agent')
for p in local_agents:
if p.resource_type == resource_type:
return True
return False
def _spawned_proc_failed(self, gproc):
log.error("ProcManager._spawned_proc_failed: %s, %s", gproc,
gproc.exception)
# for now - don't worry about the mapping, if we get a failure, just kill the container.
# leave the mapping in place for potential expansion later.
# # look it up in mapping
# if not gproc in self._spawned_proc_to_process:
# log.warn("No record of gproc %s in our map (%s)", gproc, self._spawned_proc_to_process)
# return
#
prc = self._spawned_proc_to_process.get(gproc, None)
#
# # make sure prc is in our list
# if not prc in self.procs.values():
# log.warn("prc %s not found in procs list", prc)
# return
# stop the rest of the process
if prc is not None:
try:
self.terminate_process(prc.id, False)
except Exception as e:
log.warn(
"Problem while stopping rest of failed process %s: %s",
prc, e)
finally:
self._call_proc_state_changed(prc, ProcessStateEnum.FAILED)
else:
log.warn("No ION process found for failed proc manager child: %s",
gproc)
#self.container.fail_fast("Container process (%s) failed: %s" % (svc, gproc.exception))
def _cleanup_method(self, queue_name, ep=None):
"""
Common method to be passed to each spawned ION process to clean up their process-queue.
@TODO Leaks implementation detail, should be using XOs
"""
if ep._chan is not None and not ep._chan._queue_auto_delete:
# only need to delete if AMQP didn't handle it for us already!
# @TODO this will not work with XOs (future)
try:
ch = self.container.node.channel(RecvChannel)
ch._recv_name = NameTrio(
get_sys_name(), "%s.%s" % (get_sys_name(), queue_name))
ch._destroy_queue()
except TransportError as ex:
log.warn("Cleanup method triggered an error, ignoring: %s", ex)
def add_proc_state_changed_callback(self, cb):
"""
Adds a callback to be called when a process' state changes.
The callback should take three parameters: The process, the state, and the container.
"""
self._proc_state_change_callbacks.append(cb)
def remove_proc_state_changed_callback(self, cb):
"""
Removes a callback from the process state change callback list.
If the callback is not registered, this method does nothing.
"""
if cb in self._proc_state_change_callbacks:
self._proc_state_change_callbacks.remove(cb)
def _call_proc_state_changed(self, svc, state):
"""
Internal method to call all registered process state change callbacks.
"""
log.debug("Proc State Changed (%s): %s",
ProcessStateEnum._str_map.get(state, state), svc)
for cb in self._proc_state_change_callbacks:
cb(svc, state, self.container)
def _create_listening_endpoint(self, **kwargs):
"""
Creates a listening endpoint for spawning processes.
This method exists to be able to override the type created via configuration.
In most cases it will create a ConversationRPCServer.
"""
eptypestr = CFG.get_safe(
'container.messaging.endpoint.proc_listening_type', None)
if eptypestr is not None:
module, cls = eptypestr.rsplit('.', 1)
mod = __import__(module, fromlist=[cls])
eptype = getattr(mod, cls)
ep = eptype(**kwargs)
else:
conv_enabled = CFG.get_safe(
'container.messaging.endpoint.rpc_conversation_enabled', False)
if conv_enabled:
ep = ConversationRPCServer(**kwargs)
else:
ep = ProcessRPCServer(**kwargs)
return ep
# -----------------------------------------------------------------
# PROCESS TYPE: service
def _spawn_service_process(self, process_id, name, module, cls, config):
"""
Spawn a process acting as a service worker.
Attach to service queue with service definition, attach to service pid
"""
process_instance = self._create_process_instance(
process_id, name, module, cls, config)
listen_name = get_safe(config,
"process.listen_name") or process_instance.name
log.debug("Service Process (%s) listen_name: %s", name, listen_name)
process_instance._proc_listen_name = listen_name
# Service RPC endpoint
rsvc1 = self._create_listening_endpoint(node=self.container.node,
from_name=listen_name,
service=process_instance,
process=process_instance)
# Named local RPC endpoint
rsvc2 = self._create_listening_endpoint(node=self.container.node,
from_name=process_instance.id,
service=process_instance,
process=process_instance)
# cleanup method to delete process queue
cleanup = lambda _: self._cleanup_method(process_instance.id, rsvc2)
# Start an ION process with the right kind of endpoint factory
proc = self.proc_sup.spawn(name=process_instance.id,
service=process_instance,
listeners=[rsvc1, rsvc2],
proc_name=process_instance._proc_name,
cleanup_method=cleanup)
self.proc_sup.ensure_ready(
proc, "_spawn_service_process for %s" % ",".join(
(listen_name, process_instance.id)))
# map gproc to process_instance
self._spawned_proc_to_process[proc.proc] = process_instance
# set service's reference to process
process_instance._process = proc
self._process_init(process_instance)
self._process_start(process_instance)
try:
proc.start_listeners()
except IonProcessError:
self._process_quit(process_instance)
self._call_proc_state_changed(process_instance,
ProcessStateEnum.FAILED)
raise
return process_instance
# -----------------------------------------------------------------
# PROCESS TYPE: stream process
def _spawn_stream_process(self, process_id, name, module, cls, config):
"""
Spawn a process acting as a data stream process.
Attach to subscription queue with process function.
"""
process_instance = self._create_process_instance(
process_id, name, module, cls, config)
listen_name = get_safe(config, "process.listen_name") or name
log.debug("Stream Process (%s) listen_name: %s", name, listen_name)
process_instance._proc_listen_name = listen_name
process_instance.stream_subscriber = StreamSubscriber(
process=process_instance,
exchange_name=listen_name,
callback=process_instance.call_process)
# Add publishers if any...
publish_streams = get_safe(config, "process.publish_streams")
pub_names = self._set_publisher_endpoints(process_instance,
publish_streams)
rsvc = self._create_listening_endpoint(node=self.container.node,
from_name=process_instance.id,
service=process_instance,
process=process_instance)
# cleanup method to delete process queue (@TODO: leaks a bit here - should use XOs)
def cleanup(*args):
self._cleanup_method(process_instance.id, rsvc)
for name in pub_names:
p = getattr(process_instance, name)
p.close()
proc = self.proc_sup.spawn(
name=process_instance.id,
service=process_instance,
listeners=[rsvc, process_instance.stream_subscriber],
proc_name=process_instance._proc_name,
cleanup_method=cleanup)
self.proc_sup.ensure_ready(
proc, "_spawn_stream_process for %s" % process_instance._proc_name)
# map gproc to process_instance
self._spawned_proc_to_process[proc.proc] = process_instance
# set service's reference to process
process_instance._process = proc
self._process_init(process_instance)
self._process_start(process_instance)
try:
proc.start_listeners()
except IonProcessError:
self._process_quit(process_instance)
self._call_proc_state_changed(process_instance,
ProcessStateEnum.FAILED)
raise
return process_instance
# -----------------------------------------------------------------
# PROCESS TYPE: agent
def _spawn_agent_process(self, process_id, name, module, cls, config):
"""
Spawn a process acting as agent process.
Attach to service pid.
"""
process_instance = self._create_process_instance(
process_id, name, module, cls, config)
if not isinstance(process_instance, ResourceAgent) and not isinstance(
process_instance, SimpleResourceAgent):
raise ContainerConfigError(
"Agent process must extend ResourceAgent")
listeners = []
# Set the resource ID if we get it through the config
resource_id = get_safe(process_instance.CFG, "agent.resource_id")
if resource_id:
process_instance.resource_id = resource_id
alistener = self._create_listening_endpoint(
node=self.container.node,
from_name=resource_id,
service=process_instance,
process=process_instance)
listeners.append(alistener)
rsvc = self._create_listening_endpoint(node=self.container.node,
from_name=process_instance.id,
service=process_instance,
process=process_instance)
listeners.append(rsvc)
# cleanup method to delete process/agent queue (@TODO: leaks a bit here - should use XOs)
def agent_cleanup(x):
self._cleanup_method(process_instance.id, rsvc)
if resource_id:
pass
#self._cleanup_method(resource_id, alistener)
# disabled, it's probably not architecturally correct to delete this queue
proc = self.proc_sup.spawn(name=process_instance.id,
service=process_instance,
listeners=listeners,
proc_name=process_instance._proc_name,
cleanup_method=agent_cleanup)
self.proc_sup.ensure_ready(
proc, "_spawn_agent_process for %s" % process_instance.id)
# map gproc to process_instance
self._spawned_proc_to_process[proc.proc] = process_instance
# set service's reference to process
process_instance._process = proc
# Now call the on_init of the agent.
self._process_init(process_instance)
if not process_instance.resource_id:
log.warn("New agent pid=%s has no resource_id set" % process_id)
self._process_start(process_instance)
try:
proc.start_listeners()
except IonProcessError:
self._process_quit(process_instance)
self._call_proc_state_changed(process_instance,
ProcessStateEnum.FAILED)
raise
if not process_instance.resource_id:
log.warn("Agent process id=%s does not define resource_id!!" %
process_instance.id)
return process_instance
# -----------------------------------------------------------------
# PROCESS TYPE: standalone
def _spawn_standalone_process(self, process_id, name, module, cls, config):
"""
Spawn a process acting as standalone process.
Attach to service pid.
"""
process_instance = self._create_process_instance(
process_id, name, module, cls, config)
rsvc = self._create_listening_endpoint(node=self.container.node,
from_name=process_instance.id,
service=process_instance,
process=process_instance)
# Add publishers if any...
publish_streams = get_safe(config, "process.publish_streams")
pub_names = self._set_publisher_endpoints(process_instance,
publish_streams)
# cleanup method to delete process queue (@TODO: leaks a bit here - should use XOs)
def cleanup(*args):
self._cleanup_method(process_instance.id, rsvc)
for name in pub_names:
p = getattr(process_instance, name)
p.close()
proc = self.proc_sup.spawn(name=process_instance.id,
service=process_instance,
listeners=[rsvc],
proc_name=process_instance._proc_name,
cleanup_method=cleanup)
self.proc_sup.ensure_ready(
proc, "_spawn_standalone_process for %s" % process_instance.id)
# map gproc to process_instance
self._spawned_proc_to_process[proc.proc] = process_instance
# set service's reference to process
process_instance._process = proc
self._process_init(process_instance)
self._process_start(process_instance)
try:
proc.start_listeners()
except IonProcessError:
self._process_quit(process_instance)
self._call_proc_state_changed(process_instance,
ProcessStateEnum.FAILED)
raise
return process_instance
# -----------------------------------------------------------------
# PROCESS TYPE: simple
def _spawn_simple_process(self, process_id, name, module, cls, config):
"""
Spawn a process acting as simple process.
No attachments.
"""
process_instance = self._create_process_instance(
process_id, name, module, cls, config)
# Add publishers if any...
publish_streams = get_safe(config, "process.publish_streams")
pub_names = self._set_publisher_endpoints(process_instance,
publish_streams)
# cleanup method to delete process queue (@TODO: leaks a bit here - should use XOs)
def cleanup(*args):
for name in pub_names:
p = getattr(process_instance, name)
p.close()
proc = self.proc_sup.spawn(name=process_instance.id,
service=process_instance,
listeners=[],
proc_name=process_instance._proc_name,
cleanup_method=cleanup)
self.proc_sup.ensure_ready(
proc, "_spawn_simple_process for %s" % process_instance.id)
self._process_init(process_instance)
self._process_start(process_instance)
return process_instance
# -----------------------------------------------------------------
# PROCESS TYPE: immediate
def _spawn_immediate_process(self, process_id, name, module, cls, config):
"""
Spawn a process acting as immediate one off process.
No attachments.
"""
process_instance = self._create_process_instance(
process_id, name, module, cls, config)
self._process_init(process_instance)
self._process_start(process_instance)
return process_instance
def _create_process_instance(self, process_id, name, module, cls, config):
"""
Creates an instance of a "service", be it a Service, Agent, Stream, etc.
@rtype BaseService
@return An instance of a "service"
"""
# SERVICE INSTANCE.
process_instance = for_name(module, cls)
if not isinstance(process_instance, BaseService):
raise ContainerConfigError(
"Instantiated service not a BaseService %r" % process_instance)
# Prepare service instance
process_instance.errcause = ""
process_instance.id = process_id
process_instance.container = self.container
process_instance.CFG = config
process_instance._proc_name = name
process_instance._proc_start_time = time.time()
#Unless the process has been started as part of another Org, default to the container Org or the ION Org
if config.has_key('org_name'):
process_instance.org_name = config['org_name']
else:
process_instance.org_name = CFG.get_safe(
'container.org_name', CFG.get_safe('system.root_org', 'ION'))
# Add stateful process operations
if hasattr(process_instance, "_flush_state"):
def _flush_state():
if not hasattr(process_instance, "_proc_state"):
process_instance._proc_state = {}
process_instance._proc_state_changed = False
return
process_instance.container.state_repository.put_state(
process_instance.id, process_instance._proc_state)
process_instance._proc_state_changed = False
def _load_state():
if not hasattr(process_instance, "_proc_state"):
process_instance._proc_state = {}
try:
new_state = process_instance.container.state_repository.get_state(
process_instance.id)
process_instance._proc_state.clear()
process_instance._proc_state.update(new_state)
process_instance._proc_state_changed = False
except Exception as ex:
log.warn("Process %s load state failed: %s",
process_instance.id, str(ex))
process_instance._flush_state = _flush_state
process_instance._load_state = _load_state
process_start_mode = get_safe(config, "process.start_mode")
if process_start_mode == "RESTART":
if hasattr(process_instance, "_load_state"):
process_instance._load_state()
# start service dependencies (RPC clients)
self._start_process_dependencies(process_instance)
return process_instance
def _start_process_dependencies(self, process_instance):
process_instance.errcause = "setting service dependencies"
log.debug("spawn_process dependencies: %s",
process_instance.dependencies)
# TODO: Service dependency != process dependency
for dependency in process_instance.dependencies:
client = getattr(process_instance.clients, dependency)
assert client, "Client for dependency not found: %s" % dependency
# @TODO: should be in a start_client in RPCClient chain
client.process = process_instance
client.node = self.container.node
# ensure that dep actually exists and is running
# MM: commented out - during startup (init actually), we don't need to check for service dependencies
# MM: TODO: split on_init from on_start; start consumer in on_start; check for full queues on restart
# if process_instance.name != 'bootstrap' or (process_instance.name == 'bootstrap' and process_instance.CFG.level == dependency):
# svc_de = self.container.resource_registry.find_resources(restype="Service", name=dependency, id_only=True)
# if not svc_de:
# raise ContainerConfigError("Dependency for service %s not running: %s" % (process_instance.name, dependency))
def _process_init(self, process_instance):
# Init process
process_instance.errcause = "initializing service"
process_instance.init()
def _process_start(self, process_instance):
# Start process
# THIS SHOULD BE CALLED LATER THAN SPAWN
# TODO: Check for timeout
process_instance.errcause = "starting service"
process_instance.start()
def _process_quit(self, process_instance):
"""
Common method to handle process stopping.
"""
process_instance.errcause = "quitting process"
# Give the process notice to quit doing stuff.
process_instance.quit()
# Terminate IonProcessThread (may not have one, i.e. simple process)
# @TODO: move this into process' on_quit()
if getattr(process_instance, '_process',
None) is not None and process_instance._process:
process_instance._process.notify_stop()
process_instance._process.stop()
def _set_publisher_endpoints(self,
process_instance,
publisher_streams=None):
publisher_streams = publisher_streams or {}
names = []
for name, stream_id in publisher_streams.iteritems():
# problem is here
pub = StreamPublisher(process=process_instance,
stream_id=stream_id)
setattr(process_instance, name, pub)
names.append(name)
return names
def _register_process(self, process_instance, name):
"""
Performs all actions related to registering the new process in the system.
Also performs process type specific registration, such as for services and agents
"""
# Add process instance to container's process dict
if name in self.procs_by_name:
log.warn("Process name already registered in container: %s" % name)
self.procs_by_name[name] = process_instance
self.procs[process_instance.id] = process_instance
# Add Process to resource registry
# Note: In general the Process resource should be created by the CEI PD, but not all processes are CEI
# processes. How to deal with this?
process_instance.errcause = "registering"
if process_instance._proc_type != "immediate":
proc_obj = Process(name=process_instance.id,
label=name,
proctype=process_instance._proc_type)
proc_id, _ = self.container.resource_registry.create(proc_obj)
process_instance._proc_res_id = proc_id
# Associate process with container resource
self.container.resource_registry.create_association(
self.cc_id, "hasProcess", proc_id)
else:
process_instance._proc_res_id = None
# Process type specific registration
# TODO: Factor out into type specific handler functions
if process_instance._proc_type == "service":
# Registration of SERVICE process: in resource registry
service_list, _ = self.container.resource_registry.find_resources(
restype="Service", name=process_instance.name)
if service_list:
process_instance._proc_svc_id = service_list[0]._id
else:
# We are starting the first process of a service instance
# TODO: This should be created by the HA Service agent in the future
svc_obj = Service(
name=process_instance.name,
exchange_name=process_instance._proc_listen_name,
state=ServiceStateEnum.READY)
process_instance._proc_svc_id, _ = self.container.resource_registry.create(
svc_obj)
# Create association to service definition resource
svcdef_list, _ = self.container.resource_registry.find_resources(
restype="ServiceDefinition", name=process_instance.name)
if svcdef_list:
self.container.resource_registry.create_association(
process_instance._proc_svc_id, "hasServiceDefinition",
svcdef_list[0]._id)
else:
log.error("Cannot find ServiceDefinition resource for %s",
process_instance.name)
self.container.resource_registry.create_association(
process_instance._proc_svc_id, "hasProcess", proc_id)
elif process_instance._proc_type == "agent":
# Registration of AGENT process: in Directory
caps = process_instance.get_capabilities()
self.container.directory.register(
"/Agents", process_instance.id,
**dict(name=process_instance._proc_name,
container=process_instance.container.id,
resource_id=process_instance.resource_id,
agent_id=process_instance.agent_id,
def_id=process_instance.agent_def_id,
capabilities=caps))
self._call_proc_state_changed(process_instance,
ProcessStateEnum.RUNNING)
def terminate_process(self, process_id, do_notifications=True):
"""
Terminates a process and all its resources. Termination is graceful with timeout.
@param process_id The id of the process to terminate. Should exist in the container's
list of processes or this will raise.
@param do_notifications If True, emits process state changes for TERMINATING and TERMINATED.
If False, supresses any state changes. Used near EXITED and FAILED.
"""
process_instance = self.procs.get(process_id, None)
if not process_instance:
raise BadRequest(
"Cannot terminate. Process id='%s' unknown on container id='%s'"
% (process_id, self.container.id))
log.info("ProcManager.terminate_process: %s -> pid=%s",
process_instance._proc_name, process_id)
if do_notifications:
self._call_proc_state_changed(process_instance,
ProcessStateEnum.TERMINATING)
self._process_quit(process_instance)
self._unregister_process(process_id, process_instance)
if do_notifications:
self._call_proc_state_changed(process_instance,
ProcessStateEnum.TERMINATED)
def _unregister_process(self, process_id, process_instance):
# Remove process registration in resource registry
if process_instance._proc_res_id:
try:
self.container.resource_registry.delete(
process_instance._proc_res_id, del_associations=True)
except NotFound: #, HTTPException):
# if it's already gone, it's already gone!
pass
except Exception, ex:
log.exception(ex)
pass
# Cleanup for specific process types
if process_instance._proc_type == "service":
# Check if this is the last process for this service and do auto delete service resources here
svcproc_list = []
try:
svcproc_list, _ = self.container.resource_registry.find_objects(
process_instance._proc_svc_id,
"hasProcess",
"Process",
id_only=True)
except ResourceNotFound:
# if it's already gone, it's already gone!
pass
if not svcproc_list:
try:
self.container.resource_registry.delete(
process_instance._proc_svc_id, del_associations=True)
except NotFound:
# if it's already gone, it's already gone!
pass
except Exception, ex:
log.exception(ex)
pass
def gradient_test_parser(document):
'''
This parser YIELDS a document per call or until it's done
The format for the document is CSV in this table format
Array,Instrument Class,Reference Designator,Data Product used as Input Data (DAT),Data Product used as Input Parameter X,Units of DAT,Units of X,DDATDX,MINDX,STARTDAT,TOLDAT
Document Schema:
array:
origin: Array
type: String
instrument_class:
origin: Instrument Class
type: String
reference_designator:
origin: Reference Designator
type: String
dat:
origin: Data Product used as Input Data (DAT)
type: String
x:
origin: Data Product used as Input Parameter X
type: String
units_dat:
origin: Units of DAT
type: String
units_x:
origin: Units of X
type: String
d_dat_dx:
origin: DDATDX
type: float64
min_dx:
origin: MINDX
type: float64
start_dat:
origin: STARTDAT
type: float64
tol_dat:
origin: TOLDAT
type: float64
'''
sio = StringIO()
sio.write(document)
sio.seek(0)
dr = DictReader(sio)
for row in dr:
try:
key = '_'.join([
'grad', row['Reference Designator'],
row['Data Product used as Input Data (DAT)'],
row['Data Product used as Input Parameter X']
])
document = {}
document['array'] = row['Array']
document['instrument_class'] = row['Instrument Class']
document['reference_designator'] = row['Reference Designator']
document['dat'] = row['Data Product used as Input Data (DAT)']
document['x'] = row['Data Product used as Input Parameter X']
document['units_dat'] = row['Units of DAT']
document['units_x'] = row['Units of X']
document['d_dat_dx'] = float(row['DDATDX'])
document['min_dx'] = float(row['MINDX']) if row['MINDX'] else 0.
document['start_dat'] = float(
row['STARTDAT']) if row['STARTDAT'] else 0.
document['tol_dat'] = float(row['TOLDAT'])
yield key, document
except TypeError:
log.exception("Couldn't parse row")
continue
except ValueError:
log.exception("Couldn't parse row")
continue
return