def _build_header(self, raw_msg):
"""
Builds the header for this Process-level RPC conversation.
https://confluence.oceanobservatories.org/display/syseng/CIAD+COI+OV+Common+Message+Format
"""
context = self._process.get_context()
log.debug('ProcessRPCRequestEndpointUnit._build_header has context of: %s', context)
# conv-id/seq/protocol are set in the base class
header = RPCRequestEndpointUnit._build_header(self, raw_msg)
# add our process identity to the headers
header.update({'sender-name' : self._process.name or 'unnamed-process', # @TODO
'sender' : 'todo'})
# use context to set security attributes forward
if isinstance(context, dict):
# fwd on ion-user-id and expiry, according to common message format spec
user_id = context.get('ion-user-id', None)
expiry = context.get('expiry', None)
if user_id: header['ion-user-id'] = user_id
if expiry: header['expiry'] = expiry
return header
def _fetch_file(self, url):
log.debug("Downloading: %s" % url)
file_name = url.split('/')[-1]
if(os.path.exists('/tmp/' + file_name)):
log.debug(" -- file already downloaded")
return '/tmp/' + file_name
u = urllib2.urlopen(url)
f = open('/tmp/' + file_name, 'wb')
meta = u.info()
file_size = int(meta.getheaders("Content-Length")[0])
print "Downloading: %s Bytes: %s" % (file_name, file_size)
file_size_dl = 0
block_sz = 8192
while True:
buffer = u.read(block_sz)
if not buffer:
break
file_size_dl += len(buffer)
f.write(buffer)
status = r"%10d [%3.2f%%]" % (file_size_dl, file_size_dl * 100. / file_size)
status = status + chr(8)*(len(status)+1)
print status,
f.close()
return '/tmp/' + file_name
def stop(self):
log.debug("ProcManager stopping ...")
# Call quit on procs to give them ability to clean up in reverse order
procs_list = sorted(self.procs.values(), key=lambda proc: proc._proc_start_time, reverse=True)
for proc in procs_list:
try:
self.terminate_process(proc.id)
except Exception as ex:
log.warn("Failed to terminate process (%s): %s", proc.id, ex)
# TODO: Have a choice of shutdown behaviors for waiting on children, timeouts, etc
self.proc_sup.shutdown(CFG.get_safe("container.timeout.shutdown"))
if self.procs:
log.warn("ProcManager procs not empty: %s", self.procs)
if self.procs_by_name:
log.warn("ProcManager procs_by_name not empty: %s", self.procs_by_name)
# Remove Resource registration
if self.container.has_capability(self.container.CCAP.RESOURCE_REGISTRY):
try:
self.container.resource_registry.delete(self.cc_id, del_associations=True)
except NotFound:
# already gone, this is ok
pass
if self.pd_enabled:
self._stop_process_dispatcher()
log.debug("ProcManager stopped, OK.")
def read_data_process(self, data_process_id=""):
# Read DataProcess object with _id matching id
log.debug("Reading DataProcess object id: %s" % data_process_id)
data_proc_obj = self.clients.resource_registry.read(data_process_id)
if not data_proc_obj:
raise NotFound("DataProcess %s does not exist" % data_process_id)
return data_proc_obj
def _link_resources_lowlevel(self, subject_id='', association_type='', object_id='', check_duplicates=True):
"""
create an association
@param subject_id the resource ID of the predefined type
@param association_type the predicate
@param object_id the resource ID of the type to be joined
@param check_duplicates whether to check for an existing association of this exact subj-pred-obj
@todo check for errors: does RR check for bogus ids?
"""
assert(type("") == type(subject_id) == type(object_id))
if check_duplicates:
dups = self._resource_link_exists(subject_id, association_type, object_id)
if dups:
log.debug("Create %s Association from '%s': ALREADY EXISTS"
% (self._assn_name(association_type),
self._toplevel_call()))
return dups
associate_success = self.RR.create_association(subject_id,
association_type,
object_id)
log.debug("Create %s Association from '%s': %s"
% (self._assn_name(association_type),
self._toplevel_call(),
str(associate_success)))
return associate_success
def is_trusted_request():
if request.remote_addr is not None:
log.debug("%s from: %s: %s", request.method, request.remote_addr, request.url)
if not service_gateway_instance.is_trusted_address(request.remote_addr):
abort(403)
def on_start(self):
log.debug('EventAlertTransform.on_start()')
super(EventAlertTransform, self).on_start()
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# get the algorithm to use
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
self.timer_origin = self.CFG.get_safe('process.timer_origin', 'Interval Timer')
self.instrument_origin = self.CFG.get_safe('process.instrument_origin', '')
self.counter = 0
self.event_times = []
#-------------------------------------------------------------------------------------
# Set up a listener for instrument events
#-------------------------------------------------------------------------------------
self.instrument_event_queue = gevent.queue.Queue()
def instrument_event_received(message, headers):
log.debug("EventAlertTransform received an instrument event here::: %s" % message)
self.instrument_event_queue.put(message)
self.instrument_event_subscriber = EventSubscriber(origin = self.instrument_origin,
callback=instrument_event_received)
self.instrument_event_subscriber.start()
#-------------------------------------------------------------------------------------
# Create the publisher that will publish the Alert message
#-------------------------------------------------------------------------------------
self.event_publisher = EventPublisher()
def _transform(self, obj):
# Note: This check to detect an IonObject is a bit risky (only type_)
if isinstance(obj, dict) and "type_" in obj:
objc = obj.copy()
type = objc.pop('type_')
# don't supply a dict - we want the object to initialize with all its defaults intact,
# which preserves things like IonEnumObject and invokes the setattr behavior we want there.
ion_obj = self._obj_registry.new(type.encode('ascii'))
for k, v in objc.iteritems():
setattr(ion_obj, k, v)
return ion_obj
if _have_numpy:
if isinstance(obj, dict):
msg = obj.get('numpy',False)
log.debug('message = %s', msg)
if msg:
shape = msg.get('shape')
type = msg.get('type')
data = msg.get('body')
log.debug('Numpy Array Detected:\n type: %s\n shape: %s\n body: %s',type,shape,data)
ret = np.fromstring(string=data,dtype=type).reshape(shape)
return np.array(ret)
return obj
def send_email(event, msg_recipient, smtp_client, rr_client):
"""
A common method to send email with formatting
@param event Event
@param msg_recipient str
@param smtp_client fake or real smtp client object
"""
#log.debug("Got type of event to notify on: %s", event.type_)
#------------------------------------------------------------------------------------
# the 'from' email address for notification emails
#------------------------------------------------------------------------------------
ION_NOTIFICATION_EMAIL_ADDRESS = '*****@*****.**'
smtp_sender = CFG.get_safe('server.smtp.sender', ION_NOTIFICATION_EMAIL_ADDRESS)
msg = convert_events_to_email_message([event], rr_client)
msg['From'] = smtp_sender
msg['To'] = msg_recipient
log.debug("UNS sending email from %s to %s for event type: %s", smtp_sender,msg_recipient, event.type_)
#log.debug("UNS using the smtp client: %s", smtp_client)
try:
smtp_client.sendmail(smtp_sender, [msg_recipient], msg.as_string())
except: # Can be due to a broken connection... try to create a connection
smtp_client = setting_up_smtp_client()
log.debug("Connect again...message received after ehlo exchange: %s", str(smtp_client.ehlo()))
smtp_client.sendmail(smtp_sender, [msg_recipient], msg.as_string())
def process_oms_event():
json_params = {}
# oms direct request
if request.data:
json_params = json_loads(str(request.data))
log.debug('ServiceGatewayService:process_oms_event request.data: %s', json_params)
#validate payload
if 'platform_id' not in json_params or 'message' not in json_params:
log.warning('Invalid OMS event format. payload_data: %s', json_params)
#return gateway_json_response(OMS_BAD_REQUEST_RESPONSE)
#prepare the event information
try:
#create a publisher to relay OMS events into the system as DeviceEvents
event_publisher = EventPublisher()
event_publisher.publish_event(
event_type='OMSDeviceStatusEvent',
origin_type='OMS Platform',
origin=json_params.get('platform_id', 'NOT PROVIDED'),
sub_type='',
description = json_params.get('message', ''),
status_details = json_params)
except Exception, e:
log.error('Could not publish OMS event: %s. Event data: %s', e.message, json_params)
def find_events(self, origin='', type='', min_datetime=0, max_datetime=0, limit= -1, descending=False):
"""
This method leverages couchdb view and simple filters. It does not use elastic search.
Returns a list of events that match the specified search criteria. Will throw a not NotFound exception
if no events exist for the given parameters.
@param origin str
@param event_type str
@param min_datetime int seconds
@param max_datetime int seconds
@param limit int (integer limiting the number of results (0 means unlimited))
@param descending boolean (if True, reverse order (of production time) is applied, e.g. most recent first)
@retval event_list []
@throws NotFound object with specified parameters does not exist
@throws NotFound object with specified parameters does not exist
"""
event_tuples = []
try:
event_tuples = self.container.event_repository.find_events(event_type=type, origin=origin, start_ts=min_datetime, end_ts=max_datetime, limit=limit, descending=descending)
except Exception as exc:
log.warning("The UNS find_events operation for event origin = %s and type = %s failed. Error message = %s", origin, type, exc.message)
events = [item[2] for item in event_tuples]
log.debug("(find_events) UNS found the following relevant events: %s", events)
return events
def overlay(self, transport, *methods):
for m in methods:
self._methods[m] = getattr(transport, m)
log.debug("ComposableTransport.overlay(%s) %s %s (%s)", self.channel_number, type(transport), transport, transport.channel_number)
self._transports.append(transport)
def stop_consume_impl(self, consumer_tag):
"""
Stops consuming by consumer tag.
"""
#log.debug("AMQPTransport.stop_consume_impl(%s): %s", self._client.channel_number, consumer_tag)
self._sync_call(self._client.basic_cancel, 'callback', consumer_tag)
# PIKA 0.9.5 / GEVENT interaction problem here
# we get called back too early, the basic_cancel hasn't really finished processing yet. we need
# to wait until our consumer tag is removed from the pika channel's consumers dict.
# See: https://gist.github.com/3751870
attempts = 6
while attempts > 0:
if consumer_tag not in self._client._consumers:
break
else:
log.debug("stop_consume_impl waiting for ctag to be removed from consumers, attempts rem: %s", attempts)
attempts -= 1
sleep(1) if attempts < 4 else sleep(0.1) # Wait shorter the first few times
if consumer_tag in self._client._consumers:
raise TransportError("stop_consume_impl did not complete in the expected amount of time, transport may be compromised")
def lookup(self, qualified_key='/'):
"""
Read entry residing in directory at parent node level.
"""
log.debug("Reading content at path %s" % qualified_key)
direntry = self._safe_read(qualified_key)
return direntry.attributes if direntry else None
def get_datastore(self, ds_name, profile=DataStore.DS_PROFILE.BASIC, config=None):
"""
Factory method to get a datastore instance from given name, profile and config.
@param ds_name Logical name of datastore (will be scoped with sysname)
@param profile One of known constants determining the use of the store
@param config Override config to use
"""
validate_true(ds_name, 'ds_name must be provided')
if ds_name in self._datastores:
log.debug("get_datastore(): Found instance of store '%s'" % ds_name)
return self._datastores[ds_name]
scoped_name = DatastoreManager.get_scoped_name(ds_name)
# Create a datastore instance
log.info("get_datastore(): Create instance of store '%s' as database=%s" % (ds_name, scoped_name))
new_ds = DatastoreManager.get_datastore_instance(ds_name, profile)
# Create store if not existing
if not new_ds.datastore_exists(scoped_name):
new_ds.create_datastore(scoped_name, create_indexes=True, profile=profile)
else:
# NOTE: This may be expensive if called more than once per container
# If views exist and are dropped and recreated
new_ds._define_views(profile=profile, keepviews=True)
# Set a few standard datastore instance fields
new_ds.local_name = ds_name
new_ds.ds_profile = profile
self._datastores[ds_name] = new_ds
return new_ds
def find_res_by_attribute(self, restype, attr_name, attr_value=None, id_only=False, filter=None):
log.debug("find_res_by_attribute(restype=%s, attr_name=%s, attr_value=%s)", restype, attr_name, attr_value)
if not attr_name or type(attr_name) is not str:
raise BadRequest('Argument attr_name illegal')
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
filter = filter if filter is not None else {}
qual_ds_name = self._get_datastore_name()
if id_only:
query = "SELECT id, type_, json_specialattr(doc) FROM " + qual_ds_name
else:
query = "SELECT id, type_, json_specialattr(doc), doc FROM " + qual_ds_name
query_clause = " WHERE lcstate<>'DELETED' "
query_args = dict(type_=restype, att=attr_name, val=attr_value)
if attr_value: # Note: cannot make None test here (and allow empty string because of default service args "")
query_clause += "AND json_specialattr(doc)=%(spc)s"
query_args['spc'] = "%s=%s" % (attr_name, attr_value)
else:
query_clause += "AND json_specialattr(doc) LIKE %(spc)s"
query_args['spc'] = "%s=%%" % (attr_name, )
if restype:
query_clause += " AND type_=%(type_)s"
query_clause = self._add_access_filter(filter, qual_ds_name, query_clause, query_args)
extra_clause = filter.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
res_assocs = [dict(id=self._prep_id(row[0]), type=row[1], attr_name=attr_name, attr_value=row[2].split("=",1)[-1]) for row in rows]
log.debug("find_res_by_attribute() found %s objects", len(res_assocs))
return self._prepare_find_return(rows, res_assocs, id_only=id_only)
def get_data_process_subscriptions_count(self, data_process_id=""):
if not data_process_id:
raise BadRequest("The data_process_definition_id parameter is empty")
subscription_ids, _ = self.clients.resource_registry.find_objects(subject=data_process_id, predicate=PRED.hasSubscription, id_only=True)
log.debug("get_data_process_subscriptions_count(id=%s): %s subscriptions", data_process_id, len(subscription_ids))
return len(subscription_ids)
def find_res_by_lcstate(self, lcstate, restype=None, id_only=False, filter=None):
log.debug("find_res_by_lcstate(lcstate=%s, restype=%s)", lcstate, restype)
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
if '_' in lcstate:
log.warn("Search for compound lcstate restricted to maturity: %s", lcstate)
lcstate,_ = lcstate.split("_", 1)
filter = filter if filter is not None else {}
qual_ds_name = self._get_datastore_name()
if id_only:
query = "SELECT id, name, type_, lcstate, availability FROM " + qual_ds_name
else:
query = "SELECT id, name, type_, lcstate, availability, doc FROM " + qual_ds_name
query_clause = " WHERE "
query_args = dict(type_=restype, lcstate=lcstate)
is_maturity = lcstate not in AvailabilityStates
if is_maturity:
query_clause += "lcstate=%(lcstate)s"
else:
query_clause += "availability=%(lcstate)s"
if restype:
query_clause += " AND type_=%(type_)s"
query_clause = self._add_access_filter(filter, qual_ds_name, query_clause, query_args)
extra_clause = filter.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
res_assocs = [dict(id=self._prep_id(row[0]), name=row[1], type=row[2], lcstate=row[3] if is_maturity else row[4]) for row in rows]
log.debug("find_res_by_lcstate() found %s objects", len(res_assocs))
return self._prepare_find_return(rows, res_assocs, id_only=id_only)
def find_res_by_nested_type(self, nested_type, restype=None, id_only=False, filter=None):
log.debug("find_res_by_nested_type(nested_type=%s, restype=%s)", nested_type, restype)
if not nested_type or type(nested_type) is not str:
raise BadRequest('Argument nested_type illegal')
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
filter = filter if filter is not None else {}
qual_ds_name = self._get_datastore_name()
if id_only:
query = "SELECT id, type_ FROM " + qual_ds_name
else:
query = "SELECT id, type_, doc FROM " + qual_ds_name
query_clause = " WHERE lcstate<>'DELETED' "
query_args = dict(type_=restype, nest=[nested_type])
query_clause += "AND %(nest)s <@ json_nested(doc)"
if restype:
query_clause += " AND type_=%(type_)s"
query_clause = self._add_access_filter(filter, qual_ds_name, query_clause, query_args)
extra_clause = filter.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
res_assocs = [dict(id=self._prep_id(row[0]), type=row[1], nested_type=nested_type) for row in rows]
log.debug("find_res_by_nested_type() found %s objects", len(res_assocs))
return self._prepare_find_return(rows, res_assocs, id_only=id_only)
def setting_up_smtp_client():
'''
Sets up the smtp client
'''
#------------------------------------------------------------------------------------
# the default smtp server
#------------------------------------------------------------------------------------
ION_SMTP_SERVER = 'mail.oceanobservatories.org'
smtp_host = CFG.get_safe('server.smtp.host', ION_SMTP_SERVER)
smtp_port = CFG.get_safe('server.smtp.port', 25)
smtp_sender = CFG.get_safe('server.smtp.sender')
smtp_password = CFG.get_safe('server.smtp.password')
if CFG.get_safe('system.smtp',False): #Default is False - use the fake_smtp
log.debug('Using the real SMTP library to send email notifications!')
smtp_client = smtplib.SMTP(smtp_host)
smtp_client.ehlo()
smtp_client.starttls()
smtp_client.login(smtp_sender, smtp_password)
else:
log.debug('Using a fake SMTP library to simulate email notifications!')
smtp_client = fake_smtplib.SMTP(smtp_host)
return smtp_client
def find_res_by_type(self, restype, lcstate=None, id_only=False, filter=None):
log.debug("find_res_by_type(restype=%s, lcstate=%s)", restype, lcstate)
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
if lcstate:
raise BadRequest('lcstate not supported anymore in find_res_by_type')
filter = filter if filter is not None else {}
qual_ds_name = self._get_datastore_name()
if id_only:
query = "SELECT id, name, type_, lcstate FROM " + qual_ds_name
else:
query = "SELECT id, name, type_, lcstate, doc FROM " + qual_ds_name
query_clause = " WHERE lcstate<>'DELETED' "
query_args = dict(type_=restype, lcstate=lcstate)
if restype:
query_clause += "AND type_=%(type_)s"
else:
# Returns ALL documents, only limited by filter
query_clause = ""
query_clause = self._add_access_filter(filter, qual_ds_name, query_clause, query_args)
extra_clause = filter.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
res_assocs = [dict(id=self._prep_id(row[0]), name=row[1], type=row[2]) for row in rows]
log.debug("find_res_by_type() found %s objects", len(res_assocs))
return self._prepare_find_return(rows, res_assocs, id_only=id_only)
def find_res_by_lcstate(self, lcstate, restype=None, id_only=False):
log.debug("find_res_by_lcstate(lcstate=%s, restype=%s)" % (lcstate, restype))
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
db = self.server[self.datastore_name]
view = db.view(self._get_viewname("resource","by_lcstate"), include_docs=(not id_only))
is_hierarchical = (lcstate in ResourceLifeCycleSM.STATE_ALIASES)
# lcstate is a hiearachical state and we need to treat the view differently
if is_hierarchical:
key = [1, lcstate]
else:
key = [0, lcstate]
if restype:
key.append(restype)
endkey = list(key)
endkey.append(END_MARKER)
rows = view[key:endkey]
if is_hierarchical:
res_assocs = [dict(lcstate=row['key'][3], type=row['key'][2], name=row['key'][4], id=row.id) for row in rows]
else:
res_assocs = [dict(lcstate=row['key'][1], type=row['key'][2], name=row['key'][3], id=row.id) for row in rows]
log.debug("find_res_by_lcstate() found %s objects" % (len(res_assocs)))
if id_only:
res_ids = [row.id for row in rows]
return (res_ids, res_assocs)
else:
res_docs = [self._persistence_dict_to_ion_object(row.doc) for row in rows]
return (res_docs, res_assocs)
def delete_doc(self, doc, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
try:
db = self.server[datastore_name]
except ValueError:
raise BadRequest("Data store name %s is invalid" % datastore_name)
if type(doc) is str:
log.info('Deleting object %s/%s' % (datastore_name, doc))
if self._is_in_association(doc, datastore_name):
obj = self.read(doc, datastore_name)
log.warn("XXXXXXX Attempt to delete object %s that still has associations" % str(obj))
# raise BadRequest("Object cannot be deleted until associations are broken")
try:
del db[doc]
except ResourceNotFound:
raise NotFound('Object with id %s does not exist.' % str(doc))
else:
log.info('Deleting object %s/%s' % (datastore_name, doc["_id"]))
if self._is_in_association(doc["_id"], datastore_name):
log.warn("XXXXXXX Attempt to delete object %s that still has associations" % str(doc))
# raise BadRequest("Object cannot be deleted until associations are broken")
try:
res = db.delete(doc)
except ResourceNotFound:
raise NotFound('Object with id %s does not exist.' % str(doc["_id"]))
log.debug('Delete result: %s' % str(res))
def test_comm_config_type_list(self):
types = CommConfig.valid_type_list()
log.debug( "types: %s" % types)
known_types = ['ethernet']
self.assertEqual(sorted(types), sorted(known_types))
def create_doc(self, doc, object_id=None, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
if '_id' in doc:
raise BadRequest("Doc must not have '_id'")
if '_rev' in doc:
raise BadRequest("Doc must not have '_rev'")
if object_id:
try:
self.read(object_id, '', datastore_name)
raise BadRequest("Object with id %s already exist" % object_id)
except NotFound:
pass
# Assign an id to doc (recommended in CouchDB documentation)
doc["_id"] = object_id or uuid4().hex
log.info('Creating new object %s/%s' % (datastore_name, doc["_id"]))
log.debug('create doc contents: %s', doc)
# Save doc. CouchDB will assign version to doc.
try:
res = self.server[datastore_name].save(doc)
except ResourceNotFound:
raise BadRequest("Data store %s does not exist" % datastore_name)
except ResourceConflict:
raise BadRequest("Object with id %s already exist" % doc["_id"])
except ValueError:
raise BadRequest("Data store name %s invalid" % datastore_name)
log.debug('Create result: %s' % str(res))
id, version = res
return (id, version)
def start(self):
log.debug("GovernanceController starting ...")
self._CFG = CFG
self.enabled = CFG.get_safe('interceptor.interceptors.governance.config.enabled', False)
log.info("GovernanceInterceptor enabled: %s" % str(self.enabled))
self.policy_event_subscriber = None
#containers default to not Org Boundary and ION Root Org
self._is_container_org_boundary = CFG.get_safe('container.org_boundary',False)
self._container_org_name = CFG.get_safe('container.org_name', CFG.get_safe('system.root_org', 'ION'))
self._container_org_id = None
self._system_root_org_name = CFG.get_safe('system.root_org', 'ION')
self._is_root_org_container = (self._container_org_name == self._system_root_org_name)
if self.enabled:
config = CFG.get_safe('interceptor.interceptors.governance.config')
self.initialize_from_config(config)
self.policy_event_subscriber = EventSubscriber(event_type=OT.PolicyEvent, callback=self.policy_event_callback)
self.policy_event_subscriber.start()
self.rr_client = ResourceRegistryServiceProcessClient(node=self.container.node, process=self.container)
self.policy_client = PolicyManagementServiceProcessClient(node=self.container.node, process=self.container)
def launch(self):
"""
Launches the simulator process as indicated by _COMMAND.
@return (rsn_oms, uri) A pair with the CIOMSSimulator instance and the
associated URI to establish connection with it.
"""
log.debug("[OMSim] Launching: %s", _COMMAND)
self._process = self._spawn(_COMMAND)
if not self._process or not self.poll():
msg = "[OMSim] Failed to launch simulator: %s" % _COMMAND
log.error(msg)
raise Exception(msg)
log.debug("[OMSim] process started, pid: %s", self.getpid())
# give it some time to start up
sleep(5)
# get URI:
uri = None
with open("logs/rsn_oms_simulator.yml", buffering=1) as f:
# we expect one of the first few lines to be of the form:
# rsn_oms_simulator_uri=xxxx
# where xxxx is the uri -- see oms_simulator_server.
while uri is None:
line = f.readline()
if line.index("rsn_oms_simulator_uri=") == 0:
uri = line[len("rsn_oms_simulator_uri="):].strip()
self._rsn_oms = CIOMSClientFactory.create_instance(uri)
return self._rsn_oms, uri
def __init__(self, host=None, port=None, datastore_name='prototype', options="", profile=DataStore.DS_PROFILE.BASIC):
log.debug('__init__(host=%s, port=%s, datastore_name=%s, options=%s' % (host, port, datastore_name, options))
self.host = host or CFG.server.couchdb.host
self.port = port or CFG.server.couchdb.port
# The scoped name of the datastore
self.datastore_name = datastore_name
self.auth_str = ""
try:
if CFG.server.couchdb.username and CFG.server.couchdb.password:
self.auth_str = "%s:%s@" % (CFG.server.couchdb.username, CFG.server.couchdb.password)
log.debug("Using username:password authentication to connect to datastore")
except AttributeError:
log.error("CouchDB username:password not configured correctly. Trying anonymous...")
connection_str = "http://%s%s:%s" % (self.auth_str, self.host, self.port)
#connection_str = "http://%s:%s" % (self.host, self.port)
# TODO: Security risk to emit password into log. Remove later.
log.info('Connecting to CouchDB server: %s' % connection_str)
self.server = couchdb.Server(connection_str)
# Datastore specialization
self.profile = profile
# serializers
self._io_serializer = IonObjectSerializer()
self._io_deserializer = IonObjectDeserializer(obj_registry=obj_registry)
def __init__(self, container):
log.debug("ExchangeManager initializing ...")
self.container = container
# Define the callables that can be added to Container public API
# @TODO: remove
self.container_api = [self.create_xs,
self.create_xp,
self.create_xn_service,
self.create_xn_process,
self.create_xn_queue]
# Add the public callables to Container
for call in self.container_api:
setattr(self.container, call.__name__, call)
self.default_xs = ExchangeSpace(self, ION_ROOT_XS)
self._xs_cache = {} # caching of xs names to RR objects
self._default_xs_obj = None # default XS registry object
self.org_id = None
# mappings
self.xs_by_name = { ION_ROOT_XS: self.default_xs } # friendly named XS to XSO
self.xn_by_name = {} # friendly named XN to XNO
# xn by xs is a property
self._chan = None
# @TODO specify our own to_name here so we don't get auto-behavior - tricky chicken/egg
self._ems_client = ExchangeManagementServiceProcessClient(process=self.container)
self._rr_client = ResourceRegistryServiceProcessClient(process=self.container)
def find_res_by_alternative_id(self, alt_id=None, alt_id_ns=None, id_only=False, filter=None):
log.debug("find_res_by_alternative_id(restype=%s, alt_id_ns=%s)", alt_id, alt_id_ns)
if alt_id and type(alt_id) is not str:
raise BadRequest('Argument alt_id illegal')
if alt_id_ns and type(alt_id_ns) is not str:
raise BadRequest('Argument alt_id_ns illegal')
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
filter = filter if filter is not None else {}
ds, datastore_name = self._get_datastore()
key = []
if alt_id:
key.append(alt_id)
if alt_id_ns is not None:
key.append(alt_id_ns)
endkey = self._get_endkey(key)
rows = ds.view(self._get_view_name("resource", "by_altid"), include_docs=(not id_only), start_key=key, end_key=endkey, **filter)
if alt_id_ns and not alt_id:
res_assocs = [dict(alt_id=row['key'][0], alt_id_ns=row['key'][1], id=row['id']) for row in rows if row['key'][1] == alt_id_ns]
else:
res_assocs = [dict(alt_id=row['key'][0], alt_id_ns=row['key'][1], id=row['id']) for row in rows]
log.debug("find_res_by_alternative_id() found %s objects", len(res_assocs))
self._count(find_res_by_altid_call=1, find_res_by_altid_obj=len(res_assocs))
if id_only:
res_ids = [row['id'] for row in res_assocs]
return (res_ids, res_assocs)
else:
if alt_id_ns and not alt_id:
res_docs = [self._persistence_dict_to_ion_object(self._get_row_doc(row)) for row in rows if row['key'][1] == alt_id_ns]
else:
res_docs = [self._persistence_dict_to_ion_object(self._get_row_doc(row)) for row in rows]
return (res_docs, res_assocs)
self._url = 'http://%s:%s@%s:%d' % (username, password, server, port) \
if username else 'http://%s:%d' % (server, port)
self._db_name = instance
self._server_name = server
self._max_retry = max_retry
self._server = Server(self._url)
if can_create or must_create:
try:
log.debug('creating %s on %s', instance, self._url)
self._db = self._server.create(self._db_name)
except Exception, e:
if must_create:
raise self._wrap_error('initialize', self._db_name, e)
else:
log.debug('connecting to %s on %s', instance, server)
self._db = self._server[self._db_name]
def drop(self):
log.debug('deleting %s from %s', self._db_name, self._server_name)
self._server.delete(self._db_name)
self._db = None
def _is_list(self, op, target):
# check that argument is string, dictionary or list of ONE of them
if isinstance(target, str) or isinstance(target, dict):
return False
elif all(isinstance(t, str)
for t in target) or all(isinstance(t, dict) for t in target):
return True
else:
def add_to_white_list(self, root_cert_string):
log.debug("Adding certificate <%s> to white list" % root_cert_string)
self.white_list.append(root_cert_string)
def __init__(self):
self.cont_cert = None
self.cont_key = None
self.root_cert = None
self.white_list = []
# Look for certificates and keys in "the usual places"
certstore_path = self.certstore = CFG.get_safe(
'authentication.certstore', CERTSTORE_PATH)
log.debug("certstore_path: %s" % str(certstore_path))
keystore_path = self.certstore = CFG.get_safe(
'authentication.keystore', KEYSTORE_PATH)
log.debug("keystore_path: %s" % str(keystore_path))
if certstore_path and keystore_path:
if certstore_path == 'directory':
log.debug("Container.instance.directory: " %
str(Container.instance.directory))
Container.instance.directory.load_authentication()
else:
cont_cert_path = os.path.join(certstore_path,
CONTAINER_CERT_NAME)
log.debug("cont_cert_path: %s" % cont_cert_path)
cont_key_path = os.path.join(keystore_path, CONTAINER_KEY_NAME)
log.debug("cont_key_path: %s" % cont_key_path)
root_cert_path = os.path.join(certstore_path, ORG_CERT_NAME)
log.debug("root_cert_path: %s" % root_cert_path)
if os.path.exists(cont_cert_path) and os.path.exists(
cont_key_path) and os.path.exists(root_cert_path):
with open(cont_cert_path, 'r') as f:
self.cont_cert = f.read()
log.debug("cont_cert: %s" % self.cont_cert)
self.cont_key = EVP.load_key(cont_key_path)
with open(root_cert_path, 'r') as f:
self.root_cert = f.read()
log.debug("root_cert: %s" % self.root_cert)
self.add_to_white_list(self.root_cert)
def process_batch(self, start_time = '', end_time = ''):
"""
This method is launched when an process_batch event is received. The user info dictionary maintained
by the User Notification Service is used to query the event repository for all events for a particular
user that have occurred in a provided time interval, and then an email is sent to the user containing
the digest of all the events.
@param start_time int milliseconds
@param end_time int milliseconds
"""
self.smtp_client = setting_up_smtp_client()
if end_time <= start_time:
return
for user_id, value in self.user_info.iteritems():
notifications = value['notifications']
notification_preferences = value['notification_preferences']
# Ignore users who do NOT want batch notifications or who have disabled the delivery switch
# However, if notification preferences have not been set for the user, use the default mechanism and do not bother
if notification_preferences:
if notification_preferences.delivery_mode != NotificationDeliveryModeEnum.BATCH \
or not notification_preferences.delivery_enabled:
continue
events_for_message = []
search_time = "SEARCH 'ts_created' VALUES FROM %s TO %s FROM 'events_index'" % (start_time, end_time)
for notification in notifications:
# If the notification request has expired, then do not use it in the search
if notification.temporal_bounds.end_datetime:
continue
if notification.origin:
search_origin = 'search "origin" is "%s" from "events_index"' % notification.origin
else:
search_origin = 'search "origin" is "*" from "events_index"'
if notification.origin_type:
search_origin_type= 'search "origin_type" is "%s" from "events_index"' % notification.origin_type
else:
search_origin_type= 'search "origin_type" is "*" from "events_index"'
if notification.event_type:
search_event_type = 'search "type_" is "%s" from "events_index"' % notification.event_type
else:
search_event_type = 'search "type_" is "*" from "events_index"'
search_string = search_time + ' and ' + search_origin + ' and ' + search_origin_type + ' and ' + search_event_type
# get the list of ids corresponding to the events
ret_vals = self.discovery.parse(search_string)
events_for_message.extend(self.datastore.read_mult(ret_vals))
log.debug("Found following events of interest to user, %s: %s", user_id, events_for_message)
# send a notification email to each user using a _send_email() method
if events_for_message:
self.format_and_send_email(events_for_message = events_for_message,
user_id = user_id,
smtp_client=self.smtp_client)
self.smtp_client.quit()
def create_notification(self, notification=None, user_id=''):
"""
Persists the provided NotificationRequest object for the specified Origin id.
Associate the Notification resource with the user_id string.
returned id is the internal id by which NotificationRequest will be identified
in the data store.
@param notification NotificationRequest
@param user_id str
@retval notification_id str
@throws BadRequest if object passed has _id or _rev attribute
"""
if not user_id:
raise BadRequest("User id not provided.")
log.debug("Create notification called for user_id: %s, and notification: %s", user_id, notification)
#---------------------------------------------------------------------------------------------------
# Persist Notification object as a resource if it has already not been persisted
#---------------------------------------------------------------------------------------------------
# if the notification has already been registered, simply use the old id
notification_id = self._notification_in_notifications(notification, self.notifications)
# since the notification has not been registered yet, register it and get the id
temporal_bounds = TemporalBounds()
temporal_bounds.start_datetime = get_ion_ts()
temporal_bounds.end_datetime = ''
if not notification_id:
notification.temporal_bounds = temporal_bounds
notification_id, _ = self.clients.resource_registry.create(notification)
self.notifications[notification_id] = notification
else:
log.debug("Notification object has already been created in resource registry before. No new id to be generated. notification_id: %s", notification_id)
# Read the old notification already in the resource registry
notification = self.clients.resource_registry.read(notification_id)
# Update the temporal bounds of the old notification resource
notification.temporal_bounds = temporal_bounds
# Update the notification in the resource registry
self.clients.resource_registry.update(notification)
log.debug("The temporal bounds for this resubscribed notification object with id: %s, is: %s", notification_id,notification.temporal_bounds)
# Link the user and the notification with a hasNotification association
assocs= self.clients.resource_registry.find_associations(subject=user_id,
predicate=PRED.hasNotification,
object=notification_id,
id_only=True)
if assocs:
log.debug("Got an already existing association: %s, between user_id: %s, and notification_id: %s", assocs,user_id,notification_id)
return notification_id
else:
log.debug("Creating association between user_id: %s, and notification_id: %s", user_id, notification_id )
self.clients.resource_registry.create_association(user_id, PRED.hasNotification, notification_id)
# read the registered notification request object because this has an _id and is more useful
notification = self.clients.resource_registry.read(notification_id)
# Update the user info object with the notification
self.event_processor.add_notification_for_user(new_notification=notification, user_id=user_id)
#-------------------------------------------------------------------------------------------------------------------
# Generate an event that can be picked by a notification worker so that it can update its user_info dictionary
#-------------------------------------------------------------------------------------------------------------------
log.debug("(create notification) Publishing ReloadUserInfoEvent for notification_id: %s", notification_id)
self.event_publisher.publish_event( event_type= "ReloadUserInfoEvent",
origin="UserNotificationService",
description= "A notification has been created.",
notification_id = notification_id)
return notification_id
def drop(self):
log.debug('deleting %s from %s', self._db_name, self._server_name)
self._server.delete(self._db_name)
self._db = None
def update(self, target, force=False, _depth=0):
""" update one or many entries
if force is False and _rev is missing or not the latest, the update will fail.
if force is True and _rev is missing or not the latest, will find the latest _rev and attempt to update.
since others may be updating at the same time, this may still not be the latest!
update will repeat up to max_retry times before giving up.
return value for a single item update is a tuple (success, id, exception)
return value when target is a list is a list of tuples indicating the outcome of each item updated.
NOTE: updates may not made to DB in order.
"""
if not isinstance(target, list):
result = self.update([target], force=force)
return result[0]
# update the ones that already have revisions
have_revisions = filter(lambda d: '_rev' in d.keys(), target)
results = self._db.update(have_revisions) if len(
have_revisions) else None
# if missing or old _rev, retry update
log.debug('update results: %s', repr(results))
some_attempted = len(have_revisions) > 0
some_failed = some_attempted and not reduce(lambda a, b: a and b,
[r[0] for r in results])
some_not_updated = len(have_revisions) < len(target) or some_failed
can_retry = force and _depth < self._max_retry
if can_retry and some_not_updated:
# start with elements that did not have _rev
retry_target = filter(lambda d: '_rev' not in d.keys(), target)
# now add elements that failed due to wrong _rev
successful_results = []
if results:
doc_with_result = zip(have_revisions, results)
failed_subset_with_result = filter(
lambda t: not t[1][0], doc_with_result
) # TODO: also check t[1][2] is correct exception type for bad _rev
retry_target += [t[0] for t in failed_subset_with_result]
successful_results = [
t[1] for t in filter(lambda t: t[1][0], doc_with_result)
]
# search db and update these _rev values
log.debug(' before %s', repr(retry_target))
self._update_revision(retry_target)
log.debug(' after %s', repr(retry_target))
# some just aren't in the DB any more
not_found_docs = filter(lambda t: '_rev' not in t, retry_target)
not_found_results = [
(False, t['_id'], Exception('not found in DB'))
for t in not_found_docs
] #TODO: use application exception
# for otehrs, try again with udpated _rev
found_rev = filter(lambda t: '_rev' in t, retry_target)
retry_results = self.update(found_rev,
force=True,
_depth=_depth + 1)
results = not_found_results + retry_results + successful_results
# only need to sort once on final exit from recursive retry
if _depth > 1:
return results
# sort results into original order of arguments
out = []
for d in target:
id = d['_id']
result = None
if results:
for t in results:
if t[1] == id:
result = t
break
if not result: # no revision?
if force: # force: would have read latest from DB -- so _id was not found
result = (False, id,
Exception("udpate failed: _id was not found"))
else: # no force: then cannot update
result = (False, id,
Exception("update failed without _rev or force"))
out.append(result)
return out
def create_data_process(self,
data_process_definition_id=None,
in_data_product_ids='',
out_data_products=None,
configuration=None):
"""
@param data_process_definition_id: Object with definition of the
transform to apply to the input data product
@param in_data_product_ids: ID of the input data products
@param out_data_products: list of IDs of the output data products
@retval data_process_id: ID of the newly created data process object
"""
inform = "Input Data Product: "+str(in_data_product_ids)+\
"Transformed by: "+str(data_process_definition_id)+\
"To create output Product: "+str(out_data_products)
log.debug("DataProcessManagementService:create_data_process()\n" +
inform)
if configuration is None:
configuration = {}
# Create and store a new DataProcess with the resource registry
log.debug(
"DataProcessManagementService:create_data_process - Create and store a new DataProcess with the resource registry"
)
data_process_def_obj = self.read_data_process_definition(
data_process_definition_id)
data_process_name = "process_" + data_process_def_obj.name \
+ time.ctime()
self.data_process = IonObject(RT.DataProcess, name=data_process_name)
data_process_id, version = self.clients.resource_registry.create(
self.data_process)
log.debug(
"DataProcessManagementService:create_data_process - Create and store a new DataProcess with the resource registry data_process_id: %s"
+ str(data_process_id))
# Register the data process instance as a data producer with DataAcquisitionMgmtSvc
#TODO: should this be outside this method? Called by orchestration?
data_producer_id = self.clients.data_acquisition_management.register_process(
data_process_id)
log.debug(
"DataProcessManagementService:create_data_process register process with DataAcquisitionMgmtSvc: data_producer_id: %s (L4-CI-SA-RQ-181)",
str(data_producer_id))
self.output_stream_dict = {}
#TODO: should this be outside this method? Called by orchestration?
if out_data_products is None:
raise BadRequest(
"Data Process must have output product(s) specified %s",
str(data_process_definition_id))
for name, out_data_product_id in out_data_products.iteritems():
# check that the product is not already associated with a producer
producer_ids, _ = self.clients.resource_registry.find_objects(
out_data_product_id, PRED.hasDataProducer, RT.DataProducer,
True)
if producer_ids:
raise BadRequest(
"Data Product should not already be associated to a DataProducer %s hasDataProducer %s",
str(data_process_id), str(producer_ids[0]))
#Assign each output Data Product to this producer resource
out_data_product_obj = self.clients.resource_registry.read(
out_data_product_id)
if not out_data_product_obj:
raise NotFound("Output Data Product %s does not exist" %
out_data_product_id)
# Associate with DataProcess: register as an output product for this process
log.debug(
"DataProcessManagementService:create_data_process link data process %s and output out data product: %s (L4-CI-SA-RQ-260)",
str(data_process_id), str(out_data_product_id))
self.clients.data_acquisition_management.assign_data_product(
data_process_id, out_data_product_id, create_stream=False)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.clients.resource_registry.find_objects(
out_data_product_id, PRED.hasStream, RT.Stream, True)
log.debug(
"DataProcessManagementService:create_data_process retrieve out data prod streams: %s",
str(stream_ids))
if not stream_ids:
raise NotFound("No Stream created for output Data Product " +
str(out_data_product_id))
if len(stream_ids) != 1:
raise BadRequest(
"Data Product should only have ONE stream at this time" +
str(out_data_product_id))
self.output_stream_dict[name] = stream_ids[0]
log.debug(
"DataProcessManagementService:create_data_process -Register the data process instance as a data producer with DataAcquisitionMgmtSvc, then retrieve the id of the OUTPUT stream out_stream_id: "
+ str(self.output_stream_dict[name]))
# Associate with dataProcess
self.clients.resource_registry.create_association(
data_process_id, PRED.hasProcessDefinition,
data_process_definition_id)
#check if data process has lookup tables attached
self._find_lookup_tables(data_process_definition_id, configuration)
#Todo: currently this is handled explicitly after creating the data product, that code then calls DMAS:assign_data_product
log.debug(
"DataProcessManagementService:create_data_process associate data process workflows with source data products %s hasInputProducts %s (L4-CI-SA-RQ-260)",
str(data_process_id), str(in_data_product_ids))
for in_data_product_id in in_data_product_ids:
self.clients.resource_registry.create_association(
data_process_id, PRED.hasInputProduct, in_data_product_id)
#check if in data product is attached to an instrument, check instrumentDevice and InstrumentModel for lookup table attachments
instdevice_ids, _ = self.clients.resource_registry.find_subjects(
RT.InstrumentDevice, PRED.hasOutputProduct, in_data_product_id,
True)
for instdevice_id in instdevice_ids:
log.debug(
"DataProcessManagementService:create_data_process instrument device_id assoc to the input data product of this data process: %s (L4-CI-SA-RQ-231)",
str(instdevice_id))
self._find_lookup_tables(instdevice_id, configuration)
instmodel_ids, _ = self.clients.resource_registry.find_objects(
instdevice_id, PRED.hasModel, RT.InstrumentModel, True)
for instmodel_id in instmodel_ids:
log.debug(
"DataProcessManagementService:create_data_process instmodel_id assoc to the instDevice: %s",
str(instmodel_id))
self._find_lookup_tables(instmodel_id, configuration)
#-------------------------------
# Create subscription from in_data_product, which should already be associated with a stream via the Data Producer
#-------------------------------
# # first - get the data producer associated with this IN data product
# log.debug("DataProcessManagementService:create_data_process - get the data producer associated with this IN data product")
# producer_ids, _ = self.clients.resource_registry.find_objects(in_data_product_id, PRED.hasDataProducer, RT.DataProducer, True)
# if not producer_ids:
# raise NotFound("No Data Producer created for this Data Product " + str(in_data_product_id))
# if len(producer_ids) != 1:
# raise BadRequest("Data Product should only have ONE Data Producers at this time" + str(in_data_product_id))
# in_product_producer = producer_ids[0]
# log.debug("DataProcessManagementService:create_data_process - get the data producer associated with this IN data product in_product_producer: " + str(in_product_producer))
# second - get the streams associated with this IN data products
self.in_stream_ids = []
for in_data_product_id in in_data_product_ids:
log.debug(
"DataProcessManagementService:create_data_process - get the stream associated with this IN data product"
)
stream_ids, _ = self.clients.resource_registry.find_objects(
in_data_product_id, PRED.hasStream, RT.Stream, True)
if not stream_ids:
raise NotFound("No Stream created for this IN Data Product " +
str(in_data_product_id))
if len(stream_ids) != 1:
raise BadRequest(
"IN Data Product should only have ONE stream at this time"
+ str(in_data_product_id))
log.debug(
"DataProcessManagementService:create_data_process - get the stream associated with this IN data product: %s in_stream_id: %s ",
str(in_data_product_id), str(stream_ids[0]))
self.in_stream_ids.append(stream_ids[0])
# Finally - create a subscription to the input stream
log.debug(
"DataProcessManagementService:create_data_process - Finally - create a subscription to the input stream"
)
in_data_product_obj = self.clients.data_product_management.read_data_product(
in_data_product_id)
query = StreamQuery(stream_ids=self.in_stream_ids)
#self.input_subscription_id = self.clients.pubsub_management.create_subscription(query=query, exchange_name=in_data_product_obj.name)
self.input_subscription_id = self.clients.pubsub_management.create_subscription(
query=query, exchange_name=data_process_name)
log.debug(
"DataProcessManagementService:create_data_process - Finally - create a subscription to the input stream input_subscription_id"
+ str(self.input_subscription_id))
# add the subscription id to the resource for clean up later
data_process_obj = self.clients.resource_registry.read(data_process_id)
data_process_obj.input_subscription_id = self.input_subscription_id
self.clients.resource_registry.update(data_process_obj)
procdef_ids, _ = self.clients.resource_registry.find_objects(
data_process_definition_id,
PRED.hasProcessDefinition,
RT.ProcessDefinition,
id_only=True)
if not procdef_ids:
raise BadRequest(
"Cannot find associated ProcessDefinition for DataProcessDefinition id=%s"
% data_process_definition_id)
process_definition_id = procdef_ids[0]
# Launch the transform process
log.debug(
"DataProcessManagementService:create_data_process - Launch the first transform process: "
)
log.debug(
"DataProcessManagementService:create_data_process - input_subscription_id: "
+ str(self.input_subscription_id))
log.debug(
"DataProcessManagementService:create_data_process - out_stream_id: "
+ str(self.output_stream_dict))
log.debug(
"DataProcessManagementService:create_data_process - process_definition_id: "
+ str(process_definition_id))
log.debug(
"DataProcessManagementService:create_data_process - data_process_id: "
+ str(data_process_id))
transform_id = self.clients.transform_management.create_transform(
name=data_process_id,
description=data_process_id,
in_subscription_id=self.input_subscription_id,
out_streams=self.output_stream_dict,
process_definition_id=process_definition_id,
configuration=configuration)
log.debug(
"DataProcessManagementService:create_data_process - transform_id: "
+ str(transform_id))
self.clients.resource_registry.create_association(
data_process_id, PRED.hasTransform, transform_id)
log.debug(
"DataProcessManagementService:create_data_process - Launch the first transform process transform_id"
+ str(transform_id))
# TODO: Flesh details of transform mgmt svc schedule method
# self.clients.transform_management_service.schedule_transform(transform_id)
return data_process_id
def build_packet(self, *args, **kwargs):
"""
Build and return a granule of data.
@param taxonomy the taxonomy of the granule
@data dictionary containing sample data.
@return granule suitable for publishing
"""
taxonomy_str = kwargs.get('taxonomy')
data = kwargs.get('data')
data_producer_id = kwargs.get('data_producer_id')
if not data_producer_id:
raise PacketFactoryException("data_producer_id parameter missing")
if not taxonomy_str:
raise PacketFactoryException("taxonomy parameter missing")
if not data:
raise PacketFactoryException("data parameter missing")
taxonomy = self._get_taxy_tool(taxonomy_str)
# the nick_names in the taxonomy:
nick_names = self._get_nick_names_from_taxonomy(taxonomy)
#
# TODO in general, how are groups (and the individual values
# belonging to the groups) to be determined?
#
# in this version, expect 'data' and 'coordinates' to be included in
# the taxonomy -- TODO the idea would be to be more general here?
##############################################################
# NOTE for the moment, using the flat data record dict 'rdt'
##############################################################
# if not 'data' in nick_names:
# raise PacketFactoryException("expected name 'data' in taxonomy")
# if not 'coordinates' in nick_names:
# raise PacketFactoryException("expected name 'coordinates' in taxonomy")
rdt = RecordDictionaryTool(taxonomy=taxonomy)
# data_rdt = RecordDictionaryTool(taxonomy=taxonomy)
# coordinates_rdt = RecordDictionaryTool(taxonomy=taxonomy)
#
# rdt['data'] = data_rdt
# rdt['coordinates'] = coordinates_rdt
# def is_coordinate(nick_name):
# # just an ad hoc check to determine which group the nick_names
# # belong to
# return nick_name in ['lat', 'lon', 'time', 'height']
# now, assign the values to the corresp record dicts:
for name, value in data.iteritems():
handle = -1
log.info("packetfactory: name: %s" % str(name))
if name in nick_names:
handle = taxonomy.get_handle(name)
log.info("packetfactory: handle: %s" % str(handle))
else:
handles = taxonomy.get_handles(name)
log.info("packetfactory: handles: %s" % str(handles))
if len(handles) == 1:
handle = handles.pop()
elif len(handles) > 1:
# TODO proper handling of this case
log.warn("Multiple handles found for '%s': %s" %
(name % handles))
if handle >= 0:
# ok, the nick_name has been found, either directly as a
# nick_name or via an alias; set value (using nick_name):
nick_name = taxonomy.get_nick_name(handle)
assert isinstance(value, list)
val = numpy.array(value)
# NOTE for the moment, using the flat data record dict 'rdt':
rdt[nick_name] = val
# if is_coordinate(nick_name):
# coordinates_rdt[nick_name] = val
# else:
# data_rdt[nick_name] = val
else:
# name not found.
# In the current tests this is happening with 'stream_id'
log.warning("No handle found for '%s'" % name)
log.debug("dictionary created: %s" % rdt.pretty_print())
return build_granule(data_producer_id=data_producer_id,
taxonomy=taxonomy,
record_dictionary=rdt)
def mark_bad_channel(self, ch_number):
log.debug("Marking %d as a bad channel", ch_number)
self._bad_channel_numbers.add(ch_number)
def send(self, data):
# send data from parent (coming from the instrumment) to tcp client
log.debug("DirectAccessServer.send(): data = " + str(data))
if self.server:
self.server.send(data)
class DataProcessManagementService(BaseDataProcessManagementService):
def on_init(self):
IonObject("Resource") # suppress pyflakes error
self.override_clients(self.clients)
def override_clients(self, new_clients):
"""
Replaces the service clients with a new set of them... and makes sure they go to the right places
"""
#shortcut names for the import sub-services
if hasattr(self.clients, "resource_registry"):
self.RR = self.clients.resource_registry
if hasattr(self.clients, "transform_management_service"):
self.TMS = self.clients.transform_management_service
#farm everything out to the impls
self.data_process = DataProcessImpl(self.clients)
def create_data_process_definition(self, data_process_definition=None):
result, _ = self.clients.resource_registry.find_resources(
RT.DataProcessDefinition, None, data_process_definition.name, True)
if result:
raise BadRequest(
"A data process definition named '%s' already exists" %
data_process_definition.name)
#todo: determine validation checks for a data process def
data_process_definition_id, version = self.clients.resource_registry.create(
data_process_definition)
#-------------------------------
# Process Definition
#-------------------------------
# Create the underlying process definition
process_definition = ProcessDefinition()
process_definition.name = data_process_definition.name
process_definition.description = data_process_definition.description
process_definition.executable = {
'module': data_process_definition.module,
'class': data_process_definition.class_name
}
process_definition_id = self.clients.process_dispatcher.create_process_definition(
process_definition=process_definition)
self.clients.resource_registry.create_association(
data_process_definition_id, PRED.hasProcessDefinition,
process_definition_id)
return data_process_definition_id
def update_data_process_definition(self, data_process_definition=None):
# TODO: If executable has changed, update underlying ProcessDefinition
# Overwrite DataProcessDefinition object
self.clients.resource_registry.update(data_process_definition)
def read_data_process_definition(self, data_process_definition_id=''):
# Read DataProcessDefinition object with _id matching id
log.debug("Reading DataProcessDefinition object id: %s" %
data_process_definition_id)
data_proc_def_obj = self.clients.resource_registry.read(
data_process_definition_id)
return data_proc_def_obj
def delete_data_process_definition(self, data_process_definition_id=''):
# Delete the data process
self.clients.resource_registry.delete(data_process_definition_id)
def find_data_process_definitions(self, filters=None):
"""
@param filters: dict of parameters to filter down
the list of possible data proc.
@retval
"""
#todo: add filtering
data_process_def_list, _ = self.clients.resource_registry.find_resources(
RT.DataProcessDefinition, None, None, True)
return data_process_def_list
def assign_input_stream_definition_to_data_process_definition(
self, stream_definition_id='', data_process_definition_id=''):
"""Connect the input stream with a data process definition
"""
# Verify that both ids are valid, RR will throw if not found
stream_definition_obj = self.clients.resource_registry.read(
stream_definition_id)
data_process_definition_obj = self.clients.resource_registry.read(
data_process_definition_id)
self.clients.resource_registry.create_association(
data_process_definition_id, PRED.hasInputStreamDefinition,
stream_definition_id)
def unassign_input_stream_definition_from_data_process_definition(
self, stream_definition_id='', data_process_definition_id=''):
"""
Disconnect the Data Product from the Data Producer
@param data_product_id str
@throws NotFound object with specified id does not exist
"""
# Remove the link between the Stream Definition resource and the Data Process Definition resource
associations = self.clients.resource_registry.find_associations(
data_process_definition_id,
PRED.hasInputStreamDefinition,
stream_definition_id,
id_only=True)
if not associations:
raise NotFound(
"No Input Stream Definitions associated with data process definition ID "
+ str(data_process_definition_id))
for association in associations:
self.clients.resource_registry.delete_association(association)
def assign_stream_definition_to_data_process_definition(
self, stream_definition_id='', data_process_definition_id=''):
"""Connect the output stream with a data process definition
"""
# Verify that both ids are valid, RR will throw if not found
stream_definition_obj = self.clients.resource_registry.read(
stream_definition_id)
data_process_definition_obj = self.clients.resource_registry.read(
data_process_definition_id)
self.clients.resource_registry.create_association(
data_process_definition_id, PRED.hasStreamDefinition,
stream_definition_id)
def unassign_stream_definition_from_data_process_definition(
self, stream_definition_id='', data_process_definition_id=''):
"""
Disconnect the Data Product from the Data Producer
@param data_product_id str
@throws NotFound object with specified id does not exist
"""
# Remove the link between the Stream Definition resource and the Data Process Definition resource
associations = self.clients.resource_registry.find_associations(
data_process_definition_id,
PRED.hasStreamDefinition,
stream_definition_id,
id_only=True)
if not associations:
raise NotFound(
"No Stream Definitions associated with data process definition ID "
+ str(data_process_definition_id))
for association in associations:
self.clients.resource_registry.delete_association(association)
# ------------------------------------------------------------------------------------------------
# Working with DataProcess
def create_data_process(self,
data_process_definition_id=None,
in_data_product_ids='',
out_data_products=None,
configuration=None):
"""
@param data_process_definition_id: Object with definition of the
transform to apply to the input data product
@param in_data_product_ids: ID of the input data products
@param out_data_products: list of IDs of the output data products
@retval data_process_id: ID of the newly created data process object
"""
inform = "Input Data Product: "+str(in_data_product_ids)+\
"Transformed by: "+str(data_process_definition_id)+\
"To create output Product: "+str(out_data_products)
log.debug("DataProcessManagementService:create_data_process()\n" +
inform)
if configuration is None:
configuration = {}
# Create and store a new DataProcess with the resource registry
log.debug(
"DataProcessManagementService:create_data_process - Create and store a new DataProcess with the resource registry"
)
data_process_def_obj = self.read_data_process_definition(
data_process_definition_id)
data_process_name = "process_" + data_process_def_obj.name \
+ time.ctime()
self.data_process = IonObject(RT.DataProcess, name=data_process_name)
data_process_id, version = self.clients.resource_registry.create(
self.data_process)
log.debug(
"DataProcessManagementService:create_data_process - Create and store a new DataProcess with the resource registry data_process_id: %s"
+ str(data_process_id))
# Register the data process instance as a data producer with DataAcquisitionMgmtSvc
#TODO: should this be outside this method? Called by orchestration?
data_producer_id = self.clients.data_acquisition_management.register_process(
data_process_id)
log.debug(
"DataProcessManagementService:create_data_process register process with DataAcquisitionMgmtSvc: data_producer_id: %s (L4-CI-SA-RQ-181)",
str(data_producer_id))
self.output_stream_dict = {}
#TODO: should this be outside this method? Called by orchestration?
if out_data_products is None:
raise BadRequest(
"Data Process must have output product(s) specified %s",
str(data_process_definition_id))
for name, out_data_product_id in out_data_products.iteritems():
# check that the product is not already associated with a producer
producer_ids, _ = self.clients.resource_registry.find_objects(
out_data_product_id, PRED.hasDataProducer, RT.DataProducer,
True)
if producer_ids:
raise BadRequest(
"Data Product should not already be associated to a DataProducer %s hasDataProducer %s",
str(data_process_id), str(producer_ids[0]))
#Assign each output Data Product to this producer resource
out_data_product_obj = self.clients.resource_registry.read(
out_data_product_id)
if not out_data_product_obj:
raise NotFound("Output Data Product %s does not exist" %
out_data_product_id)
# Associate with DataProcess: register as an output product for this process
log.debug(
"DataProcessManagementService:create_data_process link data process %s and output out data product: %s (L4-CI-SA-RQ-260)",
str(data_process_id), str(out_data_product_id))
self.clients.data_acquisition_management.assign_data_product(
data_process_id, out_data_product_id, create_stream=False)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.clients.resource_registry.find_objects(
out_data_product_id, PRED.hasStream, RT.Stream, True)
log.debug(
"DataProcessManagementService:create_data_process retrieve out data prod streams: %s",
str(stream_ids))
if not stream_ids:
raise NotFound("No Stream created for output Data Product " +
str(out_data_product_id))
if len(stream_ids) != 1:
raise BadRequest(
"Data Product should only have ONE stream at this time" +
str(out_data_product_id))
self.output_stream_dict[name] = stream_ids[0]
log.debug(
"DataProcessManagementService:create_data_process -Register the data process instance as a data producer with DataAcquisitionMgmtSvc, then retrieve the id of the OUTPUT stream out_stream_id: "
+ str(self.output_stream_dict[name]))
# Associate with dataProcess
self.clients.resource_registry.create_association(
data_process_id, PRED.hasProcessDefinition,
data_process_definition_id)
#check if data process has lookup tables attached
self._find_lookup_tables(data_process_definition_id, configuration)
#Todo: currently this is handled explicitly after creating the data product, that code then calls DMAS:assign_data_product
log.debug(
"DataProcessManagementService:create_data_process associate data process workflows with source data products %s hasInputProducts %s (L4-CI-SA-RQ-260)",
str(data_process_id), str(in_data_product_ids))
for in_data_product_id in in_data_product_ids:
self.clients.resource_registry.create_association(
data_process_id, PRED.hasInputProduct, in_data_product_id)
#check if in data product is attached to an instrument, check instrumentDevice and InstrumentModel for lookup table attachments
instdevice_ids, _ = self.clients.resource_registry.find_subjects(
RT.InstrumentDevice, PRED.hasOutputProduct, in_data_product_id,
True)
for instdevice_id in instdevice_ids:
log.debug(
"DataProcessManagementService:create_data_process instrument device_id assoc to the input data product of this data process: %s (L4-CI-SA-RQ-231)",
str(instdevice_id))
self._find_lookup_tables(instdevice_id, configuration)
instmodel_ids, _ = self.clients.resource_registry.find_objects(
instdevice_id, PRED.hasModel, RT.InstrumentModel, True)
for instmodel_id in instmodel_ids:
log.debug(
"DataProcessManagementService:create_data_process instmodel_id assoc to the instDevice: %s",
str(instmodel_id))
self._find_lookup_tables(instmodel_id, configuration)
#-------------------------------
# Create subscription from in_data_product, which should already be associated with a stream via the Data Producer
#-------------------------------
# # first - get the data producer associated with this IN data product
# log.debug("DataProcessManagementService:create_data_process - get the data producer associated with this IN data product")
# producer_ids, _ = self.clients.resource_registry.find_objects(in_data_product_id, PRED.hasDataProducer, RT.DataProducer, True)
# if not producer_ids:
# raise NotFound("No Data Producer created for this Data Product " + str(in_data_product_id))
# if len(producer_ids) != 1:
# raise BadRequest("Data Product should only have ONE Data Producers at this time" + str(in_data_product_id))
# in_product_producer = producer_ids[0]
# log.debug("DataProcessManagementService:create_data_process - get the data producer associated with this IN data product in_product_producer: " + str(in_product_producer))
# second - get the streams associated with this IN data products
self.in_stream_ids = []
for in_data_product_id in in_data_product_ids:
log.debug(
"DataProcessManagementService:create_data_process - get the stream associated with this IN data product"
)
stream_ids, _ = self.clients.resource_registry.find_objects(
in_data_product_id, PRED.hasStream, RT.Stream, True)
if not stream_ids:
raise NotFound("No Stream created for this IN Data Product " +
str(in_data_product_id))
if len(stream_ids) != 1:
raise BadRequest(
"IN Data Product should only have ONE stream at this time"
+ str(in_data_product_id))
log.debug(
"DataProcessManagementService:create_data_process - get the stream associated with this IN data product: %s in_stream_id: %s ",
str(in_data_product_id), str(stream_ids[0]))
self.in_stream_ids.append(stream_ids[0])
# Finally - create a subscription to the input stream
log.debug(
"DataProcessManagementService:create_data_process - Finally - create a subscription to the input stream"
)
in_data_product_obj = self.clients.data_product_management.read_data_product(
in_data_product_id)
query = StreamQuery(stream_ids=self.in_stream_ids)
#self.input_subscription_id = self.clients.pubsub_management.create_subscription(query=query, exchange_name=in_data_product_obj.name)
self.input_subscription_id = self.clients.pubsub_management.create_subscription(
query=query, exchange_name=data_process_name)
log.debug(
"DataProcessManagementService:create_data_process - Finally - create a subscription to the input stream input_subscription_id"
+ str(self.input_subscription_id))
# add the subscription id to the resource for clean up later
data_process_obj = self.clients.resource_registry.read(data_process_id)
data_process_obj.input_subscription_id = self.input_subscription_id
self.clients.resource_registry.update(data_process_obj)
procdef_ids, _ = self.clients.resource_registry.find_objects(
data_process_definition_id,
PRED.hasProcessDefinition,
RT.ProcessDefinition,
id_only=True)
if not procdef_ids:
raise BadRequest(
"Cannot find associated ProcessDefinition for DataProcessDefinition id=%s"
% data_process_definition_id)
process_definition_id = procdef_ids[0]
# Launch the transform process
log.debug(
"DataProcessManagementService:create_data_process - Launch the first transform process: "
)
log.debug(
"DataProcessManagementService:create_data_process - input_subscription_id: "
+ str(self.input_subscription_id))
log.debug(
"DataProcessManagementService:create_data_process - out_stream_id: "
+ str(self.output_stream_dict))
log.debug(
"DataProcessManagementService:create_data_process - process_definition_id: "
+ str(process_definition_id))
log.debug(
"DataProcessManagementService:create_data_process - data_process_id: "
+ str(data_process_id))
transform_id = self.clients.transform_management.create_transform(
name=data_process_id,
description=data_process_id,
in_subscription_id=self.input_subscription_id,
out_streams=self.output_stream_dict,
process_definition_id=process_definition_id,
configuration=configuration)
log.debug(
"DataProcessManagementService:create_data_process - transform_id: "
+ str(transform_id))
self.clients.resource_registry.create_association(
data_process_id, PRED.hasTransform, transform_id)
log.debug(
"DataProcessManagementService:create_data_process - Launch the first transform process transform_id"
+ str(transform_id))
# TODO: Flesh details of transform mgmt svc schedule method
# self.clients.transform_management_service.schedule_transform(transform_id)
return data_process_id
def _find_lookup_tables(self, resource_id="", configuration=None):
#check if resource has lookup tables attached
attachment_objs, _ = self.clients.resource_registry.find_objects(
resource_id, PRED.hasAttachment, RT.Attachment, False)
for attachment_obj in attachment_objs:
log.debug(
"DataProcessManagementService:_find_lookup_tables attachment %s",
str(attachment_obj))
words = set(attachment_obj.keywords)
if 'DataProcessInput' in words:
configuration[attachment_obj.name] = attachment_obj.content
log.debug(
"DataProcessManagementService:_find_lookup_tables lookup table found in attachment %s",
attachment_obj.name)
else:
log.debug(
"DataProcessManagementService:_find_lookup_tables NO lookup table in attachment %s",
attachment_obj.name)
def update_data_process(self, ):
#todo: What are valid ways to update a data process?.
return
def read_data_process(self, data_process_id=""):
# Read DataProcess object with _id matching id
log.debug("Reading DataProcess object id: %s" % data_process_id)
data_proc_obj = self.clients.resource_registry.read(data_process_id)
if not data_proc_obj:
raise NotFound("DataProcess %s does not exist" % data_process_id)
return data_proc_obj
def delete_data_process(self, data_process_id=""):
# Delete the specified DataProcessDefinition object
data_process_obj = self.clients.resource_registry.read(data_process_id)
if data_process_obj is None:
raise NotFound("Data Process %s does not exist" % data_process_id)
# Deactivates and deletes the input subscription
# todo: create did not activate the subscription, should Transform deactivate?
try:
self.clients.pubsub_management.deactivate_subscription(
data_process_obj.input_subscription_id)
log.debug(
"DataProcessManagementService:delete_data_process delete subscription"
)
self.clients.pubsub_management.delete_subscription(
data_process_obj.input_subscription_id)
except BadRequest, e:
#May not have activated the subscription so just skip - had to add this to get AS integration tests to pass - probably should be fixed
pass
self.clients.data_acquisition_management.unregister_process(
data_process_id)
# Delete the output stream, but not the output product
out_products, _ = self.clients.resource_registry.find_objects(
data_process_id, PRED.hasOutputProduct, RT.DataProduct, True)
if len(out_products) < 1:
raise NotFound(
'The the Data Process %s output product cannot be located.' %
str(data_process_id))
# outprod_associations = self.clients.resource_registry.find_associations(data_process_id, PRED.hasOutputProduct)
# for outprod_association in outprod_associations:
# log.debug("DataProcessManagementService:delete_data_process delete outprod assoc")
# self.clients.resource_registry.delete_association(outprod_association)
for out_product in out_products:
out_streams, _ = self.clients.resource_registry.find_objects(
out_product, PRED.hasStream, RT.Stream, True)
for out_stream in out_streams:
outprod_associations = self.clients.resource_registry.find_associations(
data_process_id, PRED.hasOutputProduct, out_product)
log.debug(
"DataProcessManagementService:delete_data_process delete outprod assoc"
)
self.clients.resource_registry.delete_association(
outprod_associations[0])
# delete the stream def assoc to the stream
streamdef_list = self.clients.resource_registry.find_objects(
subject=out_stream,
predicate=PRED.hasStreamDefinition,
id_only=True)
# delete the associations link first
streamdef_assocs = self.clients.resource_registry.find_associations(
out_stream, PRED.hasStreamDefinition)
for streamdef_assoc in streamdef_assocs:
log.debug(
"DataProcessManagementService:delete_data_process delete streamdef assocs"
)
self.clients.resource_registry.delete_association(
streamdef_assoc)
# for streamdef in streamdef_list:
# self.clients.resource_registry.delete_association(streamdef)
# delete the connector first
stream_associations = self.clients.resource_registry.find_associations(
out_product, PRED.hasStream)
for stream_association in stream_associations:
log.debug(
"DataProcessManagementService:delete_data_process delete stream assocs"
)
self.clients.resource_registry.delete_association(
stream_association)
log.debug(
"DataProcessManagementService:delete_data_process delete outstreams"
)
self.clients.pubsub_management.delete_stream(out_stream)
# Delete the input products link
inprod_associations = self.clients.resource_registry.find_associations(
data_process_id, PRED.hasInputProduct)
if len(inprod_associations) < 1:
raise NotFound(
'The the Data Process %s input product link cannot be located.'
% str(data_process_id))
for inprod_association in inprod_associations:
log.debug(
"DataProcessManagementService:delete_data_process delete inprod assocs"
)
self.clients.resource_registry.delete_association(
inprod_association)
# Delete the transform
transforms, _ = self.clients.resource_registry.find_objects(
data_process_id, PRED.hasTransform, RT.Transform, True)
if len(transforms) < 1:
raise NotFound(
'There is no Transform linked to this Data Process %s' %
str(data_process_id))
# delete the transform associations link first
transform_assocs = self.clients.resource_registry.find_associations(
data_process_id, PRED.hasTransform)
for transform_assoc in transform_assocs:
log.debug(
"DataProcessManagementService:delete_data_process delete transform assocs"
)
self.clients.resource_registry.delete_association(transform_assoc)
for transform in transforms:
log.debug(
"DataProcessManagementService:delete_data_process delete transform"
)
self.clients.transform_management.delete_transform(transform)
# Delete the assoc with Data Process Definition
data_process_defs = self.clients.resource_registry.find_associations(
data_process_id, PRED.hasProcessDefinition, None)
if len(data_process_defs) < 1:
raise NotFound(
'The the Data Process %s is not linked to a Data Process Definition.'
% str(data_process_id))
for data_process_def in data_process_defs:
log.debug(
"DataProcessManagementService:delete_data_process data_process_def transform"
)
self.clients.resource_registry.delete_association(data_process_def)
# other_assocs = self.clients.resource_registry.find_associations(subject=data_process_id)
# for other_assoc in other_assocs:
# log.debug("DataProcessManagementService:delete_data_process delete other assoc")
# self.clients.resource_registry.delete_association(other_assoc)
# Delete the data process
log.debug(
"DataProcessManagementService:delete_data_process the data_process"
)
self.clients.resource_registry.delete(data_process_id)
return
def send(self, data):
# send data from parent to tcp client
log.debug("TcpServer.send(): data = " + str(data))
if self.server_ready_to_send:
self._write(data)
class PolicyManagementService(BasePolicyManagementService):
def __init__(self, *args, **kwargs):
BasePolicyManagementService.__init__(self,*args,**kwargs)
self.event_pub = None # For unit tests
def on_start(self):
self.event_pub = EventPublisher()
self.policy_event_subscriber = ProcessEventSubscriber(event_type="ResourceModifiedEvent", origin_type="Policy", callback=self._policy_event_callback, process=self)
self._process.add_endpoint(self.policy_event_subscriber)
"""Provides the interface to define and manage policy and a repository to store and retrieve policy
and templates for policy definitions, aka attribute authority.
@see https://confluence.oceanobservatories.org/display/syseng/CIAD+COI+OV+Policy+Management+Service
"""
def create_resource_access_policy(self, resource_id='', policy_name='', description='', policy_rule=''):
"""Helper operation for creating an access policy for a specific resource. The id string returned
is the internal id by which Policy will be identified in the data store.
@param resource_id str
@param policy_name str
@param description str
@param policy_rule str
@retval policy_id str
@throws BadRequest If any of the paramaters are not set.
"""
if not resource_id:
raise BadRequest("The resource_id parameter is missing")
if not policy_name:
raise BadRequest("The policy_name parameter is missing")
if not description:
raise BadRequest("The description parameter is missing")
if not policy_rule:
raise BadRequest("The policy_rule parameter is missing")
resource_policy_obj = IonObject(OT.ResourceAccessPolicy, policy_rule=policy_rule, resource_id=resource_id)
policy_obj = IonObject(RT.Policy, name=policy_name, description=description, policy_type=resource_policy_obj)
policy_id = self.create_policy(policy_obj)
self._add_resource_policy(resource_id, policy_id, publish_event=False)
return policy_id
def create_service_access_policy(self, service_name='', policy_name='', description='', policy_rule=''):
"""Helper operation for creating an access policy for a specific service. The id string returned
is the internal id by which Policy will be identified in the data store.
@param service_name str
@param policy_name str
@param description str
@param policy_rule str
@retval policy_id str
@throws BadRequest If any of the paramaters are not set.
"""
if not service_name:
raise BadRequest("The service_name parameter is missing")
if not policy_name:
raise BadRequest("The policy_name parameter is missing")
if not description:
raise BadRequest("The description parameter is missing")
if not policy_rule:
raise BadRequest("The policy_rule parameter is missing")
service_policy_obj = IonObject(OT.ServiceAccessPolicy, policy_rule=policy_rule, service_name=service_name)
policy_obj = IonObject(RT.Policy, name=policy_name, description=description, policy_type=service_policy_obj)
return self.create_policy(policy_obj)
def create_common_service_access_policy(self, policy_name='', description='', policy_rule=''):
"""Helper operation for creating a service access policy common to all services. The id string returned
is the internal id by which Policy will be identified in the data store.
@param policy_name str
@param description str
@param policy_rule str
@retval policy_id str
@throws BadRequest If any of the paramaters are not set.
"""
if not policy_name:
raise BadRequest("The policy_name parameter is missing")
if not description:
raise BadRequest("The description parameter is missing")
if not policy_rule:
raise BadRequest("The policy_rule parameter is missing")
service_policy_obj = IonObject(OT.CommonServiceAccessPolicy, policy_rule=policy_rule)
policy_obj = IonObject(RT.Policy, name=policy_name, description=description, policy_type=service_policy_obj)
return self.create_policy(policy_obj)
def add_process_operation_precondition_policy(self, process_name='', op='', policy_content=''):
"""Helper operation for adding a precondition policy for a specific process operation; could be a service or agent.
The id string returned is the internal id by which Policy will be identified in the data store. The precondition
method must return a tuple (boolean, string).
@param process_name str
@param op str
@param policy_content str
@retval policy_id str
@throws BadRequest If any of the parameters are not set.
"""
if not process_name:
raise BadRequest("The process_name parameter is missing")
if not op:
raise BadRequest("The op parameter is missing")
if not policy_content:
raise BadRequest("The policy_content parameter is missing")
policy_name = process_name + "_" + op + "_Precondition_Policies"
policies,_ = self.clients.resource_registry.find_resources(restype=RT.Policy, name=policy_name)
if policies:
#Update existing policy by adding to list
if len(policies) > 1:
raise Inconsistent('There should only be one Policy object per process_name operation')
if policies[0].policy_type.op != op or policies[0].policy_type.type_ != OT.ProcessOperationPreconditionPolicy:
raise Inconsistent('There Policy object %s does not match the requested process operation %s: %s' % ( policies[0].name, process_name, op ))
policies[0].policy_type.preconditions.append(policy_content)
self.update_policy(policies[0])
return policies[0]._id
else:
#Create a new policy object
op_policy_obj = IonObject(OT.ProcessOperationPreconditionPolicy, process_name=process_name, op=op)
op_policy_obj.preconditions.append(policy_content)
policy_obj = IonObject(RT.Policy, name=policy_name, policy_type=op_policy_obj, description='List of operation precondition policies for ' + process_name)
return self.create_policy(policy_obj)
def create_policy(self, policy=None):
"""Persists the provided Policy object The id string returned
is the internal id by which Policy will be identified in the data store.
@param policy Policy
@retval policy_id str
@throws BadRequest if object passed has _id or _rev attribute
"""
if not policy:
raise BadRequest("The policy parameter is missing")
if not is_basic_identifier(policy.name):
raise BadRequest("The policy name '%s' can only contain alphanumeric and underscore characters" % policy.name)
try:
#If there is a policy_rule field then try to add the policy name and decription to the rule text
if hasattr(policy.policy_type, 'policy_rule'):
policy.policy_type.policy_rule = policy.policy_type.policy_rule % (policy.name, policy.description)
except Exception, e:
raise Inconsistent("Missing the elements in the policy rule to set the description: " + e.message)
policy_id, version = self.clients.resource_registry.create(policy)
log.debug('Policy created: ' + policy.name)
return policy_id
def _control_flow(self):
"""
Main process thread of execution method.
This method is run inside a greenlet and exists for each ION process. Listeners
attached to the process, either RPC Servers or Subscribers, synchronize their calls
by placing future calls into the queue by calling _routing_call. This is all done
automatically for you by the Container's Process Manager.
This method blocks until there are calls to be made in the synchronized queue, and
then calls from within this greenlet. Any exception raised is caught and re-raised
in the greenlet that originally scheduled the call. If successful, the AsyncResult
created at scheduling time is set with the result of the call.
"""
if self.name:
svc_name = "unnamed-service"
if self.service is not None and hasattr(self.service, 'name'):
svc_name = self.service.name
threading.current_thread().name = "%s-%s-ctrl" % (svc_name,
self.name)
self._ready_control.set()
for calltuple in self._ctrl_queue:
calling_gl, ar, call, callargs, callkwargs, context = calltuple
log.debug("control_flow making call: %s %s %s (has context: %s)",
call, callargs, callkwargs, context is not None)
res = None
start_proc_time = int(get_ion_ts())
# check context for expiration
if context is not None and 'reply-by' in context:
if start_proc_time >= int(context['reply-by']):
log.info(
"control_flow: attempting to process message already exceeding reply-by, ignore"
)
# raise a timeout in the calling thread to allow endpoints to continue processing
e = IonTimeout(
"Reply-by time has already occurred (reply-by: %s, op start time: %s)"
% (context['reply-by'], start_proc_time))
calling_gl.kill(exception=e, block=False)
continue
# also check ar if it is set, if it is, that means it is cancelled
if ar.ready():
log.info(
"control_flow: attempting to process message that has been cancelled, ignore"
)
continue
try:
with self.service.push_context(context):
with self.service.container.context.push_context(context):
self._ctrl_current = ar
res = call(*callargs, **callkwargs)
except OperationInterruptedException:
# endpoint layer takes care of response as it's the one that caused this
log.debug("Operation interrupted")
pass
except Exception as e:
# raise the exception in the calling greenlet, and don't
# wait for it to die - it's likely not going to do so.
# try decorating the args of the exception with the true traceback
# this should be reported by ThreadManager._child_failed
exc = PyonThreadTraceback(
"IonProcessThread _control_flow caught an exception (call: %s, *args %s, **kwargs %s, context %s)\nTrue traceback captured by IonProcessThread' _control_flow:\n\n%s"
% (call, callargs, callkwargs, context,
traceback.format_exc()))
e.args = e.args + (exc, )
# HACK HACK HACK
# we know that we only handle TypeError and IonException derived things, so only forward those if appropriate
if isinstance(e, (TypeError, IonException)):
calling_gl.kill(exception=e, block=False)
else:
# otherwise, swallow/record/report and hopefully we can continue on our way
self._errors.append(
(call, callargs, callkwargs, context, e, exc))
log.warn(exc)
log.warn("Attempting to continue...")
# have to raise something friendlier on the client side
calling_gl.kill(exception=ContainerError(str(exc)),
block=False)
finally:
proc_time = int(get_ion_ts()) - start_proc_time
self._proc_time += proc_time
self._ctrl_current = None
ar.set(res)
def _handler(self):
"The actual handler to which the client is connected. Must be implemented in any server that inherits from this class"
log.debug("TcpServer._handler(): not implemented")
raise NotImplementedException('TcpServer._handler() not implemented.')
def start(self):
log.debug("AppManager starting ...")
def stop(self):
log.debug("SFlowManager.stop")
if self._gl_counter:
self._gl_counter.kill()
def __init__(self, *args, **kwargs):
log.debug('GovernanceDispatcher.__init__()')
def stop(self):
log.debug("AppManager stopping ...")
# Stop apps in reverse order of startup
for appdef in reversed(self.apps):
self.stop_app(appdef)
log.debug("AppManager stopped, OK.")
def start(container, starttype, app_definition, config):
log.debug("Hello app started")
return (None, None)
except Exception, ex:
log.exception("Appl %s start from processapp failed" %
appdef.name)
else:
# Case 2: Appdef contains full app start params
modpath = appdef.mod
try:
mod = named_any(modpath)
appdef._mod_loaded = mod
# Start the app
supid, state = mod.start(self.container, START_PERMANENT,
appdef, config)
appdef._supid = supid
appdef._state = state
log.debug("App '%s' started. Root sup-id=%s" %
(appdef.name, supid))
self.apps.append(appdef)
except Exception, ex:
log.exception("Appl %s start from appdef failed" % appdef.name)
def stop_app(self, appdef):
log.debug("App '%s' stopping" % appdef.name)
try:
if '_mod_loaded' in appdef:
appdef._mod_loaded.stop(self.container, appdef._state)
except Exception, ex:
log.exception("Application %s stop failed" % appdef.name)
def on_quit(self, *args, **kwargs):
log.debug("Hello service quit")
self.container.governance_controller.unregister_process_operation_precondition(
self, 'noop', self.deny_noop)
def stop(container, state):
log.debug("Hello app stopped")
def on_init(self, *args, **kwargs):
log.debug("Hello service init. Self.id=%s" % self.id)
self.container.governance_controller.register_process_operation_precondition(
self, 'noop', self.deny_noop)
def hello(self, text=''):
log.debug("In hello_service.hello. Text=%s" % text)
return "BACK:%s" % text
def _stop_process(self, data_process):
log.debug("stopping data process '%s'" % data_process.process_id)
pid = data_process.process_id
self.clients.process_dispatcher.cancel_process(pid)
def on_start(self, *args, **kwargs):
log.debug("Hello service start")
def create_data_product_resource_full(self):
geospatial_bounds = {
"geospatial_latitude_limit_north": 1.0,
"geospatial_latitude_limit_south": 1.0,
"geospatial_longitude_limit_east": 1.0,
"geospatial_longitude_limit_west": 1.0,
"geospatial_vertical_min": 1.0,
"geospatial_vertical_max": 1.0
}
#create a paramter dictionary the a stream definition for this data product
DICTIONARY_NAME = 'ctd_parsed_param_dict'
response = self.test_app.get(
'/ion-service/resource_registry/find_resources?name=' +
convert_unicode(DICTIONARY_NAME) +
'&id_only=True&user_id=123abc456')
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
param_dictionary_id = convert_unicode(response_data[0])
test = str(param_dictionary_id[0])
log.debug(
'create_data_product_resource_full param_dictionary_id: %s',
param_dictionary_id)
log.debug('create_data_product_resource_full test: %s', test)
response = self.test_app.get(
'/ion-service/pubsub_management/create_stream_definition?name=CTDData¶meter_dictionary_id='
+ convert_unicode(param_dictionary_id[0]))
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
log.debug('create_data_product_resource_full response_data: %s',
response_data)
stream_definition_id = convert_unicode(response_data)
log.debug(
'create_data_product_resource_full stream_definition_id: %s',
stream_definition_id)
data_product_create_request = {
"serviceRequest": {
"serviceName": "data_product_management",
"serviceOp": "create_data_product",
"params": {
"data_product": {
"type_": "DataProduct",
"lcstate": "DRAFT",
"description": DATA_PRODUCT_DESCRIPTION,
"name": DATA_PRODUCT_NAME,
'geospatial_bounds': geospatial_bounds,
'temporal_domain': TEMPORAL_DOMAIN
},
"stream_definition_id": stream_definition_id
}
}
}
response = self.test_app.post(
'/ion-service/data_product_management/create_data_product',
{'payload': simplejson.dumps(data_product_create_request)})
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
log.debug(
'create_data_product_resource_full create_data_product response_data: %s',
response_data)
#self.assertEqual(len(response_data), 2 )
data_product_id = convert_unicode(response_data)
return data_product_id
def incoming(self, invocation):
log.debug("EncodeInterceptor.incoming: %s", invocation)
log.debug("Pre-transform: %s", invocation.message)
invocation.message = msgpack.unpackb(invocation.message, object_hook=decode_ion, use_list=1)
log.debug("Post-transform: %s", invocation.message)
return invocation