def create_stream_definition(self,
name='',
parameter_dictionary=None,
parameter_dictionary_id='',
stream_type='',
description='',
available_fields=None,
stream_configuration=None):
parameter_dictionary = parameter_dictionary or {}
stream_configuration = stream_configuration or {}
existing = self.clients.resource_registry.find_resources(
restype=RT.StreamDefinition, name=name, id_only=True)[0]
if name and existing:
if parameter_dictionary_id:
pdict_ids, _ = self.clients.resource_registry.find_objects(
subject=existing[0],
predicate=PRED.hasParameterDictionary,
id_only=True)
if pdict_ids and parameter_dictionary_id == pdict_ids[0]:
return existing[0]
else:
raise Conflict(
'StreamDefinition with the specified name already exists. (%s)'
% name)
stream_def = self.read_stream_definition(existing[0])
if self._compare_pdicts(parameter_dictionary,
stream_def.parameter_dictionary):
return existing[0]
raise Conflict(
'StreamDefinition with the specified name already exists. (%s)'
% name)
name = name or create_unique_identifier()
stream_definition = StreamDefinition(
parameter_dictionary=parameter_dictionary,
stream_type=stream_type,
name=name,
description=description,
available_fields=available_fields,
stream_configuration=stream_configuration)
stream_definition_id, _ = self.clients.resource_registry.create(
stream_definition)
if parameter_dictionary_id:
self._associate_pdict_with_definition(parameter_dictionary_id,
stream_definition_id)
return stream_definition_id
def create_parameter_dictionary(self,
name='',
parameter_context_ids=None,
temporal_context='',
description=''):
res, _ = self.clients.resource_registry.find_resources(
restype=RT.ParameterDictionary, name=name, id_only=True)
if len(res):
context_ids, _ = self.clients.resource_registry.find_objects(
subject=res[0],
predicate=PRED.hasParameterContext,
id_only=True)
context_ids.sort()
parameter_context_ids.sort()
if context_ids == parameter_context_ids:
return res[0]
else:
raise Conflict(
'A parameter dictionary with name %s already exists and has a different definition'
% name)
validate_true(name, 'Name field may not be empty.')
parameter_context_ids = parameter_context_ids or []
pd_res = ParameterDictionaryResource(name=name,
temporal_context=temporal_context,
description=description)
pd_res_id, ver = self.clients.resource_registry.create(pd_res)
for pc_id in parameter_context_ids:
self._link_pcr_to_pdr(pc_id, pd_res_id)
return pd_res_id
def insert(self, target):
""" insert one or many entries. must have _id and must not have _rev
if target is a list, docs are updated with _rev, but not if target is dict
(side effect of underlying library)
returns tuple (success, _id, Exception) when target is a dictionary
returns list of tuples when target is a list
"""
if not self._check_attr(target, '_rev', False):
raise Conflict('cannot insert document with revision')
if not self._check_attr(target, '_id', True):
raise BadRequest('inserted documents must have _id')
if isinstance(target, list):
return self._db.update(target)
else:
id = target['_id']
try:
self._db.create(target)
return True, id, None
except ResourceConflict, e:
return False, id, BadRequest(
'Object with id %s already exists' % id)
except Exception, e:
return False, id, self._wrap_error('insert', id, e)
def update_doc(self, doc, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
if '_id' not in doc:
raise BadRequest("Doc must have '_id'")
if '_rev' not in doc:
raise BadRequest("Doc must have '_rev'")
try:
datastore_dict = self.root[datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name + ' does not exist.')
try:
object_id = doc["_id"]
# Find the next doc version
version_counter_key = '__' + object_id + '_version_counter'
baseVersion = doc["_rev"]
version_counter = datastore_dict[version_counter_key] + 1
if baseVersion != str(version_counter - 1):
raise Conflict('Object not based on most current version')
except KeyError:
raise BadRequest("Object missing required _id and/or _rev values")
log.debug('Saving new version of object %s/%s' % (datastore_name, doc["_id"]))
doc["_rev"] = str(version_counter)
# Overwrite HEAD and version counter dicts, add new version dict
datastore_dict[object_id] = doc
datastore_dict[version_counter_key] = version_counter
datastore_dict[object_id + '_version_' + str(version_counter)] = doc
res = [object_id, str(version_counter)]
log.debug('Update result: %s' % str(res))
return res
def signon(self, certificate='', ignore_date_range=False):
log.debug("Signon with certificate:\n%s" % certificate)
# Check the certificate is currently valid
if not ignore_date_range:
if not self.authentication.is_certificate_within_date_range(certificate):
raise BadRequest("Certificate expired or not yet valid")
# Extract subject line
attributes = self.authentication.decode_certificate_string(certificate)
subject = attributes["subject"]
valid_until_str = attributes["not_valid_after"]
log.debug("Signon request for subject %s with string valid_until %s" % (subject, valid_until_str))
valid_until_tuple = time.strptime(valid_until_str, "%b %d %H:%M:%S %Y %Z")
valid_until = str(int(time.mktime(valid_until_tuple)) * 1000)
# Look for matching UserCredentials object
objects, assocs = self.clients.resource_registry.find_resources(RT.UserCredentials, None, subject, True)
if len(objects) > 1:
raise Conflict("More than one UserCredentials object was found for subject %s" % subject)
if len(objects) == 1:
log.debug("Signon known subject %s" % (subject))
# Known user, get ActorIdentity object
user_credentials_id = objects[0]
subjects, assocs = self.clients.resource_registry.find_subjects(RT.ActorIdentity, PRED.hasCredentials, user_credentials_id)
if len(subjects) == 0:
raise Conflict("ActorIdentity object with subject %s was previously created but is not associated with a ActorIdentity object" % subject)
actor_id = subjects[0]._id
# Find associated UserInfo
registered = True
try:
self.find_user_info_by_id(actor_id)
except NotFound:
registered = False
log.debug("Signon returning actor_id, valid_until, registered: %s, %s, %s" % (actor_id, valid_until, str(registered)))
return actor_id, valid_until, registered
else:
log.debug("Signon new subject %s" % (subject))
# New user. Create ActorIdentity and UserCredentials
actor_name = "Identity for %s" % subject
actor_identity = IonObject("ActorIdentity", name=actor_name)
actor_id = self.create_actor_identity(actor_identity)
user_credentials = IonObject("UserCredentials", name=subject, description="Default credentials for %s" % subject)
self.register_user_credentials(actor_id, user_credentials)
log.debug("Signon returning actor_id, valid_until, registered: %s, %s, False" % (actor_id, valid_until))
return actor_id, valid_until, False
def apply_remote_config(system_cfg):
from pyon.core.bootstrap import get_sys_name
from pyon.core.exception import Conflict
from pyon.ion.directory_standalone import DirectoryStandalone
directory = DirectoryStandalone(sysname=get_sys_name(), config=system_cfg)
de = directory.lookup("/Config/CFG")
if not de:
raise Conflict("Expected /Config/CFG in directory. Correct Org??")
apply_configuration(system_cfg, de)
def create_user_info(self, actor_id="", user_info=None):
# Ensure UserInfo association does not already exist
objects, assocs = self.clients.resource_registry.find_objects(actor_id, PRED.hasInfo, RT.UserInfo)
if objects:
raise Conflict("UserInfo already exists for user id %s" % (actor_id))
# Create UserInfo object
user_info_id, version = self.clients.resource_registry.create(user_info)
# Create association with user identity object
self.clients.resource_registry.create_association(actor_id, PRED.hasInfo, user_info_id)
return user_info_id
def update_doc(self, doc, datastore_name=""):
ds, datastore_name = self._get_datastore(datastore_name)
if '_id' not in doc:
raise BadRequest("Doc must have '_id'")
if '_rev' not in doc:
raise BadRequest("Doc must have '_rev'")
log.debug('update doc contents: %s', doc)
try:
res = ds.save(doc)
except ResourceConflict:
raise Conflict('Object not based on most current version')
log.debug('Update result: %s', str(res))
id, version = res
return (id, version)
def unregister_user_credentials(self, actor_id='', credentials_name=''):
# Read UserCredentials
objects, matches = self.clients.resource_registry.find_resources(RT.UserCredentials, None, credentials_name, id_only=False)
if not objects or len(objects) == 0:
raise NotFound("UserCredentials %s does not exist" % credentials_name)
if len(objects) > 1:
raise Conflict("Multiple UserCredentials objects found for subject %s" % credentials_name)
user_credentials_id = objects[0]._id
# Find and break association with ActorIdentity
assocs = self.clients.resource_registry.find_associations(actor_id, PRED.hasCredentials, user_credentials_id)
if not assocs or len(assocs) == 0:
raise NotFound("ActorIdentity to UserCredentials association for user id %s to credential %s does not exist" % (actor_id, credentials_name))
association_id = assocs[0]._id
self.clients.resource_registry.delete_association(association_id)
# Delete the UserCredentials
self.clients.resource_registry.delete(user_credentials_id)
def get_coverage_function(self, parameter_function):
func = None
if parameter_function.function_type == PFT.PYTHON:
func = PythonFunction(name=parameter_function.name,
owner=parameter_function.owner,
func_name=parameter_function.function,
arg_list=parameter_function.args,
kwarg_map=None,
param_map=None,
egg_uri=parameter_function.egg_uri)
elif parameter_function.function_type == PFT.NUMEXPR:
func = NumexprFunction(name=parameter_function.name,
expression=parameter_function.function,
arg_list=parameter_function.args)
if not isinstance(func, AbstractFunction):
raise Conflict("Incompatible parameter function loaded: %s" %
parameter_function._id)
return func
def load_parameter_function(self, row):
name = row['Name']
ftype = row['Function Type']
func_expr = row['Function']
owner = row['Owner']
args = ast.literal_eval(row['Args'])
#kwargs = row['Kwargs']
descr = row['Description']
data_process_management = DataProcessManagementServiceProcessClient(
self)
function_type = None
if ftype == 'PythonFunction':
function_type = PFT.PYTHON
elif ftype == 'NumexprFunction':
function_type = PFT.NUMEXPR
else:
raise Conflict('Unsupported Function Type: %s' % ftype)
parameter_function = ParameterFunctionResource(
name=name,
function=func_expr,
function_type=function_type,
owner=owner,
args=args,
description=descr)
parameter_function.alt_ids = ['PRE:' + row['ID']]
parameter_function_id = self.create_parameter_function(
parameter_function)
dpd = DataProcessDefinition()
dpd.name = name
dpd.description = 'Parameter Function Definition for %s' % name
dpd.data_process_type = DataProcessTypeEnum.PARAMETER_FUNCTION
dpd.parameters = args
data_process_management.create_data_process_definition(
dpd, parameter_function_id)
return parameter_function_id
def enqueue_command(self, command=None, link=False):
"""
Enqueue command for remote processing.
"""
if link and self._link_status != TelemetryStatusType.AVAILABLE:
raise Conflict('Cannot forward while link is down.')
if not isinstance(command, RemoteCommand):
raise BadRequest('Invalid command parameter.')
command.time_queued = time.time()
if not command.command_id:
command.command_id = str(uuid.uuid4())
self._tx_dict[command.command_id] = command
self._client.enqueue(command)
self._update_queue_resource()
self._publisher.publish_event(event_type='RemoteQueueModifiedEvent',
origin=self._xs_name,
queue_size=len(self._client._queue))
return command
def delete_doc(self, doc, datastore_name=None, **kwargs):
"""
Remove all versions of specified raw doc from the data store.
This method will check the '_rev' value to ensure that the doc
provided is the most recent known doc version. If not, a
Conflict exception is thrown.
If object id (str) is given instead of an object, deletes the
object with the given id.
"""
ds, datastore_name = self._get_datastore(datastore_name)
doc_id = doc if type(doc) is str else doc["_id"]
try:
if type(doc) is str:
del ds[doc_id]
else:
ds.delete(doc)
except ResourceNotFound:
raise NotFound('Object with id %s does not exist.' % doc_id)
except ResourceConflict:
raise Conflict("Object with id %s revision conflict" % doc["_id"])
def update_doc(self, doc, datastore_name=""):
ds, datastore_name = self._get_datastore(datastore_name)
if '_id' not in doc:
raise BadRequest("Doc must have '_id'")
if '_rev' not in doc:
raise BadRequest("Doc must have '_rev'")
# First, try to read document to ensure it exists
# Have to do this because save will create a new doc
# if it doesn't exist. We don't want this side-effect.
self.read_doc(doc["_id"], doc["_rev"], datastore_name)
log.info('Saving new version of object %s/%s' %
(datastore_name, doc["_id"]))
log.debug('update doc contents: %s', doc)
try:
res = ds.save(doc)
except ResourceConflict:
raise Conflict('Object not based on most current version')
log.debug('Update result: %s' % str(res))
id, version = res
return (id, version)
def test_exception_map(self):
# only testing one of the handlers. assuming they all share the decorator
with patch(
'ion.services.cei.process_dispatcher_service._PYON_DASHI_EXC_MAP',
self.mock_pyon_dashi_exc_map):
self.mock_backend.read_definition.side_effect = NotFound()
with self.assertRaises(FakeDashiNotFoundError):
self.pd_dashi_handler.describe_definition("some-def")
self.mock_backend.read_definition.side_effect = Conflict()
with self.assertRaises(FakeDashiWriteConflictError):
self.pd_dashi_handler.describe_definition("some-def")
self.mock_backend.read_definition.side_effect = BadRequest()
with self.assertRaises(FakeDashiBadRequestError):
self.pd_dashi_handler.describe_definition("some-def")
# try with an unmapped IonException. should get passed through directly
self.mock_backend.read_definition.side_effect = IonException()
with self.assertRaises(IonException):
self.pd_dashi_handler.describe_definition("some-def")
def save_doc(self, doc, object_id=None, datastore_name=None):
"""
Create or update document
@param doc A dict with a document to create or update
@oaram object_id The ID for the new document
"""
ds, datastore_name = self._get_datastore(datastore_name)
# Assign an id to doc (recommended in CouchDB documentation)
if "_id" not in doc:
doc["_id"] = object_id or uuid4().hex
try:
obj_id, version = ds.save(doc)
except ResourceConflict:
if "_rev" in doc:
raise Conflict("Object with id %s revision conflict" %
doc["_id"])
else:
raise BadRequest("Object with id %s already exists" %
doc["_id"])
return obj_id, version
def create_stream(self,
name='',
exchange_point='',
topic_ids=None,
credentials=None,
stream_definition_id='',
description=''):
# Argument Validation
if name and self.clients.resource_registry.find_resources(
restype=RT.Stream, name=name, id_only=True)[0]:
raise Conflict('The named stream already exists')
validate_true(exchange_point, 'An exchange point must be specified')
exchange_point_id = None
try:
xp_obj = self.clients.exchange_management.read_exchange_point(
exchange_point)
exchange_point_id = exchange_point
exchange_point = xp_obj.name
except NotFound:
self.container.ex_manager.create_xp(exchange_point)
xp_objs, _ = self.clients.resource_registry.find_resources(
restype=RT.ExchangePoint, name=exchange_point, id_only=True)
if not xp_objs:
raise BadRequest('failed to create an ExchangePoint: ' +
exchange_point)
exchange_point_id = xp_objs[0]
topic_ids = topic_ids or []
if not name: name = create_unique_identifier()
# Get topic names and topics
topic_names = []
associated_topics = []
for topic_id in topic_ids:
topic = self.read_topic(topic_id)
if topic.exchange_point == exchange_point:
topic_names.append(self._sanitize(topic.name))
associated_topics.append(topic_id)
else:
log.warning(
'Attempted to attach stream %s to topic %s with different exchange points',
name, topic.name)
stream = Stream(name=name, description=description)
routing_key = '.'.join([self._sanitize(name)] + topic_names +
['stream'])
if len(routing_key) > 255:
raise BadRequest('There are too many topics for this.')
stream.stream_route.exchange_point = exchange_point
stream.stream_route.routing_key = routing_key
#@todo: validate credentials
stream.stream_route.credentials = credentials
stream_id, rev = self.clients.resource_registry.create(stream)
self._associate_stream_with_exchange_point(stream_id,
exchange_point_id)
if stream_definition_id: #@Todo: what if the stream has no definition?!
self._associate_stream_with_definition(stream_id,
stream_definition_id)
for topic_id in associated_topics:
self._associate_topic_with_stream(topic_id, stream_id)
log.info('Stream %s: %s', name, routing_key)
return stream_id, stream.stream_route
def _load_config(self):
de = self.lookup("/Config/CFG")
if not de:
raise Conflict("Expected /Config/CFG in directory. Correct Org??")