def create_data_product(self, data_product=None, stream_definition_id='', parameter_dictionary=None, exchange_point=''):
"""
@param data_product IonObject which defines the general data product resource
@param source_resource_id IonObject id which defines the source for the data
@retval data_product_id
"""
res, _ = self.clients.resource_registry.find_resources(restype=RT.DataProduct, name=data_product.name, id_only=True)
validate_false(len(res), 'A data product with the name %s already exists.' % data_product.name)
log.info('Creating DataProduct: %s', data_product.name)
log.debug('%s', data_product.__dict__)
# Create will validate and register a new data product within the system
# If the stream definition has a parameter dictionary, use that
validate_is_not_none(stream_definition_id, 'A stream definition id must be passed to register a data product')
stream_def_obj = self.clients.pubsub_management.read_stream_definition(stream_definition_id) # Validates and checks for param_dict
parameter_dictionary = stream_def_obj.parameter_dictionary or parameter_dictionary
validate_is_not_none(parameter_dictionary , 'A parameter dictionary must be passed to register a data product')
validate_is_not_none(data_product, 'A data product (ion object) must be passed to register a data product')
exchange_point = exchange_point or 'science_data'
#--------------------------------------------------------------------------------
# Register - create and store a new DataProduct resource using provided metadata
#--------------------------------------------------------------------------------
data_product_id, rev = self.clients.resource_registry.create(data_product)
#-----------------------------------------------------------------------------------------------
#Create the stream and a dataset if a stream definition is provided
#-----------------------------------------------------------------------------------------------
#if stream_definition_id:
#@todo: What about topics?
stream_id,route = self.clients.pubsub_management.create_stream(name=data_product.name,
exchange_point=exchange_point,
description=data_product.description,
stream_definition_id=stream_definition_id)
# Associate the Stream with the main Data Product and with the default data product version
self.data_product.link_stream(data_product_id, stream_id)
# create a dataset...
data_set_id = self.clients.dataset_management.create_dataset( name= 'data_set_%s' % stream_id,
stream_id=stream_id,
parameter_dict=parameter_dictionary,
temporal_domain=data_product.temporal_domain,
spatial_domain=data_product.spatial_domain)
# link dataset with data product. This creates the association in the resource registry
self.data_product.link_data_set(data_product_id=data_product_id, data_set_id=data_set_id)
# Return the id of the new data product
return data_product_id
def activate_subscription(self, subscription_id=''):
validate_false(self.subscription_is_active(subscription_id), 'Subscription is already active.')
subscription = self.read_subscription(subscription_id)
streams, assocs = self.clients.resource_registry.find_objects(subject=subscription_id, object_type=RT.Stream, predicate=PRED.hasStream,id_only=False)
topic_ids, assocs = self.clients.resource_registry.find_objects(subject=subscription_id, predicate=PRED.hasTopic, id_only=True)
data_product_ids, assocs = self.clients.resource_registry.find_objects(subject=subscription_id, predicate=PRED.hasDataProduct, id_only=True)
topic_topology = set()
topics = []
for topic_id in topic_ids:
topic_tree = self._child_topics(topic_id)
topic_topology = topic_topology.union(topic_tree)
if topic_topology:
topics = self.clients.resource_registry.read_mult(object_ids=list(topic_topology))
for stream in streams:
log.info('%s -> %s', stream.name, subscription.exchange_name)
if dot.isEnabledFor(logging.INFO):
import re
queue_name = re.sub(r'[ -]','_',subscription.exchange_name)
dot.info(' %s -> %s' %(stream.stream_route.routing_key[:-len('.stream')], queue_name))
self._bind(stream.stream_route.exchange_point, subscription.exchange_name, stream.stream_route.routing_key)
for exchange_point in subscription.exchange_points:
log.info('Exchange %s -> %s', exchange_point, subscription.exchange_name)
self._bind(exchange_point, subscription.exchange_name, '*')
for topic in topics:
log.info('Topic %s -> %s', topic.name, subscription.exchange_name)
self._bind(topic.exchange_point, subscription.exchange_name, '#.%s.#' % self._sanitize(topic.name))
for data_product_id in data_product_ids:
streams, assocs = self.clients.resource_registry.find_objects(subject=data_product_id, predicate=PRED.hasStream, id_only=False)
for stream in streams:
log.info('%s -> %s', stream.name, subscription.exchange_name)
if dot.isEnabledFor(logging.INFO):
import re
queue_name = re.sub(r'[ -]','_',subscription.exchange_name)
dot.info(' %s -> %s' %(stream.stream_route.routing_key[:-len('.stream')], queue_name))
self._bind(stream.stream_route.exchange_point, subscription.exchange_name, stream.stream_route.routing_key)
subscription.activated = True
self.clients.resource_registry.update(subscription)
def activate_subscription(self, subscription_id=''):
validate_false(self.subscription_is_active(subscription_id),
'Subscription is already active.')
subscription = self.read_subscription(subscription_id)
streams, assocs = self.clients.resource_registry.find_objects(
subject=subscription_id,
object_type=RT.Stream,
predicate=PRED.hasStream,
id_only=False)
topic_ids, assocs = self.clients.resource_registry.find_objects(
subject=subscription_id, predicate=PRED.hasTopic, id_only=True)
topic_topology = set()
topics = []
for topic_id in topic_ids:
topic_tree = self._child_topics(topic_id)
topic_topology = topic_topology.union(topic_tree)
if topic_topology:
topics = self.clients.resource_registry.read_mult(
object_ids=list(topic_topology))
for stream in streams:
log.info('%s -> %s', stream.name, subscription.exchange_name)
if dot.isEnabledFor(logging.INFO):
import re
queue_name = re.sub(r'[ -]', '_', subscription.exchange_name)
dot.info(' %s -> %s' %
(stream.stream_route.routing_key[:-len('.stream')],
queue_name))
self._bind(stream.stream_route.exchange_point,
subscription.exchange_name,
stream.stream_route.routing_key)
for exchange_point in subscription.exchange_points:
log.info('Exchange %s -> %s', exchange_point,
subscription.exchange_name)
self._bind(exchange_point, subscription.exchange_name, '*')
for topic in topics:
log.info('Topic %s -> %s', topic.name, subscription.exchange_name)
self._bind(topic.exchange_point, subscription.exchange_name,
'#.%s.#' % self._sanitize(topic.name))
subscription.activated = True
self.clients.resource_registry.update(subscription)
def test_validations(self):
import pyon.util.arg_check as arg_check
with self.assertRaises(BadRequest):
arg_check.validate_true(False, 'test')
with self.assertRaises(BadRequest):
arg_check.validate_equal(3, 4, 'test')
with self.assertRaises(BadRequest):
arg_check.validate_not_equal(4, 4, 'test')
with self.assertRaises(BadRequest):
arg_check.validate_false(True, 'test')
with self.assertRaises(BadRequest):
one = list()
two = list()
arg_check.validate_is(one, two, 'test')
with self.assertRaises(BadRequest):
one = list()
two = one
arg_check.validate_is_not(one, two, 'test')
with self.assertRaises(BadRequest):
c = None
arg_check.validate_is_not_none(c, 'test')
with self.assertRaises(BadRequest):
one = list([1, 3])
two = 2
arg_check.validate_in(two, one, 'test')
with self.assertRaises(BadRequest):
one = list([1, 2, 3])
two = 2
arg_check.validate_not_in(two, one, 'test')
with self.assertRaises(BadRequest):
one = list()
arg_check.validate_is_instance(one, dict, 'test')
with self.assertRaises(BadRequest):
one = list()
arg_check.validate_not_is_instance(one, list, 'test')
def test_validations(self):
import pyon.util.arg_check as arg_check
with self.assertRaises(BadRequest):
arg_check.validate_true(False,'test')
with self.assertRaises(BadRequest):
arg_check.validate_equal(3,4,'test')
with self.assertRaises(BadRequest):
arg_check.validate_not_equal(4,4,'test')
with self.assertRaises(BadRequest):
arg_check.validate_false(True,'test')
with self.assertRaises(BadRequest):
one = list()
two = list()
arg_check.validate_is(one,two,'test')
with self.assertRaises(BadRequest):
one = list()
two = one
arg_check.validate_is_not(one,two,'test')
with self.assertRaises(BadRequest):
c = None
arg_check.validate_is_not_none(c,'test')
with self.assertRaises(BadRequest):
one = list([1,3])
two = 2
arg_check.validate_in(two,one,'test')
with self.assertRaises(BadRequest):
one = list([1,2,3])
two = 2
arg_check.validate_not_in(two,one,'test')
with self.assertRaises(BadRequest):
one = list()
arg_check.validate_is_instance(one,dict,'test')
with self.assertRaises(BadRequest):
one = list()
arg_check.validate_not_is_instance(one,list,'test')