def start_app(self, appdef=None, config=None):
"""
@brief Start an app from an app definition.
Note: apps can come in one of 2 variants:
1 processapp: In-line defined process to be started
2 regular app: Full app definition
"""
log.debug("AppManager.start_app(appdef=%s) ..." % appdef)
appdef = DotDict(appdef)
if 'config' in appdef:
app_cfg = appdef.config.copy()
if config:
dict_merge(app_cfg, config, inplace=True)
config = app_cfg
if 'processapp' in appdef:
# Case 1: Appdef contains definition of process to start
name, module, cls = appdef.processapp
try:
pid = self.container.spawn_process(name, module, cls, config)
appdef._pid = pid
self.apps.append(appdef)
except Exception, ex:
log.exception("Appl %s start from processapp failed" % appdef.name)
def _run_consumer(self, ctag, queue_name, gqueue, callback):
cnt = 0
while True:
m = gqueue.get()
if isinstance(m, self.ConsumerClosedMessage):
break
exchange, routing_key, body, props = m
# create method frame
method_frame = DotDict()
method_frame['consumer_tag'] = ctag
method_frame['redelivered'] = False # @TODO
method_frame['exchange'] = exchange
method_frame['routing_key'] = routing_key
# create header frame
header_frame = DotDict()
header_frame['headers'] = props.copy()
# make delivery tag for ack/reject later
dtag = self._generate_dtag(ctag, cnt)
cnt += 1
with self._lock_unacked:
self._unacked[dtag] = (ctag, queue_name, m)
method_frame['delivery_tag'] = dtag
# deliver to callback
try:
callback(self, method_frame, header_frame, body)
except Exception:
log.exception("delivering to consumer, ignore!")
def start_app(self, appdef=None, config=None):
"""
@brief Start an app from an app definition.
Note: apps can come in one of 2 variants:
1 processapp: In-line defined process to be started
2 regular app: Full app definition
"""
log.debug("AppManager.start_app(appdef=%s) ..." % appdef)
appdef = DotDict(appdef)
if 'config' in appdef:
app_cfg = appdef.config.copy()
if config:
dict_merge(app_cfg, config, inplace=True)
config = app_cfg
if 'processapp' in appdef:
# Case 1: Appdef contains definition of process to start
name, module, cls = appdef.processapp
try:
pid = self.container.spawn_process(name, module, cls, config)
appdef._pid = pid
self.apps.append(appdef)
except Exception, ex:
log.exception("Appl %s start from processapp failed" %
appdef.name)
def load_data_process(self, stream_id=""):
dpms_client = DataProcessManagementServiceClient()
dataprocess_details = dpms_client.read_data_process_for_stream(stream_id)
dataprocess_details = DotDict(dataprocess_details or {})
dataprocess_id = dataprocess_details.dataprocess_id
#set metrics attributes
dataprocess_details.granule_counter = 0
self._dataprocesses[dataprocess_id] = dataprocess_details
#add the stream id to the map
if 'in_stream_id' in dataprocess_details:
if dataprocess_details['in_stream_id'] in self._streamid_map:
(self._streamid_map[ dataprocess_details['in_stream_id'] ]).append(dataprocess_id)
else:
self._streamid_map[ dataprocess_details['in_stream_id'] ] = [dataprocess_id]
#todo: add transform worker id
self.event_publisher.publish_event(origin=dataprocess_id, origin_type='DataProcess', status=DataProcessStatusType.NORMAL,
description='data process loaded into transform worker')
#create a publisher for output stream
self.create_publisher(dataprocess_id, dataprocess_details)
return [dataprocess_id]
def setUp(self):
super(DiscoveryIntTest, self).setUp()
self._start_container()
self.addCleanup(DiscoveryIntTest.es_cleanup)
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.discovery = DiscoveryServiceClient()
self.catalog = CatalogManagementServiceClient()
self.ims = IndexManagementServiceClient()
self.rr = ResourceRegistryServiceClient()
if use_es:
self.es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
self.es_port = CFG.get_safe('server.elasticsearch.port', '9200')
CFG.server.elasticsearch.shards = 1
CFG.server.elasticsearch.replicas = 0
CFG.server.elasticsearch.river_shards = 1
CFG.server.elasticsearch.river_replicas = 0
self.es = ep.ElasticSearch(
host=self.es_host,
port=self.es_port,
timeout=10,
verbose=True
)
op = DotDict(CFG)
op.op = 'clean_bootstrap'
self.container.spawn_process('index_bootstrap','ion.processes.bootstrap.index_bootstrap','IndexBootStrap', op)
def test_create_catalog(self):
self.rr_find_res.return_value = ([],[])
fake_indexes = [
DotDict({'_id':'1', 'name':'one', 'options' : {'attribute_match':['attr_field'],'range_fields':[], 'geo_fields':[] }}),
DotDict({'_id':'1', 'name':'two', 'options' : {'attribute_match':[],'range_fields':['range_field'], 'geo_fields':[] }}),
DotDict({'_id':'1', 'name':'three', 'options' : {'attribute_match':[],'range_fields':['range_field'], 'geo_fields':['geo_field'] }})
]
def find_res(*args, **kwargs):
retval = {}
for d in fake_indexes:
retval[d.name] = d
return retval
self.ims_list_indexes.side_effect = find_res
self.rr_create.return_value = 'cat_id','rev'
retval = self.catalog_management.create_catalog('catalog_name',['range_field','geo_field'])
self.assertTrue(self.rr_create_assoc.call_count==1,
'Not enough associations %d' % self.rr_create_assoc.call_count)
retval = self.catalog_management.create_catalog('catalog_name',['range_field'])
self.assertTrue(self.rr_create_assoc.call_count==3,
'Not enough associations %d' % self.rr_create_assoc.call_count)
self.assertTrue(retval=='cat_id')
def test_define_replay_no_data(self):
#mocks
self.mock_ps_create_stream.return_value = '12345'
self.mock_rr_create.return_value = ('replay_id', 'garbage')
self.mock_ds_read.return_value = DotDict({
'datastore_name':
'unittest',
'view_name':
'garbage',
'primary_view_key':
'primary key'
})
document = DotDict({'stream_resource_id': '0'})
self.mock_pd_schedule.return_value = 'process_id'
self.datastore.query_view.return_value = [] # Raises index error
config = {
'process': {
'query': 'myquery',
'datastore_name': 'unittest',
'view_name': 'garbage',
'key_id': 'primary key',
'delivery_format': None,
'publish_streams': {
'output': '12345'
}
}
}
with self.assertRaises(NotFound):
self.data_retriever_service.define_replay(dataset_id='dataset_id',
query='myquery')
def preload(self):
config = DotDict()
config.op = 'load'
config.scenario='BETA'
config.categories='Parser,Reference'
config.path='master'
self.container.spawn_process('ion_loader', 'ion.processes.bootstrap.ion_loader','IONLoader',config)
def test_get_last_granule(self, mock_bb, dsm_cli, dsm):
mock_bb().sync_rdt_with_coverage = Mock()
mock_bb().to_granule.return_value = {'test':True}
dsm_cli().read_dataset = Mock()
dataset = DotDict()
dataset.datastore_name = 'test'
dataset.view_name = 'bogus/view'
dsm._get_coverage = Mock()
dsm._get_coverage.return_value = {}
datastore = DotDict()
datastore.query_view = Mock()
datastore.query_view.return_value = [{'doc':{'ts_create':0}}]
container = DotDict()
container.datastore_manager.get_datastore = Mock()
container.datastore_manager.get_datastore.return_value = datastore
retval = self.replay.get_last_granule(container,'dataset_id')
self.assertEquals(retval,{'test':True})
def setUp(self):
self.mock_backend = DotDict()
self.mock_backend['create_definition'] = Mock()
self.mock_backend['read_definition'] = Mock()
self.mock_backend['read_definition_by_name'] = Mock()
self.mock_backend['update_definition'] = Mock()
self.mock_backend['delete_definition'] = Mock()
self.mock_backend['create'] = Mock()
self.mock_backend['schedule'] = Mock()
self.mock_backend['read_process'] = Mock()
self.mock_backend['list'] = Mock()
self.mock_backend['cancel'] = Mock()
self.mock_backend['set_system_boot'] = Mock()
self.mock_dashi = DotDict()
self.mock_dashi['handle'] = Mock()
self.mock_pyon_dashi_exc_map = {
NotFound: FakeDashiNotFoundError,
BadRequest: FakeDashiBadRequestError,
Conflict: FakeDashiWriteConflictError
}
self.pd_dashi_handler = PDDashiHandler(self.mock_backend,
self.mock_dashi)
def create_data_process_logger(self, data_product_id, clone_id, argument_map):
'''
Launches a data process that just prints input
'''
out_name = argument_map.values()[0]
# Make the transfofm function
tf_obj = IonObject(RT.TransformFunction,
name='stream_logger',
description='',
function='stream_logger',
module='ion.services.sa.test.test_data_process_functions',
arguments=['x'],
function_type=TransformFunctionType.TRANSFORM)
func_id = self.data_process_management.create_transform_function(tf_obj)
self.addCleanup(self.data_process_management.delete_transform_function, func_id)
# Make the data process definition
dpd_obj = IonObject(RT.DataProcessDefinition,
name='stream_logger',
description='logs some stream stuff',
data_process_type=DataProcessTypeEnum.RETRIEVE_PROCESS)
configuration = DotDict()
configuration.publish_limit = 40
dpd_id = self.data_process_management.create_data_process_definition(dpd_obj, func_id)
data_process_id = self.data_process_management.create_data_process(
data_process_definition_id=dpd_id,
inputs=[data_product_id],
outputs=[clone_id],
configuration=configuration,
argument_map=argument_map,
out_param_name=out_name)
return data_process_id
class ChatServerService(BaseChatsService):
def on_init(self):
print "INIT CHAT SERVER"
self.clients = DotDict()
def register(self, user_name='', proc_id=''):
print "Registering user %s, client %s" % (user_name, proc_id)
client = ProcessRPCClient(node=self.container.node, name=proc_id, iface=IChatcService, process=self)
self.clients[user_name] = DotDict(procid=proc_id, user_name=user_name, client=client)
return "OK"
def unregister(self, user_name=''):
log.debug("Unregistering client %s" % proc_id)
del self.clients[user_name]
return "OK"
def message(self, from_name='', to_name='', text=''):
if to_name == "all":
for cl in self.clients.values():
cl['client'].message(from_name=from_name, text=text)
else:
client = self.clients.get(to_name, None)
if client:
client.client.message(from_name=from_name, text=text)
else:
return "USER NOT FOUND"
return "OK"
def list_users(self):
return str(self.clients.keys())
def test_dot_dict_constant(self):
d = DotDict({"foo": "bar"})
self.assertEqual("bar", d.foo)
d.foo = "somethingnew"
self.assertEqual("somethingnew", d.foo)
# DotDict only checks that an assignment operation is happening when it creates dummy entries
# ... it doesn't check that the dummy entry is on the left hand side of the assignment
k = d.foo1
self.assertIn("foo1", dir(d))
d.lock()
# test assigning directly to a locked dict
with self.assertRaises(AttributeError):
d.foo = "somethingelse"
self.assertEqual("somethingnew", d.foo)
# test dummy-creation-on-assignment loophole
with self.assertRaises(AttributeError):
k = d.foo2
self.assertNotIn("foo2", dir(d))
# test alternate dummy creation method: calling a function with it
with self.assertRaises(AttributeError):
k = lambda _: True
k(d.foo3)
self.assertNotIn("foo3", dir(d))
self.assertNotIn(DICT_LOCKING_ATTR, dir(d))
def test_dot_dict(self):
dotDict = DotDict({"foo": {"bar": {"bah": "fah"}}})
val = dotDict.foo.bar.bah
self.assertEqual(val, "fah")
dotDict.a = "1"
self.assertEqual(dotDict.a, "1")
self.assertTrue('a' in dotDict)
def test_dot_dict(self):
dotDict = DotDict({"foo": {"bar": {"bah": "fah"}}})
val = dotDict.foo.bar.bah
self.assertEqual(val, "fah")
dotDict.a = "1"
self.assertEqual(dotDict.a, "1")
self.assertTrue("a" in dotDict)
def setUp(self):
super(DiscoveryIntTest, self).setUp()
self._start_container()
self.addCleanup(DiscoveryIntTest.es_cleanup)
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.discovery = DiscoveryServiceClient()
self.catalog = CatalogManagementServiceClient()
self.ims = IndexManagementServiceClient()
self.rr = ResourceRegistryServiceClient()
if use_es:
self.es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
self.es_port = CFG.get_safe('server.elasticsearch.port', '9200')
CFG.server.elasticsearch.shards = 1
CFG.server.elasticsearch.replicas = 0
CFG.server.elasticsearch.river_shards = 1
CFG.server.elasticsearch.river_replicas = 0
self.es = ep.ElasticSearch(
host=self.es_host,
port=self.es_port,
timeout=10,
verbose=True
)
op = DotDict(CFG)
op.op = 'clean_bootstrap'
self.container.spawn_process('index_bootstrap','ion.processes.bootstrap.index_bootstrap','IndexBootStrap', op)
def test_create_network_definition_from_ci_config_bad(self):
CFG = DotDict({
'device_type': "bad_device_type",
})
# device_type
with self.assertRaises(PlatformDefinitionException):
NetworkUtil.create_network_definition_from_ci_config(CFG)
CFG = DotDict({
'device_type': "PlatformDevice",
})
# missing platform_id
with self.assertRaises(PlatformDefinitionException):
NetworkUtil.create_network_definition_from_ci_config(CFG)
CFG = DotDict({
'device_type': "PlatformDevice",
'platform_config': {
'platform_id': 'Node1D'
},
})
# missing driver_config
with self.assertRaises(PlatformDefinitionException):
NetworkUtil.create_network_definition_from_ci_config(CFG)
def test_dotdict_copy(self):
d = DotDict({"foo": "bar"})
d2 = copy.copy(d)
self.assertTrue(hasattr(d2, "foo"))
self.assertEqual("bar", d2.foo)
# output_streams = copy(self.CFG.get_safe('process.publish_streams'))
v = "a12345"
CFG = DotDict()
CFG.process.publish_streams.salinity = v
print "CFG =", CFG
self.assertTrue(hasattr(CFG.process.publish_streams, "salinity"))
self.assertEqual(v, CFG.process.publish_streams.salinity)
self.assertEqual(v, CFG.get_safe("process.publish_streams").salinity)
self.assertEqual(
v,
copy.copy(CFG.get_safe("process.publish_streams")).salinity)
output_streams = copy.copy(CFG.get_safe("process.publish_streams"))
print "output_streams =", output_streams
self.assertTrue(hasattr(output_streams, "salinity"))
print "output_streams.salinity =", output_streams.salinity
self.assertEqual(v, output_streams.salinity)
first_stream = output_streams.popitem()
print "first_stream =", first_stream
self.assertEqual(v, first_stream[1])
d.lock()
dl = copy.copy(d)
self.assertTrue(hasattr(dl, "foo"))
self.assertEqual("bar", dl.foo)
with self.assertRaises(AttributeError):
d.foo2 = "nope"
class ChatServerService(BaseChatsService):
def on_init(self):
print "INIT CHAT SERVER"
self.clients = DotDict()
def register(self, user_name='', proc_id=''):
print "Registering user %s, client %s" % (user_name, proc_id)
client = ProcessRPCClient(node=self.container.node,
name=proc_id,
iface=IChatcService,
process=self)
self.clients[user_name] = DotDict(procid=proc_id,
user_name=user_name,
client=client)
return "OK"
def unregister(self, user_name=''):
log.debug("Unregistering client %s" % proc_id)
del self.clients[user_name]
return "OK"
def message(self, from_name='', to_name='', text=''):
if to_name == "all":
for cl in self.clients.values():
cl['client'].message(from_name=from_name, text=text)
else:
client = self.clients.get(to_name, None)
if client:
client.client.message(from_name=from_name, text=text)
else:
return "USER NOT FOUND"
return "OK"
def list_users(self):
return str(self.clients.keys())
def setUp(self):
mock_clients = self._create_service_mock('index_management')
self.index_management = IndexManagementService()
self.index_management.clients = mock_clients
self.rr_create = mock_clients.resource_registry.create
self.rr_read = mock_clients.resource_registry.read
self.rr_update = mock_clients.resource_registry.update
self.rr_delete = mock_clients.resource_registry.delete
self.rr_find_resources = mock_clients.resource_registry.find_resources
self.rr_find_assocs = mock_clients.resource_registry.find_associations
self.rr_find_subj = mock_clients.resource_registry.find_subjects
self.rr_find_obj = mock_clients.resource_registry.find_objects
self.rr_delete_assoc = mock_clients.resource_registry.delete_association
self.get_datastore = Mock()
self.db_create = Mock()
self.get_datastore.return_value = DotDict(
{'datastore_name': 'test_datastore'})
self.index_management.container = DotDict({
'datastore_manager':
DotDict({'get_datastore': self.get_datastore})
})
self.index_name = 'test_index'
def test_create_schedule(self):
proc_def = DotDict()
proc_def['name'] = "someprocess"
proc_def['executable'] = {'module': 'my_module', 'class': 'class', 'url': 'myurl'}
mock_read_definition = Mock()
mock_read_definition.return_value = proc_def
self.pd_service.backend.read_definition = mock_read_definition
pid = self.pd_service.create_process("fake-process-def-id")
proc_schedule = DotDict()
proc_schedule['target'] = DotDict()
proc_schedule.target['constraints'] = {"hats": 4}
proc_schedule.target['node_exclusive'] = None
proc_schedule.target['execution_engine_id'] = None
configuration = {"some": "value"}
pid2 = self.pd_service.schedule_process("fake-process-def-id",
proc_schedule, configuration, pid)
self.assertTrue(pid.startswith(proc_def.name) and pid != proc_def.name)
self.assertEqual(pid, pid2)
self.assertTrue(pid.startswith(proc_def.name) and pid != proc_def.name)
self.assertEqual(self.mock_core.schedule_process.call_count, 1)
def _launch_transform(self, name_of_transform = '', data_proc_def_id = None, input_dpod_id = None, output_dpod_id = None):
# We need the key name here to be "L2_stream", since when the data process is launched, this name goes into
# the config as in config.process.publish_streams.L2_stream when the config is used to launch the data process
if name_of_transform in ['L0', 'L1']:
binding = '%s_stream' % name_of_transform
elif name_of_transform == 'L2_salinity':
binding = 'salinity'
elif name_of_transform == 'L2_density':
binding = 'density'
output_products = {binding : output_dpod_id}
config = None
if name_of_transform == 'L1':
config = self._create_calibration_coefficients_dict()
elif name_of_transform == 'L2_density':
config = DotDict()
config.process = {'lat' : 32.7153, 'lon' : 117.1564}
data_proc_id = self.data_process_management.create_data_process( data_proc_def_id, [input_dpod_id], output_products, config)
self.addCleanup(self.data_process_management.delete_data_process, data_proc_id)
self.data_process_management.activate_data_process(data_proc_id)
self.addCleanup(self.data_process_management.deactivate_data_process, data_proc_id)
log.debug("Created a data process for ctdbp %s transform: id = %s", name_of_transform, data_proc_id)
return data_proc_id
def test_list_indexes(self):
# Mocks
self.rr_find_resources.return_value = ([
DotDict({
'_id': '1',
'name': '1'
}),
DotDict({
'_id': '2',
'name': '2'
}),
DotDict({
'_id': '3',
'name': '3'
}),
DotDict({
'_id': '4',
'name': '4'
})
], [1, 2, 3, 4])
# Execution
retval = self.index_management.list_indexes()
# Assertions
self.assertTrue(retval == {
'1': '1',
'2': '2',
'3': '3',
'4': '4'
}, 'Index mismatch')
def start_app(self, appdef=None, config=None):
"""
@brief Start an app from an app definition.
Note: apps can come in one of 2 variants:
1 processapp: In-line defined process to be started
2 regular app: Full app definition
"""
log.debug("AppManager.start_app(appdef=%s) ..." % appdef)
appdef = DotDict(appdef)
app_config = DictModifier(CFG)
if 'config' in appdef:
# Apply config from app file
app_file_cfg = DotDict(appdef.config)
app_config.update(app_file_cfg)
if config:
# Nest dict modifier and apply config from rel file
app_config = DictModifier(app_config, config)
if 'processapp' in appdef:
# Case 1: Appdef contains definition of process to start
name, module, cls = appdef.processapp
try:
pid = self.container.spawn_process(name, module, cls,
app_config)
appdef._pid = pid
self.apps.append(appdef)
except Exception, ex:
log.exception("Appl %s start from processapp failed" %
appdef.name)
def start_app(self, appdef=None, config=None):
"""
@brief Start an app from an app definition.
Note: apps can come in one of 2 variants:
1 processapp: In-line defined process to be started
2 regular app: Full app definition
"""
log.debug("AppManager.start_app(appdef=%s) ..." % appdef)
appdef = DotDict(appdef)
app_config = DictModifier(CFG)
if 'config' in appdef:
# Apply config from app file
app_file_cfg = DotDict(appdef.config)
app_config.update(app_file_cfg)
if config:
# Nest dict modifier and apply config from rel file
app_config = DictModifier(app_config, config)
if 'processapp' in appdef:
# Case 1: Appdef contains definition of process to start
name, module, cls = appdef.processapp
try:
pid = self.container.spawn_process(name, module, cls, app_config)
appdef._pid = pid
self.apps.append(appdef)
except Exception, ex:
log.exception("Appl %s start from processapp failed" % appdef.name)
def cache_resources(self, resource_type, specific_ids=None):
"""
Save all resources of a given type to memory, for in-memory lookup ops
This is a PREFETCH operation, and EnhancedResourceRegistryClient objects that use the cache functionality
should NOT be kept across service calls.
"""
#log.info("Caching resources: %s", resource_type)
#log.debug("This cache is %s", self)
time_caching_start = get_ion_ts()
resource_objs = []
if specific_ids is None:
resource_objs, _ = self.RR.find_resources(restype=resource_type, id_only=False)
else:
assert type(specific_ids) is list
if specific_ids:
resource_objs = self.RR.read_mult(specific_ids)
lookups = DotDict()
lookups.by_id = {}
lookups.by_name = {}
self._cached_resources[resource_type] = lookups
for r in resource_objs:
self._add_resource_to_cache(resource_type, r)
time_caching_stop = get_ion_ts()
total_time = int(time_caching_stop) - int(time_caching_start)
class TestContainerExchangeToEms(IonIntegrationTestCase):
# these tests should auto contact the EMS to do the work
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2coi.yml')
self.ems = ExchangeManagementServiceClient()
self.rr = ResourceRegistryServiceClient()
# we want the ex manager to do its thing, but without actual calls to broker
# just mock out the transport
self.container.ex_manager._priviledged_transport = Mock(BaseTransport)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Test reaches into container, doesn\'t work with CEI')
def test_create_xs_talks_to_ems(self):
self.patch_cfg('pyon.ion.exchange.CFG',
container=DotDict(CFG.container,
exchange=DotDict(auto_register=True)))
xs = self.container.ex_manager.create_xs('house')
self.addCleanup(xs.delete)
# should have called EMS and set RR items
res, _ = self.rr.find_resources(RT.ExchangeSpace, name='house')
self.assertEquals(res[0].name, 'house')
# should have tried to call broker as well
self.assertEquals(
self.container.ex_manager._priviledged_transport.
declare_exchange_impl.call_count, 1)
self.assertIn(
'house', self.container.ex_manager._priviledged_transport.
declare_exchange_impl.call_args[0][0])
@patch.dict('pyon.ion.exchange.CFG',
container=DotDict(CFG.container,
exchange=DotDict(auto_register=False)))
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Test reaches into container, doesn\'t work with CEI')
def test_create_xs_with_no_flag_only_uses_ex_manager(self):
self.patch_cfg('pyon.ion.exchange.CFG',
container=DotDict(
CFG.container,
exchange=DotDict(auto_register=False)))
xs = self.container.ex_manager.create_xs('house')
self.addCleanup(xs.delete)
e1, e2 = self.rr.find_resources(RT.ExchangeSpace, name='house')
self.assertEquals(e1, [])
self.assertEquals(e2, [])
self.assertEquals(
self.container.ex_manager._priviledged_transport.
declare_exchange_impl.call_count, 1)
self.assertIn(
'house', self.container.ex_manager._priviledged_transport.
declare_exchange_impl.call_args[0][0])
def test_is_persisted(self):
stream = DotDict()
stream.persisted = True
self.pubsub_read.return_value = stream
retval = self.ingestion_management.is_persisted("stream_id")
self.assertEquals(retval, True)
def preload_ui(self):
config = DotDict()
config.op='loadui'
config.loadui=True
config.attachments='res/preload/r2_ioc/attachments'
config.ui_path = "http://userexperience.oceanobservatories.org/database-exports/Candidates"
self.container.spawn_process('preloader', 'ion.processes.bootstrap.ion_loader', 'IONLoader', config)
def lc_preload(self):
config = DotDict()
config.op = 'load'
config.scenario = 'BASE,LC_TEST'
config.categories = 'ParameterFunctions,ParameterDefs,ParameterDictionary'
config.path = 'res/preload/r2_ioc'
self.container.spawn_process('preload','ion.processes.bootstrap.ion_loader','IONLoader', config)
def _create_instrument_config_builder(self):
clients = DotDict()
clients.resource_registry = self.RR
clients.pubsub_management = self.PSC
clients.dataset_management = self.DSC
iconfig_builder = InstrumentAgentConfigurationBuilder(clients)
return iconfig_builder
def test_bridge_create_schedule(self):
pdcfg = dict(uri="amqp://hello", topic="pd", exchange="123")
self.pd_service.CFG = DotDict()
self.pd_service.CFG['process_dispatcher_bridge'] = pdcfg
self.pd_service.init()
self.assertIsInstance(self.pd_service.backend, PDBridgeBackend)
event_pub = Mock()
self.pd_service.backend.event_pub = event_pub
# sneak in and replace dashi connection method
mock_dashi = Mock()
mock_dashi.consume.return_value = lambda : None
self.pd_service.backend._init_dashi = lambda : mock_dashi
self.pd_service.start()
self.assertEqual(mock_dashi.handle.call_count, 1)
proc_def = DotDict()
proc_def['name'] = "someprocess"
proc_def['executable'] = {'module':'my_module', 'class':'class'}
self.mock_rr_read.return_value = proc_def
pid = self.pd_service.create_process("fake-process-def-id")
proc_schedule = DotDict()
proc_schedule['target'] = DotDict()
proc_schedule.target['constraints'] = {"hats" : 4}
configuration = {"some": "value"}
pid2 = self.pd_service.schedule_process("fake-process-def-id",
proc_schedule, configuration, pid)
self.assertTrue(pid.startswith(proc_def.name) and pid != proc_def.name)
self.assertEqual(pid, pid2)
self.assertTrue(pid.startswith(proc_def.name) and pid != proc_def.name)
self.assertEqual(mock_dashi.call.call_count, 1)
call_args, call_kwargs = mock_dashi.call.call_args
self.assertEqual(set(call_kwargs),
set(['upid', 'spec', 'subscribers', 'constraints']))
self.assertEqual(call_kwargs['constraints'],
proc_schedule.target['constraints'])
self.assertEqual(call_kwargs['subscribers'],
self.pd_service.backend.pd_process_subscribers)
self.assertEqual(call_args, ("pd", "dispatch_process"))
self.assertEqual(event_pub.publish_event.call_count, 0)
# trigger some fake async state updates from dashi. first
# should not trigger an event
self.pd_service.backend._process_state(dict(upid=pid,
state="400-PENDING"))
self.assertEqual(event_pub.publish_event.call_count, 0)
self.pd_service.backend._process_state(dict(upid=pid,
state="500-RUNNING"))
self.assertEqual(event_pub.publish_event.call_count, 1)
def test_query_request_association(self):
query = DotDict()
query.association = 'resource_id'
self.discovery.query_association = Mock()
self.discovery.query_association.return_value = 'test'
retval = self.discovery.query_request(query)
self.assertTrue(retval == 'test')
def test_query_request_collection(self):
query = DotDict()
query.collection = 'test'
self.discovery.query_collection = Mock()
self.discovery.query_collection.return_value = 'test'
retval = self.discovery.query_request(query)
self.assertTrue(retval == 'test')
def test_get_dataset_info(self):
coverage = DotDict()
coverage.info = 1
self.dataset_management._get_coverage = Mock()
self.dataset_management._get_coverage.return_value = coverage
retval = self.dataset_management.get_dataset_info('dataset_id')
self.assertEquals(retval,1)
def test_query_request_collection(self):
query = DotDict()
query.collection = 'test'
self.discovery.query_collection = Mock()
self.discovery.query_collection.return_value = 'test'
retval = self.discovery.query_request(query)
self.assertTrue(retval == 'test')
def test_query_request_association(self):
query = DotDict()
query.association = 'resource_id'
self.discovery.query_association = Mock()
self.discovery.query_association.return_value = 'test'
retval = self.discovery.query_request(query)
self.assertTrue(retval == 'test')
def test_dotdict_builtin_error(self):
# Ensures that you can not override the builtin methods
base = DotDict()
with self.assertRaises(AttributeError):
base.pop = "shouldnt work"
with self.assertRaises(AttributeError):
base.__getitem__ = "really shouldnt work"
with self.assertRaises(AttributeError):
base.another.chained.pop = "again should not work"
def test_dotdict_builtin_error(self):
# Ensures that you can not override the builtin methods
base = DotDict()
with self.assertRaises(AttributeError):
base.pop = 'shouldnt work'
with self.assertRaises(AttributeError):
base.__getitem__ = 'really shouldnt work'
with self.assertRaises(AttributeError):
base.another.chained.pop = 'again should not work'
def test_query_request_term_search(self):
query = DotDict()
query.index = 'index_id'
query.field = 'field'
query.value = 'value'
self.discovery.query_term = Mock()
self.discovery.query_term.return_value = 'test'
retval = self.discovery.query_request(query)
self.assertTrue(retval == 'test', '%s' % retval)
def test_query_request_term_search(self):
query = DotDict()
query.index = 'index_id'
query.field = 'field'
query.value = 'value'
self.discovery.query_term = Mock()
self.discovery.query_term.return_value = 'test'
retval = self.discovery.query_request(query)
self.assertTrue(retval == 'test', '%s' % retval)
def test_dotdict_chaining(self):
base = DotDict({'test': None})
base.chained.example.provides.utility = True
self.assertTrue(base['chained']['example']['provides']['utility'])
base.setting = True
self.assertTrue(base.setting)
self.assertTrue(base['setting'])
base.map = {'key': 'value'}
self.assertIsInstance(base.map, DotDict, '%s' % type(base.map))
self.assertTrue(base.map.key == 'value')
def test_query_request_range(self):
query = DotDict()
query['range'] = {'from':0, 'to':90}
query.index = 'index_id'
query.field = 'field'
self.discovery.query_range = Mock()
self.discovery.query_range.return_value = 'test'
retval = self.discovery.query_request(query)
self.assertTrue(retval == 'test')
def test_dotdict_chaining(self):
base = DotDict({'test':None})
base.chained.example.provides.utility = True
self.assertTrue(base['chained']['example']['provides']['utility'])
base.setting = True
self.assertTrue(base.setting)
self.assertTrue(base['setting'])
base.map = {'key':'value'}
self.assertIsInstance(base.map,DotDict, '%s' % type(base.map))
self.assertTrue(base.map.key=='value')
def test_query_request_range(self):
query = DotDict()
query['range'] = {'from':0, 'to':90}
query.index = 'index_id'
query.field = 'field'
self.discovery.query_range = Mock()
self.discovery.query_range.return_value = 'test'
retval = self.discovery.query_request(query)
self.assertTrue(retval == 'test')
def test_delete_ingestion(self):
testval = DotDict()
testval.o = "sub_id"
self.rr_find_assocs.return_value = [testval]
self.ingestion_management.delete_ingestion_configuration("test")
self.rr_del_assoc.assert_called_once_with(testval)
self.pubsub_del_sub.assert_called_once_with(testval.o)
self.rr_delete.assert_called_once_with("test")
def test_delete_ingestion(self):
testval = DotDict()
testval.o = 'sub_id'
self.rr_find_assocs.return_value = [testval]
self.ingestion_management.delete_ingestion_configuration('test')
self.rr_del_assoc.assert_called_once_with(testval)
self.pubsub_del_sub.assert_called_once_with(testval.o)
self.rr_delete.assert_called_once_with('test')
def setUp(self):
self.pd_service = ProcessDispatcherService()
self.pd_service.container = DotDict()
self.pd_service.container['spawn_process'] = Mock()
self.pd_service.container['id'] = 'mock_container_id'
self.pd_service.container['proc_manager'] = DotDict()
self.pd_service.container['resource_registry'] = Mock()
self.pd_service.container.proc_manager['terminate_process'] = Mock()
self.pd_service.container.proc_manager['procs'] = {}
pdcfg = dict(dashi_uri="amqp://hello",
dashi_exchange="123",
static_resources=True,
backend="native")
self.pd_service.CFG = DotDict()
self.pd_service.CFG['processdispatcher'] = pdcfg
self.mock_dashi = Mock()
with patch.multiple('ion.services.cei.process_dispatcher_service',
get_dashi=DEFAULT,
ProcessDispatcherCore=DEFAULT,
get_processdispatcher_store=DEFAULT,
EngineRegistry=DEFAULT,
PDMatchmaker=DEFAULT,
PDDoctor=DEFAULT) as mocks:
mocks['get_dashi'].return_value = self.mock_dashi
mocks[
'get_processdispatcher_store'].return_value = self.mock_store = Mock(
)
mocks[
'ProcessDispatcherCore'].return_value = self.mock_core = Mock(
)
mocks['PDMatchmaker'].return_value = self.mock_matchmaker = Mock()
mocks['PDDoctor'].return_value = self.mock_doctor = Mock()
mocks[
'EngineRegistry'].return_value = self.mock_engineregistry = Mock(
)
self.pd_service.init()
# replace the core and matchmaker with mocks
self.pd_service.backend.beat_subscriber = self.mock_beat_subscriber = Mock(
)
self.assertIsInstance(self.pd_service.backend, PDNativeBackend)
self.pd_service.backend.rr = self.mock_rr = Mock()
self.event_pub = Mock()
self.pd_service.backend.event_pub = self.event_pub
self.pd_service.start()
self.assertEqual(self.mock_dashi.handle.call_count, 1)
self.mock_matchmaker.start_election.assert_called_once_with()
self.mock_beat_subscriber.start.assert_called_once_with()
def test_dotdict_chaining(self):
base = DotDict({"test": None})
base.chained.example.provides.utility = True
self.assertTrue(base["chained"]["example"]["provides"]["utility"])
base.setting = True
self.assertTrue(base.setting)
self.assertTrue(base["setting"])
base.map = {"key": "value"}
self.assertIsInstance(base.map, DotDict, "%s" % type(base.map))
self.assertTrue(base.map.key == "value")
def _create_platform_config_builder(self):
clients = DotDict()
clients.resource_registry = self.RR
clients.pubsub_management = self.PSC
clients.dataset_management = self.DSC
pconfig_builder = PlatformAgentConfigurationBuilder(clients)
# can't do anything without an agent instance obj
log.debug("Testing that preparing a launcher without agent instance raises an error")
self.assertRaises(AssertionError, pconfig_builder.prepare, will_launch=False)
return pconfig_builder
def on_initial_bootstrap(self, process, config, **kwargs):
if config.get_safe('system.elasticsearch') and config.get_safe('bootstrap.use_es'):
#---------------------------------------------
# Spawn the index bootstrap
#---------------------------------------------
config = DotDict(config)
config.op = 'clean_bootstrap'
process.container.spawn_process('index_bootstrap','ion.processes.bootstrap.index_bootstrap','IndexBootStrap',config)
#---------------------------------------------
else:
log.info("Not creating the ES indexes.")
def recv_packet(self, msg, stream_route, stream_id):
'''
The callback method. For situations like bad or no data, publish an alert event.
@param msg granule
@param stream_route StreamRoute object
@param stream_id str
'''
log.debug("DemoStreamAlertTransform received a packet!: %s" % msg)
log.debug("type of packet received by transform: %s", type(msg))
#-------------------------------------------------------------------------------------
# Set up the config to use to pass info to the transform algorithm
#-------------------------------------------------------------------------------------
config = DotDict()
config.valid_values = self.valid_values
config.variable_name = self.instrument_variable_name
config.time_field_name = self.time_field_name
#-------------------------------------------------------------------------------------
# Store the granule received
#-------------------------------------------------------------------------------------
self.granules.put(msg)
#-------------------------------------------------------------------------------------
# Check for good and bad values in the granule
#-------------------------------------------------------------------------------------
bad_values, bad_value_times, self.origin = AlertTransformAlgorithm.execute(
msg, config=config)
log.debug(
"DemoStreamAlertTransform got the origin of the event as: %s" %
self.origin)
#-------------------------------------------------------------------------------------
# If there are any bad values, publish an alert event for the granule
#-------------------------------------------------------------------------------------
if bad_values:
# Publish the event
self.publisher.publish_event(
event_type='DeviceStatusEvent',
origin=self.origin,
origin_type='PlatformDevice',
sub_type=self.instrument_variable_name,
values=bad_values,
time_stamps=bad_value_times,
valid_values=self.valid_values,
state=DeviceStatusType.OUT_OF_RANGE,
description="Event to deliver the status of instrument.")
log.debug("DemoStreamAlertTransform published a BAD DATA event")
def setUp(self):
self.mock_real_client = DotDict()
self.mock_real_client.read_process_definition = Mock()
self.mock_real_client.create_process = Mock()
self.mock_real_client.schedule_process = Mock()
self.mock_real_client.read_process = Mock()
self.mock_eventpub = DotDict()
self.mock_eventpub.publish_event = Mock()
self.mock_container = Mock()
self.client = ProcessDispatcherSimpleAPIClient('fake',
real_client=self.mock_real_client, container=self.mock_container)
self.client.event_pub = self.mock_eventpub
def preload_sptest(self):
config = DotDict()
config.op = 'load'
config.loadui=True
config.ui_path = "http://userexperience.oceanobservatories.org/database-exports/Candidates"
config.attachments = "res/preload/r2_ioc/attachments"
config.scenario = 'BETA,SP_TEST'
config.path = 'master'
config.categories='ParameterFunctions,ParameterDefs,ParameterDictionary'
self.container.spawn_process('preloader', 'ion.processes.bootstrap.ion_loader', 'IONLoader', config)
def setUp(self):
self.cfg = DotDict()
self.cfg.eeagent = DotDict()
self.cfg.eeagent.heartbeat = 1
self.factory = Mock()
self.process_id = 'fake'
self.process = DotDict()
self.process._process = DotDict()
self.process._process.heartbeat = Mock(return_value=self.heartbeater_not_ok)
self.process._process.listeners = []
self.heartbeater = HeartBeater(self.cfg, self.factory, self.process_id, self.process)
def load_data_process(self, stream_id=""):
dpms_client = DataProcessManagementServiceClient()
dataprocess_details_list = dpms_client.read_data_process_for_stream(
stream_id)
dataprocess_ids = []
#this returns a list of data process info dicts
for dataprocess_details in dataprocess_details_list:
dataprocess_details = DotDict(dataprocess_details or {})
dataprocess_id = dataprocess_details.dataprocess_id
#set metrics attributes
dataprocess_details.granule_counter = 0
self._dataprocesses[dataprocess_id] = dataprocess_details
log.debug('load_data_process dataprocess_id: %s', dataprocess_id)
log.debug('load_data_process dataprocess_details: %s',
dataprocess_details)
# validate details
# if not outstream info avaialable log a warning but TF may publish an event so proceed
if not dataprocess_details.out_stream_def or not dataprocess_details.output_param:
log.warning(
'No output stream details provided for data process %s, will not publish a granule',
dataprocess_id)
#add the stream id to the map
if 'in_stream_id' in dataprocess_details:
if dataprocess_details['in_stream_id'] in self._streamid_map:
(self._streamid_map[dataprocess_details['in_stream_id']]
).append(dataprocess_id)
else:
self._streamid_map[dataprocess_details['in_stream_id']] = [
dataprocess_id
]
#todo: add transform worker id
self.event_publisher.publish_event(
origin=dataprocess_id,
origin_type='DataProcess',
status=DataProcessStatusType.NORMAL,
description='data process loaded into transform worker')
#create a publisher for output stream
self.create_publisher(dataprocess_id, dataprocess_details)
dataprocess_ids.append(dataprocess_id)
return dataprocess_ids
def recv_packet(self, msg, stream_route, stream_id):
'''
The callback method. For situations like bad or no data, publish an alert event.
@param msg granule
@param stream_route StreamRoute object
@param stream_id str
'''
log.debug("DemoStreamAlertTransform received a packet!: %s" % msg)
log.debug("type of packet received by transform: %s", type(msg))
#-------------------------------------------------------------------------------------
# Set up the config to use to pass info to the transform algorithm
#-------------------------------------------------------------------------------------
config = DotDict()
config.valid_values = self.valid_values
config.variable_name = self.instrument_variable_name
config.time_field_name = self.time_field_name
#-------------------------------------------------------------------------------------
# Store the granule received
#-------------------------------------------------------------------------------------
self.granules.put(msg)
#-------------------------------------------------------------------------------------
# Check for good and bad values in the granule
#-------------------------------------------------------------------------------------
bad_values, bad_value_times, self.origin = AlertTransformAlgorithm.execute(msg, config = config)
log.debug("DemoStreamAlertTransform got the origin of the event as: %s" % self.origin)
#-------------------------------------------------------------------------------------
# If there are any bad values, publish an alert event for the granule
#-------------------------------------------------------------------------------------
if bad_values:
# Publish the event
self.publisher.publish_event(
event_type = 'DeviceStatusEvent',
origin = self.origin,
origin_type='PlatformDevice',
sub_type = self.instrument_variable_name,
values = bad_values,
time_stamps = bad_value_times,
valid_values = self.valid_values,
state = DeviceStatusType.OUT_OF_RANGE,
description = "Event to deliver the status of instrument."
)
log.debug("DemoStreamAlertTransform published a BAD DATA event")
def test_schedule_haagent_name(self):
haa_proc_def = DotDict()
haa_proc_def['name'] = "haagent"
haa_proc_def['executable'] = {'module': 'my_module', 'class': 'class'}
payload_proc_def = DotDict()
payload_proc_def['name'] = "payload_process"
payload_proc_def['executable'] = {
'module': 'my_module',
'class': 'class'
}
proc_defs = {
"haa_proc_def_id": haa_proc_def,
"payload_proc_def_id": payload_proc_def
}
read_definition_mock = Mock()
read_definition_mock.side_effect = proc_defs.get
self.pd_service.backend.read_definition = read_definition_mock
# not used for anything in local mode
proc_schedule = DotDict()
configuration = {
"highavailability": {
"process_definition_id": "payload_proc_def_id"
}
}
self.pd_service.schedule_process("haa_proc_def_id", proc_schedule,
configuration)
self.assertEqual(self.mock_core.schedule_process.call_count, 1)
name = self.mock_core.schedule_process.call_args[1]['name']
self.assertTrue(name.startswith("payload_process-ha"))
# now try with scheduling by process definition name instead of ID
self.mock_core.schedule_process.reset_mock()
configuration = {
"highavailability": {
"process_definition_name": "payload_process"
}
}
self.pd_service.schedule_process("haa_proc_def_id", proc_schedule,
configuration)
self.assertEqual(self.mock_core.schedule_process.call_count, 1)
name = self.mock_core.schedule_process.call_args[1]['name']
self.assertTrue(name.startswith("payload_process-ha"))
def test_index_bootstrap(self, mock_es, ims_cli):
#---------------------------------------------
# Mocks
#---------------------------------------------
mock_es().index_create.return_value = {'ok': True, 'status': 200}
mock_es().raw.return_value = {'ok': True, 'status': 200}
mock_es().river_couchdb_create.return_value = {
'ok': True,
'status': 200
}
db = DotDict()
db.datastore_name = 'test'
db.server.test.create = Mock()
container = DotDict()
container.datastore_manager.get_datastore = Mock()
container.datastore_manager.get_datastore.return_value = db
config = CFG
config.system.force_clean = False
config.system.elasticsearch = True
config.server.elasticsearch.host = ''
config.server.elasticsearch.port = ''
config.op = 'index_bootstrap'
#---------------------------------------------
# Execution
#---------------------------------------------
ibs = IndexBootStrap()
ibs.CFG = config
ibs.container = container
ibs.on_start()
index_count = len(STD_INDEXES) + len(EDGE_INDEXES) + 1 # for _river
self.assertTrue(
mock_es().index_create.call_count == index_count,
'(%s != %s) Improper number of indices created' %
(mock_es().index_create.call_count, index_count))
river_count = len(STD_INDEXES) + len(EDGE_INDEXES)
self.assertTrue(
mock_es().river_couchdb_create.call_count == river_count,
'Improper number of rivers created')
total_count = len(STD_INDEXES) + len(COUCHDB_INDEXES) + len(
EDGE_INDEXES)
self.assertTrue(ims_cli().create_index.call_count == total_count,
'Improper number of index resources created')
def test_cached_predicate_search(self):
d = "d_id"
m = "m_id"
x = "x_id"
good_assn = DotDict(s=d,
st=RT.InstrumentDevice,
p=PRED.hasModel,
o=m,
ot=RT.InstrumentModel)
bad_assn = DotDict(s=d,
st=RT.PlatformDevice,
p=PRED.hasModel,
o=m,
ot=RT.PlatformModel)
self.rr.find_associations.return_value = [good_assn, bad_assn]
self.RR2.cache_predicate(PRED.hasModel)
self.assertTrue(self.RR2.has_cached_predicate(PRED.hasModel))
self.rr.find_associations.assert_called_once_with(
predicate=PRED.hasModel, id_only=False)
# object searches that should return 0, 0, 1 results
results = self.RR2.find_objects(x, PRED.hasModel, RT.InstrumentModel,
True)
self.assertEqual([], results)
results = self.RR2.find_instrument_model_ids_of_instrument_device_using_has_model(
x)
self.assertEqual([], results)
results = self.RR2.find_instrument_model_ids_of_instrument_device_using_has_model(
d)
self.assertEqual([m], results)
self.assertEqual(0, self.rr.find_objects.call_count)
# subject searches that should return 0, 0, 1 results
results = self.RR2.find_subjects(RT.InstrumentDevice, PRED.hasModel, x,
True)
self.assertEqual([], results)
results = self.RR2.find_instrument_device_ids_by_instrument_model_using_has_model(
x)
self.assertEqual([], results)
results = self.RR2.find_instrument_device_ids_by_instrument_model_using_has_model(
m)
self.assertEqual([d], results)
self.assertEqual(0, self.rr.find_subjects.call_count)
