def _start_event_dispatch(self):
"""
Registers the event listener by using a URL that is composed from
CFG.server.oms.host, CFG.server.oms.port, and CFG.server.oms.path.
NOTE: the same listener URL will be registered by multiple RSN platform
drivers. See other related notes in this file.
@see https://jira.oceanobservatories.org/tasks/browse/OOIION-1287
@see https://jira.oceanobservatories.org/tasks/browse/OOIION-968
"""
# gateway host and port to compose URL:
host = CFG.get_safe('server.oms.host', "localhost")
port = CFG.get_safe('server.oms.port', "5000")
path = CFG.get_safe('server.oms.path', "/ion-service/oms_event")
self.listener_url = "http://%s:%s%s" % (host, port, path)
self._register_event_listener(self.listener_url)
# start OMSDeviceStatusEvent listener to notify the agent about those:
def event_received(evt, *args, **kwargs):
log.debug('%r: OmsEventListener received: %s', self._platform_id, evt)
self._send_event(ExternalEventDriverEvent(evt))
self._event_listener = self._create_event_subscriber(
event_type = 'OMSDeviceStatusEvent',
origin = self._platform_id,
origin_type = 'OMS Platform',
callback = event_received)
log.debug("%r: started OMSDeviceStatusEvent listener", self._platform_id)
return "OK"
def setting_up_smtp_client():
"""
Sets up the smtp client
"""
#------------------------------------------------------------------------------------
# the default smtp server
#------------------------------------------------------------------------------------
# smtp_client = None
smtp_host = CFG.get_safe('server.smtp.host')
smtp_port = CFG.get_safe('server.smtp.port', 25)
# smtp_sender = CFG.get_safe('server.smtp.sender')
# smtp_password = CFG.get_safe('server.smtp.password')
if CFG.get_safe('system.smtp',False): #Default is False - use the fake_smtp
log.debug('Using the real SMTP library to send email notifications! host = %s', smtp_host)
# smtp_client = smtplib.SMTP(smtp_host)
# smtp_client.ehlo()
# smtp_client.starttls()
# smtp_client.login(smtp_sender, smtp_password)
smtp_client = smtplib.SMTP(smtp_host, smtp_port)
log.debug("In setting up smtp client using the smtp client: %s", smtp_client)
log.debug("Message received after ehlo exchange: %s", str(smtp_client.ehlo()))
# smtp_client.login(smtp_sender, smtp_password)
else:
log.debug('Using a fake SMTP library to simulate email notifications!')
smtp_client = fake_smtplib.SMTP(smtp_host)
return smtp_client
def setUp(self):
super(DiscoveryIntTest, self).setUp()
self._start_container()
self.addCleanup(DiscoveryIntTest.es_cleanup)
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.discovery = DiscoveryServiceClient()
self.catalog = CatalogManagementServiceClient()
self.ims = IndexManagementServiceClient()
self.rr = ResourceRegistryServiceClient()
if use_es:
self.es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
self.es_port = CFG.get_safe('server.elasticsearch.port', '9200')
CFG.server.elasticsearch.shards = 1
CFG.server.elasticsearch.replicas = 0
CFG.server.elasticsearch.river_shards = 1
CFG.server.elasticsearch.river_replicas = 0
self.es = ep.ElasticSearch(
host=self.es_host,
port=self.es_port,
timeout=10,
verbose=True
)
op = DotDict(CFG)
op.op = 'clean_bootstrap'
self.container.spawn_process('index_bootstrap','ion.processes.bootstrap.index_bootstrap','IndexBootStrap', op)
def on_start(self):
self.data_source_subscriber = EventSubscriber(
event_type=OT.ResourceModifiedEvent,
origin_type=RT.DataSource,
callback=self._register_data_source)
self.provider_subscriber = EventSubscriber(
event_type=OT.ResourceModifiedEvent,
origin_type=RT.ExternalDataProvider,
callback=self._register_provider)
self.data_source_subscriber.start()
self.provider_subscriber.start()
self.rr = self.container.resource_registry
self.using_eoi_services = CFG.get_safe('eoi.meta.use_eoi_services',
False)
self.server = CFG.get_safe(
'eoi.importer_service.server', "localhost") + ":" + str(
CFG.get_safe('eoi.importer_service.port', 8844))
log.info("Using geoservices=" + str(self.using_eoi_services))
if not self.using_eoi_services:
log.warn("not using geoservices...")
self.importer_service_available = self.check_for_importer_service()
if not self.importer_service_available:
log.warn("not using importer service...")
def on_init(self):
log.info("Ingestion starting")
self.exchange_name = "ingestion_process"
plugin_cls = CFG.get_safe(CONFIG_KEY + ".plugin")
self.plugin = named_any(plugin_cls)(self)
log.info("Started ingestion plugin '%s'", plugin_cls)
self.persistence_formats = {}
self.persistence_objects = {}
self.default_persistence_format = CFG.get_safe(CONFIG_KEY + ".persist.persistence_format")
self._require_persistence_layer(self.default_persistence_format)
self.stream_sub = StreamSubscriber(process=self, exchange_name=self.exchange_name,
callback=self.process_package)
streams = CFG.get_safe(CONFIG_KEY + ".stream_subscriptions") or []
for stream in streams:
if isinstance(stream, list):
stream = StreamRoute(exchange_point=stream[0], routing_key=stream[1])
log.info("Ingestion subscribed to stream '%s'", stream)
self.stream_sub.add_stream_subscription(stream)
self.plugin.on_init()
self.stream_sub.start()
def _start_event_dispatch(self):
"""
Registers the event listener by using a URL that is composed from
CFG.server.oms.host, CFG.server.oms.port, and CFG.server.oms.path.
NOTE: the same listener URL will be registered by multiple RSN platform
drivers. See other related notes in this file.
@see https://jira.oceanobservatories.org/tasks/browse/OOIION-1287
@see https://jira.oceanobservatories.org/tasks/browse/OOIION-968
"""
# gateway host and port to compose URL:
host = CFG.get_safe('server.oms.host', "localhost")
port = CFG.get_safe('server.oms.port', "5000")
path = CFG.get_safe('server.oms.path', "/ion-service/oms_event")
self.listener_url = "http://%s:%s%s" % (host, port, path)
self._register_event_listener(self.listener_url)
# start OMSDeviceStatusEvent listener to notify the agent about those:
def event_received(evt, *args, **kwargs):
log.debug('%r: OmsEventListener received: %s', self._platform_id, evt)
self._send_event(ExternalEventDriverEvent(evt))
self._event_listener = self._create_event_subscriber(
event_type = 'OMSDeviceStatusEvent',
origin = self._platform_id,
origin_type = 'OMS Platform',
callback = event_received)
log.debug("%r: started OMSDeviceStatusEvent listener", self._platform_id)
return "OK"
def setUp(self):
super(DiscoveryIntTest, self).setUp()
self._start_container()
self.addCleanup(DiscoveryIntTest.es_cleanup)
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.discovery = DiscoveryServiceClient()
self.catalog = CatalogManagementServiceClient()
self.ims = IndexManagementServiceClient()
self.rr = ResourceRegistryServiceClient()
if use_es:
self.es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
self.es_port = CFG.get_safe('server.elasticsearch.port', '9200')
CFG.server.elasticsearch.shards = 1
CFG.server.elasticsearch.replicas = 0
CFG.server.elasticsearch.river_shards = 1
CFG.server.elasticsearch.river_replicas = 0
self.es = ep.ElasticSearch(
host=self.es_host,
port=self.es_port,
timeout=10,
verbose=True
)
op = DotDict(CFG)
op.op = 'clean_bootstrap'
self.container.spawn_process('index_bootstrap','ion.processes.bootstrap.index_bootstrap','IndexBootStrap', op)
def do_auto_preload(self):
""" Load configured preload scenarios and remembers in directory which ones
have been loaded. """
log.info("Executing auto preload")
preload_entries = self.process.container.directory.lookup("/System/Preload")
preload_changed = False
if preload_entries is None:
preload_entries = dict(scenarios={})
preload_changed = True
# "HACK" Preload core to get app specifics in
if "core" not in preload_entries["scenarios"]:
preload_entries["scenarios"]["core"] = self._get_dir_entry("core")
self.do_preload_core()
preload_changed = True
preload_scenarios = CFG.get_safe("scion.preload.scenarios") or []
preload_scope = CFG.get_safe("scion.preload.scope")
for scenario_info in preload_scenarios:
scope, scenario = scenario_info
if scope == "all" or scope == preload_scope:
if scenario not in preload_entries["scenarios"]:
preload_entries["scenarios"][scenario] = self._get_dir_entry(scenario)
changed = self.do_preload_master(scenario)
preload_changed = preload_changed or changed
if preload_changed:
self.process.container.directory.register("/System", "Preload", scenarios=preload_entries["scenarios"])
def setting_up_smtp_client():
'''
Sets up the smtp client
'''
#------------------------------------------------------------------------------------
# the default smtp server
#------------------------------------------------------------------------------------
ION_SMTP_SERVER = 'mail.oceanobservatories.org'
smtp_host = CFG.get_safe('server.smtp.host', ION_SMTP_SERVER)
smtp_port = CFG.get_safe('server.smtp.port', 25)
smtp_sender = CFG.get_safe('server.smtp.sender')
smtp_password = CFG.get_safe('server.smtp.password')
if CFG.get_safe('system.smtp',False): #Default is False - use the fake_smtp
log.debug('Using the real SMTP library to send email notifications!')
smtp_client = smtplib.SMTP(smtp_host)
smtp_client.ehlo()
smtp_client.starttls()
smtp_client.login(smtp_sender, smtp_password)
else:
log.debug('Using a fake SMTP library to simulate email notifications!')
smtp_client = fake_smtplib.SMTP(smtp_host)
return smtp_client
def _start_event_dispatch(self):
"""
Registers the event listener by using a URL that is composed from
CFG.server.oms.host, CFG.server.oms.port, and CFG.server.oms.path.
NOTE: the same listener URL will be registered by multiple RSN platform
drivers. See other related notes in this file.
@see https://jira.oceanobservatories.org/tasks/browse/OOIION-1287
@see https://jira.oceanobservatories.org/tasks/browse/OOIION-968
"""
# gateway host and port to compose URL:
host = CFG.get_safe('server.oms.host', "localhost")
port = CFG.get_safe('server.oms.port', "5000")
path = CFG.get_safe('server.oms.path', "/ion-service/oms_event")
#the above are defined in pyon.cfg
#we will override local host for debugging inside the VM
host = "10.208.79.19"
self.listener_url = "http://%s:%s%s" % (host, port, path)
self._register_event_listener(self.listener_url)
return "OK"
def test_pydap(self):
if not CFG.get_safe('bootstrap.use_pydap',False):
raise unittest.SkipTest('PyDAP is off (bootstrap.use_pydap)')
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition('example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
tdom, sdom = time_series_domain()
dp = DataProduct(name='example')
dp.spatial_domain = sdom.dump()
dp.temporal_domain = tdom.dump()
data_product_id = self.data_product_management.create_data_product(dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product, data_product_id)
self.data_product_management.activate_data_product_persistence(data_product_id)
self.addCleanup(self.data_product_management.suspend_data_product_persistence, data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt,10)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.event.wait(10))
gevent.sleep(1) # Yield to other greenlets, had an issue with connectivity
pydap_host = CFG.get_safe('server.pydap.host','localhost')
pydap_port = CFG.get_safe('server.pydap.port',8001)
url = 'http://%s:%s/%s' %(pydap_host, pydap_port, dataset_id)
ds = open_url(url)
np.testing.assert_array_equal(ds['time'][:], np.arange(10))
untested = []
for k,v in rdt.iteritems():
if k==rdt.temporal_parameter:
continue
context = rdt.context(k)
if isinstance(context.param_type, QuantityType):
np.testing.assert_array_equal(ds[k][k][:][0], rdt[k])
elif isinstance(context.param_type, ArrayType):
values = np.empty(rdt[k].shape, dtype='O')
for i,obj in enumerate(rdt[k]):
values[i] = str(obj)
np.testing.assert_array_equal(ds[k][k][:][0], values)
elif isinstance(context.param_type, ConstantType):
np.testing.assert_array_equal(ds[k][k][:][0], rdt[k])
elif isinstance(context.param_type, CategoryType):
np.testing.assert_array_equal(ds[k][k][:][0], rdt[k])
else:
untested.append('%s (%s)' % (k,context.param_type))
if untested:
raise AssertionError('Untested parameters: %s' % untested)
def on_start(self): # pragma no cover
super(DiscoveryService,self).on_start()
self.use_es = CFG.get_safe('system.elasticsearch',False)
self.elasticsearch_host = CFG.get_safe('server.elasticsearch.host','localhost')
self.elasticsearch_port = CFG.get_safe('server.elasticsearch.port','9200')
self.ep = EventPublisher(event_type = 'SearchBufferExceededEvent')
self.heuristic_cutoff = 4
def on_start(self): # pragma no cover
super(DiscoveryService,self).on_start()
self.use_es = CFG.get_safe('system.elasticsearch',False)
self.elasticsearch_host = CFG.get_safe('server.elasticsearch.host','localhost')
self.elasticsearch_port = CFG.get_safe('server.elasticsearch.port','9200')
self.ep = EventPublisher(event_type = 'SearchBufferExceededEvent')
self.heuristic_cutoff = 4
def on_start(self): # pragma: no cover
super(DiscoveryService, self).on_start()
self.use_es = CFG.get_safe("system.elasticsearch", False)
self.elasticsearch_host = CFG.get_safe("server.elasticsearch.host", "localhost")
self.elasticsearch_port = CFG.get_safe("server.elasticsearch.port", "9200")
self.ep = EventPublisher(event_type="SearchBufferExceededEvent")
self.heuristic_cutoff = 4
def es_cleanup():
es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
es_port = CFG.get_safe('server.elasticsearch.port', '9200')
es = ep.ElasticSearch(host=es_host, port=es_port, timeout=10)
indexes = STD_INDEXES.keys()
indexes.append('%s_resources_index' % get_sys_name().lower())
indexes.append('%s_events_index' % get_sys_name().lower())
for index in indexes:
IndexManagementService._es_call(es.river_couchdb_delete, index)
IndexManagementService._es_call(es.index_delete, index)
def es_cleanup():
es_host = CFG.get_safe("server.elasticsearch.host", "localhost")
es_port = CFG.get_safe("server.elasticsearch.port", "9200")
es = ep.ElasticSearch(host=es_host, port=es_port, timeout=10)
indexes = STD_INDEXES.keys()
indexes.append("%s_resources_index" % get_sys_name().lower())
indexes.append("%s_events_index" % get_sys_name().lower())
for index in indexes:
IndexManagementService._es_call(es.river_couchdb_delete, index)
IndexManagementService._es_call(es.index_delete, index)
def test_ccov_visualization(self):
'''
Tests Complex Coverage aggregation of array types and proper splitting of coverages
tests pydap and the visualization
'''
if not CFG.get_safe('bootstrap.use_pydap',False):
raise unittest.SkipTest('PyDAP is off (bootstrap.use_pydap)')
data_product_id, stream_def_id = self.make_array_data_product()
# Make a granule with an array type, give it a few values
# Send it to google_dt transform, verify output
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(2208988800, 2208988810)
rdt['temp_sample'] = np.arange(10*4).reshape(10,4)
rdt['cond_sample'] = np.arange(10*4).reshape(10,4)
dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(data_product_id)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
self.ph.publish_rdt_to_data_product(data_product_id, rdt, connection_id='abc1', connection_index='1')
self.assertTrue(dataset_monitor.event.wait(10))
dataset_monitor.event.clear()
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(2208988810, 2208988820)
rdt['temp_sample'] = np.arange(10*4).reshape(10,4)
rdt['cond_sample'] = np.arange(10*4).reshape(10,4)
self.ph.publish_rdt_to_data_product(data_product_id, rdt, connection_id='abc2', connection_index='1')
self.assertTrue(dataset_monitor.event.wait(10))
dataset_monitor.event.clear()
qstring = '{"stride_time": 1, "parameters": [], "query_type": "highcharts_data", "start_time": 0, "use_direct_access": 0, "end_time": 19}'
graph = self.visualization.get_visualization_data(data_product_id, qstring)
self.assertIn('temp_sample[3]', graph)
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['temp_sample'][0], np.arange(4))
pydap_host = CFG.get_safe('server.pydap.host','localhost')
pydap_port = CFG.get_safe('server.pydap.port',8001)
url = 'http://%s:%s/%s' %(pydap_host, pydap_port, dataset_id)
ds = open_url(url)
temp_sample, time = ds['temp_sample']
temp_values, dim = temp_sample[0]
np.testing.assert_array_equal(temp_values, np.array(['0.0,1.0,2.0,3.0']))
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
path = CFG.get_safe('server.pydap.data_path', "RESOURCE:ext/pydap")
ext_path = FileSystem.get_extended_url(path)
self.cov,self.filename = _make_coverage(ext_path)
self.nt = 5
self.cov.insert_timesteps(self.nt)
self.time_data = [i+1 for i in range(self.nt)]
self.cov.set_parameter_values("time", value=self.time_data)
host = CFG.get_safe('container.pydap_gateway.web_server.host', 'localhost')
port = CFG.get_safe('container.pydap_gateway.web_server.port', '8001')
self.request_url = "http://"+host+":"+str(port)+os.sep+os.path.basename(self.filename)
def process_query():
'''
Processes a query from the user
'''
elasticsearch_host = CFG.get_safe('server.elasticsearch.host','localhost')
elasticsearch_port = CFG.get_safe('server.elasticsearch.port','9200')
try:
import requests
data = request.stream.read()
# pass thru
r = requests.post('http://%s:%s/_search' % (elasticsearch_host, elasticsearch_port), data=data, headers={'content-type':'application/json'})
return r.content
except Exception as e:
return build_error_page(traceback.format_exc())
def on_init(self):
# clients
self.resource_registry = self.container.resource_registry
self.event_publisher = EventPublisher(OT.NotificationSentEvent)
# SMTP client configurations
self.smtp_from = CFG.get_safe('server.smtp.from', '*****@*****.**')
self.smtp_host = CFG.get_safe('server.smtp.host', 'localhost')
self.smtp_port = CFG.get_safe('server.smtp.port', 25)
# Jinja2 template environment
self.jinja_env = Environment(loader=FileSystemLoader('res/templates'), trim_blocks=True, lstrip_blocks=True)
super(NotificationWorker, self).on_init()
def on_start(self): # pragma no cover
super(DiscoveryService,self).on_start()
self.use_es = CFG.get_safe('system.elasticsearch', False)
self.elasticsearch_host = CFG.get_safe('server.elasticsearch.host','localhost')
self.elasticsearch_port = CFG.get_safe('server.elasticsearch.port','9200')
self.ep = EventPublisher(event_type = 'SearchBufferExceededEvent')
self.heuristic_cutoff = 4
self.cfg_datastore = CFG.get_safe('container.datastore.default_server', "couchdb")
self.ds_discovery = None
if self.cfg_datastore != "couchdb":
self.ds_discovery = DatastoreDiscovery(self)
def on_init(self):
self.http_server = None
self.server_hostname = self.CFG.get_safe(CFG_PREFIX + '.web_server.hostname', DEFAULT_WEB_SERVER_HOSTNAME)
self.server_port = self.CFG.get_safe(CFG_PREFIX + '.web_server.port', DEFAULT_WEB_SERVER_PORT)
self.url_prefix = self.CFG.get_safe(CFG_PREFIX + '.url_prefix') or ""
self.web_server_enabled = True
self.logging = None
self.interaction_observer = None
self.plugin = None
app.secret_key = self.__class__.__name__ # Enables sessions (for mscweb)
#retain a pointer to this object for use in ProcessRPC calls
global adminui_instance
adminui_instance = self
#Start the gevent web server unless disabled
if self.web_server_enabled:
self.start_service(self.server_hostname, self.server_port)
plugin_cls = CFG.get_safe(CFG_PREFIX + '.plugin')
if plugin_cls:
cls = named_any(plugin_cls)
self.plugin = cls(app, self)
def send_email(event, msg_recipient, smtp_client, rr_client):
"""
A common method to send email with formatting
@param event Event
@param msg_recipient str
@param smtp_client fake or real smtp client object
"""
#log.debug("Got type of event to notify on: %s", event.type_)
#------------------------------------------------------------------------------------
# the 'from' email address for notification emails
#------------------------------------------------------------------------------------
ION_NOTIFICATION_EMAIL_ADDRESS = '*****@*****.**'
smtp_sender = CFG.get_safe('server.smtp.sender', ION_NOTIFICATION_EMAIL_ADDRESS)
msg = convert_events_to_email_message([event], rr_client)
msg['From'] = smtp_sender
msg['To'] = msg_recipient
log.debug("UNS sending email from %s to %s for event type: %s", smtp_sender,msg_recipient, event.type_)
#log.debug("UNS using the smtp client: %s", smtp_client)
try:
smtp_client.sendmail(smtp_sender, [msg_recipient], msg.as_string())
except: # Can be due to a broken connection... try to create a connection
smtp_client = setting_up_smtp_client()
log.debug("Connect again...message received after ehlo exchange: %s", str(smtp_client.ehlo()))
smtp_client.sendmail(smtp_sender, [msg_recipient], msg.as_string())
def _gen_service_spec(self, service_name):
output = {
"swagger": "2.0",
"info": {},
"host": CFG.get_safe("system.web_ui_url", "").rstrip("/").split("//", 1)[1],
"basePath": CFG.get_safe(UISG_CFG_PREFIX + ".url_prefix"),
"schemes": ["http"],
}
output["info"].update(self.config.get("info", {}))
if self.config.get("externalDocs", {}):
output["externalDocs"] = self.config["externalDocs"]
output["tags"] = self._gen_tags(service_name)
output["paths"] = self._gen_paths(service_name)
return output
def on_init(self):
#get the fields from the header that need to be logged
self.header_fields = CFG.get_safe(
'container.messaging.conversation_log.header_fields', [])
# Time in between event persists
self.persist_interval = 1.0
# Holds received events FIFO
self.conv_queue = Queue()
# Temporarily holds list of events to persist while datastore operation not yet completed
self.convs_to_persist = None
# bookkeeping for timeout greenlet
self._persist_greenlet = None
self._terminate_persist = Event(
) # when set, exits the timeout greenlet
# The event subscriber
self.conv_sub = None
# The conv repository
self.conv_repository = None
def _get_cov(self, name, nt):
path = CFG.get_safe('server.pydap.data_path', "RESOURCE:ext/pydap")
ext_path = FileSystem.get_extended_url(path)
cov,filename = _make_coverage(ext_path, "the_cov")
cov.insert_timesteps(nt)
cov.set_parameter_values("time", value=nt)
return cov, filename
def on_start_streaming(self, streaming_args):
log.info('Orb_DataAgentPlugin..on_start_streaming: args %s' % str(streaming_args))
self.streaming_args = streaming_args
script_path = os.path.dirname(__file__) + "/orbstart.py" # On production, run from inside an egg
data_dir = Container.instance.file_system.get("TEMP/orb_data")
cmd_args = ['orb_reap', script_path, streaming_args['orb_name'],
streaming_args['select'], "--datadir", data_dir]
if 'reject' in streaming_args:
cmd_args.append('--reject')
cmd_args.append(streaming_args['reject'])
if 'after' in streaming_args:
cmd_args.append('--after')
cmd_args.append(streaming_args['after'])
if 'timeout' in streaming_args:
cmd_args.append('--timeout')
cmd_args.append(streaming_args['timeout'])
if 'qsize' in streaming_args:
cmd_args.append('--qsize')
cmd_args.append(streaming_args['qsize'])
log.info('Orb reap args: ' + str(cmd_args))
self.data_dir = data_dir + '/%s/' % (streaming_args['select'].replace('/', '-'))
if os.path.exists(self.data_dir):
shutil.rmtree(self.data_dir)
antelope_path = CFG.get_safe("scion.antelope.path", "/opt/antelope/5.6")
cmd_args = [str(arg) for arg in cmd_args]
self.proc = subprocess.Popen(cmd_args, executable=antelope_path+'/bin/python')
log.info('Orb reap process started, %i' % self.proc.pid)
def _load_system_actors(self):
"""
Retrieve system and webauth actors and headers for later use.
"""
# Retrieve and store system actor and headers.
system_actor, _ = self.container.resource_registry.find_resources(
RT.ActorIdentity,
name=CFG.system.system_actor,
id_only=False)
self.system_actor = system_actor[0] if system_actor else None
self.system_actor_id = system_actor[0]._id if system_actor \
else 'anonymous'
self.system_actor_headers = {
'ion-actor-id': self.system_actor_id,
'ion-actor-roles': {'ION': ['ION_MANAGER', 'ORG_MANAGER']},
'expiry':'0'
}
# Retrieve and store webauth actor and headers.
webauth_actor, _ = self.container.resource_registry.find_resources(
RT.ActorIdentity,
name=CFG.get_safe("system.web_authentication_actor",
"web_authentication"), id_only=False)
self.webauth_actor = webauth_actor[0] if webauth_actor else None
self.webauth_actor_id = webauth_actor[0]._id if webauth_actor \
else 'anonymous'
self.webauth_actor_headers = {
'ion-actor-id': self.webauth_actor_id,
'ion-actor-roles': {'ION': ['ION_MANAGER', 'ORG_MANAGER']},
'expiry':'0'
}
def install_terminate_handler():
import gevent
from pyon.core.bootstrap import CFG
shutdown_timeout = int(CFG.get_safe("container.timeout.shutdown") or 0)
terminate_gl = None
def cancel_func():
if terminate_gl:
terminate_gl.kill()
if shutdown_timeout > 0:
pid = os.getpid()
def terminate_unresponsive():
print >> sys.stderr, "ERROR: Container shutdown seems unresponsive. Timeout elapsed (%s sec) -- TERMINATE process (%s)" % (
shutdown_timeout, pid)
os.kill(pid, signal.SIGKILL)
terminate_gl = gevent.spawn_later(shutdown_timeout,
terminate_unresponsive)
terminate_gl._glname = "Container terminate timeout"
log.info(
"Container termination timeout set to %s sec (process %s)",
shutdown_timeout, pid)
return cancel_func
def on_start_streaming(self, streaming_args):
log.info('Orb_DataAgentPlugin..on_start_streaming: args %s' %
str(streaming_args))
self.streaming_args = streaming_args
script_path = os.path.dirname(
__file__) + "/orbstart.py" # On production, run from inside an egg
data_dir = Container.instance.file_system.get("TEMP/orb_data")
cmd_args = [
'orb_reap', script_path, streaming_args['orb_name'],
streaming_args['select'], "--datadir", data_dir
]
if 'reject' in streaming_args:
cmd_args.append('--reject')
cmd_args.append(streaming_args['reject'])
if 'after' in streaming_args:
cmd_args.append('--after')
cmd_args.append(streaming_args['after'])
if 'timeout' in streaming_args:
cmd_args.append('--timeout')
cmd_args.append(streaming_args['timeout'])
if 'qsize' in streaming_args:
cmd_args.append('--qsize')
cmd_args.append(streaming_args['qsize'])
log.info('Orb reap args: ' + str(cmd_args))
self.data_dir = data_dir + '/%s/' % (streaming_args['select'].replace(
'/', '-'))
if os.path.exists(self.data_dir):
shutil.rmtree(self.data_dir)
antelope_path = CFG.get_safe("scion.antelope.path",
"/opt/antelope/5.6")
cmd_args = [str(arg) for arg in cmd_args]
self.proc = subprocess.Popen(cmd_args,
executable=antelope_path + '/bin/python')
log.info('Orb reap process started, %i' % self.proc.pid)
def on_initial_bootstrap(self, process, config, **kwargs):
if os.environ.get('PYCC_MODE'):
# This environment is an ion integration test
log.info('PYCC_MODE: skipping qc_post_processor launch')
return
if self.process_exists(process, 'qc_post_processor'):
# Short circuit the bootstrap to make sure not more than one is ever started
return
self.scheduler_service = SchedulerServiceProcessClient(process=process)
self.process_dispatcher = ProcessDispatcherServiceProcessClient(process=process)
self.run_interval = CFG.get_safe('service.qc_processing.run_interval', 24)
interval_key = uuid4().hex # Unique identifier for this process
config = DotDict()
config.process.interval_key = interval_key
process_definition = ProcessDefinition(name='qc_post_processor',
executable={'module':'ion.processes.data.transforms.qc_post_processing', 'class':'QCPostProcessing'})
process_definition_id = self.process_dispatcher.create_process_definition(process_definition)
process_id = self.process_dispatcher.create_process(process_definition_id)
self.process_dispatcher.schedule_process(process_definition_id, process_id=process_id, configuration=config)
timer_id = self.scheduler_service.create_interval_timer(start_time=str(time.time()),
end_time='-1', #Run FOREVER
interval=3600*self.run_interval,
event_origin=interval_key)
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Instantiate a process to represent the test
process = TransformWorkerTestProcess()
self.dataset_management_client = DatasetManagementServiceClient(
node=self.container.node)
self.pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceProcessClient(
node=self.container.node, process=process)
self.time_dom, self.spatial_dom = time_series_domain()
self.ph = ParameterHelper(self.dataset_management_client,
self.addCleanup)
self.wait_time = CFG.get_safe('endpoint.receive.timeout', 10)
def process_index():
try:
from pyon.public import CFG
from pyon.core.bootstrap import get_sys_name
default_ds_server = CFG.get_safe("container.datastore.default_server", "postgresql")
fragments = [
"<h1>SciON Admin UI</h1>",
"<p><ul>",
"<li><a href='/restypes'><b>Browse Resource Registry and Resource Objects</b></a>",
"<ul>",
"<li>Org/Users: <a href='/list/Org'>Org</a>, <a href='/list/UserRole'>UserRole</a>, <a href='/list/ActorIdentity'>ActorIdentity</a></li>",
"<li>Computing: <a href='/list/Process'>Process</a>, <a href='/list/ProcessDefinition'>ProcessDefinition</a>, <a href='/list/Service'>Service</a>, <a href='/list/ServiceDefinition'>ServiceDefinition</a>, <a href='/list/CapabilityContainer'>CapabilityContainer</a></li>",
"<li>Messaging: <a href='/list/ExchangeSpace'>ExchangeSpace</a>, <a href='/list/ExchangePoint'>ExchangePoint</a>, <a href='/list/ExchangeName'>ExchangeName</a>, <a href='/list/ExchangeBroker'>ExchangeBroker</a></li>",
"<li>Governance: <a href='/list/Commitment'>Commitment</a>, <a href='/list/Negotiation'>Negotiation</a>, <a href='/list/Policy'>Policy</a></li>",
_get_resmenu_extension(),
"</ul></li>",
"<li><a href='/events'><b>Browse Events</b></a></li>",
"<li><a href='/viewobj'><b>View Objects</b></a></li>",
"<li><a href='/viewstate'><b>View Process State</b></a></li>",
"<li><a href='/dir'><b>Browse SciON Directory</b></a></li>",
#"<li><a href='/mscweb'><b>Show system messages (MSCWeb)</b></a>",
#"<ul>",
#"<li><a href='/mscaction/stop'>Stop system message recording</a></li>",
#"</ul></li>",
"<li><a href='http://localhost:4000'><b>Application Web UI (if running)</b></a></li>",
"<li><a href='http://" + CFG.get_safe("server.amqp.host") + ":15672/'><b>RabbitMQ Management UI (if running)</b></a></li>",
"<li><a href='http://localhost:9001/'><b>Supervisord UI (if running)</b></a></li>",
"</ul></p>",
"<h2>System and Container Properties</h2>",
"<p><table>",
"<tr><th>Property</th><th>Value</th></tr>",
"<tr><td>system.name (sysname)</td><td>%s</td></tr>" % get_sys_name(),
"<tr><td>Message Broker</td><td>%s</td></tr>" % "%s:%s" % (CFG.server.amqp.host, CFG.server.amqp.port),
"<tr><td>Database</td><td>%s</td></tr>" % "%s:%s" % (CFG.get_safe("server.%s.host" % default_ds_server), CFG.get_safe("server.%s.port" % default_ds_server)),
"<tr><td>Container ID</td><td>%s</td></tr>" % Container.instance.id,
"<tr><td>Read-Only</td><td>%s</td></tr>" % is_read_only(),
"</table></p>",
]
content = "\n".join(fragments)
return build_page(content)
except Exception as e:
return build_error_page(traceback.format_exc())
def do_work(container):
"""
Performs initial startup actions with the container as requested in arguments.
Then remains in container shell or infinite wait until container stops.
Returns when container should stop. Raises an exception if anything failed.
"""
if opts.proc:
# Run a one-off process (with the -x argument)
mod, proc = opts.proc.rsplit('.', 1)
print "pycc: Starting process %s" % opts.proc
container.spawn_process(proc,
mod,
proc,
config={'process': {
'type': 'immediate'
}})
# And end
return
if opts.rel:
# Start a rel file
start_ok = container.start_rel_from_url(opts.rel)
if not start_ok:
raise Exception("Cannot start deploy file '%s'" % opts.rel)
if opts.mx:
from pyon.public import CFG
port = CFG.get_safe('container.flask_webapp.port', 8080)
container.spawn_process("ContainerUI", "ion.core.containerui",
"ContainerUI")
print "pycc: Container UI started ... listening on http://localhost:%s" % port
if opts.signalparent:
import os
import signal
print 'pycc: Signal parent pid %d that pycc pid %d service start process is complete...' % (
os.getppid(), os.getpid())
os.kill(os.getppid(), signal.SIGUSR1)
def is_parent_gone():
while os.getppid() != 1:
gevent.sleep(1)
print 'pycc: Now I am an orphan ... notifying serve_forever to stop'
os.kill(os.getpid(), signal.SIGINT)
import gevent
ipg = gevent.spawn(is_parent_gone)
container.gl_parent_watch = ipg
if not opts.noshell and not opts.daemon:
# Keep container running while there is an interactive shell
from pyon.container.shell_api import get_shell_api
setup_ipython_shell(get_shell_api(container))
elif not opts.nomanhole:
from pyon.container.shell_api import get_shell_api
setup_ipython_embed(get_shell_api(container))
else:
container.serve_forever()
def setUp(self):
self.username = CFG.get_safe('eoi.geoserver.user_name', 'admin')
self.PASSWORD = CFG.get_safe('eoi.geoserver.password', 'geoserver')
self.gs_host = CFG.get_safe('eoi.geoserver.server', 'http://localhost:8080')
self.gs_rest_url = ''.join([self.gs_host, '/geoserver/rest'])
self.gs_ows_url = ''.join([self.gs_host, '/geoserver/ows'])
IMPORTER_SERVICE_SERVER = CFG.get_safe('eoi.importer_service.server', 'http://localhost')
IMPORTER_SERVICE_PORT = str(CFG.get_safe('eoi.importer_service.port', 8844))
self.importer_service_url = ''.join([IMPORTER_SERVICE_SERVER, ':', IMPORTER_SERVICE_PORT])
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dataset_management = DatasetManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.resource_registry = self.container.resource_registry
def on_start(self): # pragma no cover
super(DiscoveryService,self).on_start()
cfg_datastore = CFG.get_safe('container.datastore.default_server')
if cfg_datastore != "postgresql":
raise Exception("Discovery service does not support datastores other than postgresql")
self.ds_discovery = DatastoreDiscovery(self)
def on_start(self):
super(DiscoveryService, self).on_start()
cfg_datastore = CFG.get_safe('container.datastore.default_server')
if cfg_datastore != "postgresql":
raise Exception("Discovery service does not support datastores other than postgresql")
self.ds_discovery = DatastoreDiscovery(self)
def get_data_url(self, data_product_id=''):
# The unique pointer to this set of data
ret = IonObject(OT.ComputedStringValue)
ret.value = ""
erddap_host = CFG.get_safe('server.erddap.host','localhost')
errdap_port = CFG.get_safe('server.erddap.port','8080')
try:
dataset_id = self._get_dataset_id(data_product_id)
ret.value = string.join( ["http://", erddap_host, ":", str(errdap_port),"/erddap/griddap/", str(dataset_id), "_0.html"],'')
ret.status = ComputedValueAvailability.PROVIDED
log.debug("get_data_url: data_url: %s", ret.value)
except NotFound:
ret.status = ComputedValueAvailability.NOTAVAILABLE
ret.reason = "Dataset for this Data Product could not be located"
return ret
def on_init(self):
# clients
self.resource_registry = self.container.resource_registry
self.event_publisher = EventPublisher(OT.NotificationSentEvent)
# SMTP client configurations
self.smtp_from = CFG.get_safe('server.smtp.from',
'*****@*****.**')
self.smtp_host = CFG.get_safe('server.smtp.host', 'localhost')
self.smtp_port = CFG.get_safe('server.smtp.port', 25)
# Jinja2 template environment
self.jinja_env = Environment(loader=FileSystemLoader('res/templates'),
trim_blocks=True,
lstrip_blocks=True)
super(NotificationWorker, self).on_init()
def process_index():
try:
from pyon.public import CFG
from pyon.core.bootstrap import get_sys_name
default_ds_server = CFG.get_safe("container.datastore.default_server", "postgresql")
attrs = dict(listurl=_link("/list"), base=_link(""))
fragments = [
"<h1>SciON Admin UI</h1>",
"<p><ul>",
"<li><a href='%s'><b>Browse Resource Registry and Resource Objects</b></a>" % _link("/restypes"),
"<ul>",
"<li>Org/Users: <a href='%(listurl)s/Org'>Org</a>, <a href='%(listurl)s/UserRole'>UserRole</a>, <a href='%(listurl)s/ActorIdentity'>ActorIdentity</a></li>" % attrs,
"<li>Computing: <a href='%(listurl)s/Process'>Process</a>, <a href='%(listurl)s/ProcessDefinition'>ProcessDefinition</a>, <a href='%(listurl)s/Service'>Service</a>, <a href='%(listurl)s/ServiceDefinition'>ServiceDefinition</a>, <a href='%(listurl)s/CapabilityContainer'>CapabilityContainer</a></li>" % attrs,
"<li>Messaging: <a href='%(listurl)s/ExchangeSpace'>ExchangeSpace</a>, <a href='%(listurl)s/ExchangePoint'>ExchangePoint</a>, <a href='%(listurl)s/ExchangeName'>ExchangeName</a>, <a href='%(listurl)s/ExchangeBroker'>ExchangeBroker</a></li>" % attrs,
"<li>Governance: <a href='%(listurl)s/Commitment'>Commitment</a>, <a href='%(listurl)s/Negotiation'>Negotiation</a>, <a href='%(listurl)s/Policy'>Policy</a></li>" % attrs,
_get_resmenu_extension(),
"</ul></li>",
"<li><a href='%s'><b>System Commands</b></a></li>" % _link("/syscmds"),
"<li><a href='%s'><b>Browse Events</b></a></li>" % _link("/events"),
"<li><a href='%s'><b>Browse SciON Directory</b></a></li>" % _link("/dir"),
"<li><a href='%s'><b>View Objects</b></a></li>" % _link("/viewobj"),
"<li><a href='%s'><b>View Process State</b></a></li>" % _link("/viewstate"),
"<li><a href='http://localhost:4000'><b>Application Web UI (if running)</b></a></li>",
"<li><a href='http://" + CFG.get_safe("server.amqp.host") + ":15672/'><b>RabbitMQ Management UI (if running)</b></a></li>",
"<li><a href='http://localhost:9001/'><b>Supervisord UI (if running)</b></a></li>",
"</ul></p>",
"<h2>System and Container Properties</h2>",
"<p><table>",
"<tr><th>Property</th><th>Value</th></tr>",
"<tr><td>Sysname (system.name)</td><td>%s</td></tr>" % get_sys_name(),
"<tr><td>Message Broker</td><td>%s</td></tr>" % "%s:%s" % (CFG.server.amqp.host, CFG.server.amqp.port),
"<tr><td>Database</td><td>%s</td></tr>" % "%s:%s" % (CFG.get_safe("server.%s.host" % default_ds_server), CFG.get_safe("server.%s.port" % default_ds_server)),
"<tr><td>Container ID</td><td>%s</td></tr>" % Container.instance.id,
"<tr><td>Read-Only</td><td>%s</td></tr>" % is_read_only(),
"</table></p>",
]
content = "\n".join(fragments)
return build_page(content)
except Exception as e:
return build_error_page(traceback.format_exc())
def _get_resmenu_extension():
resmenu_ext = CFG.get_safe(CFG_PREFIX + '.menu.extensions')
if not resmenu_ext:
return ""
ext_str = ""
for ext in resmenu_ext:
if isinstance(ext, basestring):
ext = ext.split(",")
ext_str += "<li>%s: %s</li>\n" % (ext[0], ", ".join("<a href='/list/%s'>%s</a>" % (rex, rex) for rex in ext[1:]))
return ext_str
def _get_resmenu_extension():
resmenu_ext = CFG.get_safe(CFG_PREFIX + '.menu.extensions')
if not resmenu_ext:
return ""
ext_str = ""
for ext in resmenu_ext:
if isinstance(ext, basestring):
ext = ext.split(",")
ext_str += "<li>%s: %s</li>\n" % (ext[0], ", ".join("<a href='/list/%s'>%s</a>" % (rex, rex) for rex in ext[1:]))
return ext_str
def do_preload_master(self, master):
if CFG.get_safe("scion.preload.enabled", False) is not True:
return False
log.info("############## PRELOAD SCION RESOURCES (%s) ##############", master)
skip_steps = CFG.get_safe("skipsteps")
if skip_steps:
skip_steps = skip_steps.split(",")
self.initialize_preloader(self.process, {})
if os.path.isdir("res/preload/{}".format(master)):
self.preload_master("res/preload/{}/actions.yml".format(master), skip_steps=skip_steps)
elif os.path.exists("res/preload/{}".format(master)):
self.preload_master("res/preload/{}".format(master), skip_steps=skip_steps)
else:
raise BadRequest("Cannot find preload master")
log.info("############## END PRELOAD ##############")
return True
def _es_call(es, *args, **kwargs):
res = AsyncResult()
def async_call(es, *args, **kwargs):
res.set(es(*args,**kwargs))
spawn(async_call,es,*args,**kwargs)
try:
retval = res.get(timeout=CFG.get_safe('server.elasticsearch.timeout', 10))
except Timeout:
raise exceptions.Timeout("Call to ElasticSearch timed out.")
return retval
def request_password_reset(self, username=''):
actor_id = self.find_actor_identity_by_username(username)
actor = self.rr.read(actor_id)
validity = CFG.get_safe(CFG_PREFIX + '.password_token_validity', 60*60)
actor.passwd_reset_token = self._create_token(actor_id=actor_id, validity=validity,
token_type=TokenTypeEnum.ACTOR_RESET_PASSWD)
self.rr.update(actor)
return actor.passwd_reset_token.token_string
def _gen_service_spec(self, service_name):
output = {
"swagger":
"2.0",
"info": {},
"host":
CFG.get_safe("system.web_ui_url", "").rstrip("/").split("//",
1)[1],
"basePath":
CFG.get_safe(UISG_CFG_PREFIX + ".url_prefix"),
"schemes": ["http"],
}
output["info"].update(self.config.get("info", {}))
if self.config.get("externalDocs", {}):
output["externalDocs"] = self.config["externalDocs"]
output["tags"] = self._gen_tags(service_name)
output["paths"] = self._gen_paths(service_name)
return output
def setUp(self):
self.username = CFG.get_safe('eoi.geoserver.user_name', 'admin')
self.PASSWORD = CFG.get_safe('eoi.geoserver.password', 'geoserver')
self.gs_host = CFG.get_safe('eoi.geoserver.server',
'http://localhost:8080')
self.gs_rest_url = ''.join([self.gs_host, '/geoserver/rest'])
self.gs_ows_url = ''.join([self.gs_host, '/geoserver/ows'])
IMPORTER_SERVICE_SERVER = CFG.get_safe('eoi.importer_service.server',
'http://localhost')
IMPORTER_SERVICE_PORT = str(
CFG.get_safe('eoi.importer_service.port', 8844))
self.importer_service_url = ''.join(
[IMPORTER_SERVICE_SERVER, ':', IMPORTER_SERVICE_PORT])
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dataset_management = DatasetManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.resource_registry = self.container.resource_registry
def setting_up_smtp_client():
'''
Sets up the smtp client
'''
#------------------------------------------------------------------------------------
# the default smtp server
#------------------------------------------------------------------------------------
smtp_client = None
smtp_host = CFG.get_safe('server.smtp.host')
smtp_port = CFG.get_safe('server.smtp.port', 25)
smtp_sender = CFG.get_safe('server.smtp.sender')
smtp_password = CFG.get_safe('server.smtp.password')
if CFG.get_safe('system.smtp',
False): #Default is False - use the fake_smtp
log.debug(
'Using the real SMTP library to send email notifications! host = %s',
smtp_host)
# smtp_client = smtplib.SMTP(smtp_host)
# smtp_client.ehlo()
# smtp_client.starttls()
# smtp_client.login(smtp_sender, smtp_password)
smtp_client = smtplib.SMTP(smtp_host, smtp_port)
log.debug("In setting up smtp client using the smtp client: %s",
smtp_client)
log.debug("Message received after ehlo exchange: %s",
str(smtp_client.ehlo()))
# smtp_client.login(smtp_sender, smtp_password)
else:
log.debug('Using a fake SMTP library to simulate email notifications!')
smtp_client = fake_smtplib.SMTP(smtp_host)
return smtp_client
def request_password_reset(self, username=''):
actor_id = self.find_actor_identity_by_username(username)
actor = self.rr.read(actor_id)
validity = CFG.get_safe(CFG_PREFIX + '.password_token_validity',
60 * 60)
actor.passwd_reset_token = self._create_token(
actor_id=actor_id,
validity=validity,
token_type=TokenTypeEnum.ACTOR_RESET_PASSWD)
self.rr.update(actor)
return actor.passwd_reset_token.token_string
def on_start(self):
load_from_file = CFG.get("load_from_file", None)
if load_from_file:
if os.path.exists(load_from_file):
self.load_from_files(self.directory_path, [load_from_file])
else:
print "LoadConfiguration: Error couldn't find the file path\n"
else:
# load from all files
self.load_object_model_interface()
self.load_service_interface()
self.load_resource_configuration()
def __init__(self, notification_request, user_id):
super(EmailEventProcessor, self).__init__(notification_request,
user_id)
smtp_host = CFG.get_safe('server.smtp.host', ION_SMTP_SERVER)
smtp_port = CFG.get_safe('server.smtp.port', 25)
self.smtp_sender = CFG.get_safe('server.smtp.sender')
smtp_password = CFG.get_safe('server.smtp.password')
log.info('smtp_host: %s' % str(smtp_host))
log.info('smtp_port: %s' % str(smtp_port))
if CFG.get_safe('system.smtp',
False): #Default is False - use the fake_smtp
log.warning(
'Using the real SMTP library to send email notifications!')
#@todo - for now hard wire for gmail account
#msg_sender = '*****@*****.**'
#gmail_pwd = 'ooici777'
self.smtp_client = smtplib.SMTP(smtp_host)
self.smtp_client.ehlo()
self.smtp_client.starttls()
self.smtp_client.login(self.smtp_sender, smtp_password)
log.warning("Using smpt host: %s" % smtp_host)
else:
# Keep this as a warning
log.warning(
'Using a fake SMTP library to simulate email notifications!')
#@todo - what about port etc??? What is the correct interface to fake?
self.smtp_client = fake_smtplib.SMTP(smtp_host)
log.debug("UserEventProcessor.__init__(): email for user %s " %
self.user_id)
def test_pydap_handlers(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict')
stream_def_id = self.create_stream_definition('ctd', parameter_dictionary_id=pdict_id)
data_product_id = self.create_data_product('ctd', stream_def_id=stream_def_id)
self.activate_data_product(data_product_id)
dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(data_product_id)
rdt = self.ph.get_rdt(stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
self.ph.publish_rdt_to_data_product(data_product_id,rdt)
dataset_monitor.event.wait(10)
from pydap.client import open_url
pydap_host = CFG.get_safe('server.pydap.host','localhost')
pydap_port = CFG.get_safe('server.pydap.port',8001)
url = 'http://%s:%s/%s' %(pydap_host, pydap_port, dataset_id)
ds = open_url(url)
ds['temp']['temp'][:]
def _cmd_start_rel(self, command):
log.debug("Start rel")
rel = command["rel_def"]
max_proc_replicas = int(CFG.get_safe("container.process.max_replicas", 0))
for rel_app_cfg in rel["apps"]:
app_name = rel_app_cfg["name"]
log.debug("app definition in rel: %s", str(rel_app_cfg))
# Decide where process should go
target_engine = self._determine_target_engine(rel_app_cfg)
log.debug("Dispatch app %s to engine %s", app_name, target_engine)
if 'processapp' in rel_app_cfg:
name, module, cls = rel_app_cfg["processapp"]
rel_cfg = None
if 'config' in rel_app_cfg:
rel_cfg = deepcopy(rel_app_cfg["config"])
if 'replicas' in rel_app_cfg:
proc_replicas = int(rel_app_cfg["replicas"])
if max_proc_replicas > 0:
if proc_replicas > max_proc_replicas:
log.info("Limiting number of proc replicas to %s from %s", max_proc_replicas, proc_replicas)
proc_replicas = min(proc_replicas, max_proc_replicas)
if proc_replicas < 1 or proc_replicas > 100:
log.warn("Invalid number of process replicas: %s", proc_replicas)
proc_replicas = 1
for i in xrange(proc_replicas):
cont_info = self._determine_target_container(rel_app_cfg, target_engine)
container_name = self._get_cc_agent_name(cont_info)
proc_name = "%s.%s" % (name, i) if i else name
action_res = self._add_spawn_process_action(cc_agent=container_name, proc_name=proc_name,
module=module, cls=cls, config=rel_cfg)
proc_id = action_res.wait()
self._slot_process(cont_info, proc_id, dict(proc_name=proc_name, state=ProcessStateEnum.RUNNING))
else:
cont_info = self._determine_target_container(rel_app_cfg, target_engine)
container_name = self._get_cc_agent_name(cont_info)
action_res = self._add_spawn_process_action(cc_agent=container_name, proc_name=name,
module=module, cls=cls, config=rel_cfg)
proc_id = action_res.wait()
self._slot_process(cont_info, proc_id, dict(proc_name=name, state=ProcessStateEnum.RUNNING))
else:
log.warn("App file not supported")
def test_dashi(self):
import dashi
dashi_conn = dashi.DashiConnection(
"something",
self._haa_dashi_uri,
self._haa_dashi_exchange,
sysname=CFG.get_safe("dashi.sysname"))
status = dashi_conn.call(self._haa_dashi_name, "status")
assert status in ('PENDING', 'READY', 'STEADY')
new_policy = {'preserve_n': 0}
dashi_conn.call(self._haa_dashi_name,
"reconfigure_policy",
new_policy=new_policy)
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dataset_management_client = DatasetManagementServiceClient(
node=self.container.node)
self.pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.wait_time = CFG.get_safe('endpoint.receive.timeout', 10)
def setUp(self):
self.dashi = None
self._start_container()
from pyon.public import CFG
self.container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self.container = self.container_client._get_container_instance()
app = dict(name="process_dispatcher",
processapp=("process_dispatcher",
"ion.services.cei.process_dispatcher_service",
"ProcessDispatcherService"))
self.container.start_app(app, config=pd_config)
self.rr_cli = self.container.resource_registry
self.pd_cli = ProcessDispatcherServiceClient(node=self.container.node)
self.process_definition = ProcessDefinition(name='test_process')
self.process_definition.executable = {
'module': 'ion.services.cei.test.test_process_dispatcher',
'class': 'TestProcess'
}
self.process_definition_id = self.pd_cli.create_process_definition(
self.process_definition)
self._eea_pids = []
self._eea_pid_to_resource_id = {}
self._eea_pid_to_persistence_dir = {}
self._tmpdirs = []
self.dashi = get_dashi(
uuid.uuid4().hex,
pd_config['processdispatcher']['dashi_uri'],
pd_config['processdispatcher']['dashi_exchange'],
sysname=CFG.get_safe("dashi.sysname"))
#send a fake node_state message to PD's dashi binding.
self.node1_id = uuid.uuid4().hex
self._send_node_state("engine1", self.node1_id)
self._initial_eea_pid = self._start_eeagent(self.node1_id)
self.waiter = ProcessStateWaiter()