class TestSchedulerService(IonIntegrationTestCase):
def setUp(self):
self.interval_timer_count = 0
self.interval_timer_sent_time = 0
self.interval_timer_received_time = 0
self.interval_timer_interval = 3
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
process = FakeProcess()
self.ssclient = SchedulerServiceProcessClient(node=self.container.node, process=process)
self.rrclient = ResourceRegistryServiceProcessClient(node=self.container.node, process=process)
def tearDown(self):
pass
def now_utc(self):
return time.mktime(datetime.datetime.utcnow().timetuple())
def test_create_interval_timer(self):
# create the interval timer resource
# create the event listener
# call scheduler to set the timer
# receive a few intervals, validate that arrival time is as expected
# cancel the timer
# wait until after next interval to verify that timer was correctly cancelled
self.interval_timer_count = 0
self.interval_timer_sent_time = 0
self.interval_timer_received_time = 0
self.interval_timer_interval = 3
event_origin = "Interval_Timer_233"
sub = EventSubscriber(event_type="TimerEvent", callback=self.interval_timer_callback, origin=event_origin)
sub.start()
self.addCleanup(sub.stop)
start_time = self.now_utc()
self.interval_timer_end_time = start_time + 10
id = self.ssclient.create_interval_timer(start_time="now" , interval=self.interval_timer_interval,
end_time=self.interval_timer_end_time,
event_origin=event_origin, event_subtype="")
self.interval_timer_sent_time = datetime.datetime.utcnow()
self.assertEqual(type(id), str)
# Validate the timer is stored in RR
ss = self.rrclient.read(id)
self.assertEqual(ss.entry.event_origin, event_origin)
# Wait until two events are published
gevent.sleep((self.interval_timer_interval * 2) + 1)
time_diff = (datetime.datetime.utcnow() - self.interval_timer_sent_time).seconds
timer_counts = math.floor(time_diff/self.interval_timer_interval)
#Cancle the timer
ss = self.ssclient.cancel_timer(id)
# wait until after next interval to verify that timer was correctly cancelled
gevent.sleep(self.interval_timer_interval)
# Validate the timer correctly cancelled
with self.assertRaises(BadRequest):
self.ssclient.cancel_timer(id)
# Validate the timer is removed from resource regsitry
with self.assertRaises(NotFound):
self.rrclient.read(id)
# Validate the number of timer counts
self.assertEqual(self.interval_timer_count, timer_counts, "Invalid number of timeouts generated. Number of timeout: %d Expected timeout: %d Timer id: %s " %(self.interval_timer_count, timer_counts, id))
def test_system_restart(self):
# create the interval timer resource
# create the event listener
# call scheduler to set the timer
# receive a few intervals, validate that arrival time is as expected
# cancel the timer
# wait until after next interval to verify that timer was correctly cancelled
self.interval_timer_count = 0
self.interval_timer_sent_time = 0
self.interval_timer_received_time = 0
self.interval_timer_interval = 3
event_origin = "Interval_Timer_4444"
sub = EventSubscriber(event_type="TimerEvent", callback=self.on_restart_callback, origin=event_origin)
sub.start()
self.addCleanup(sub.stop)
start_time = self.now_utc()
self.interval_timer_end_time = start_time + 20
id = self.ssclient.create_interval_timer(start_time="now" , interval=self.interval_timer_interval,
end_time=self.interval_timer_end_time,
event_origin=event_origin, event_subtype="")
self.interval_timer_sent_time = datetime.datetime.utcnow()
self.assertEqual(type(id), str)
# Validate the timer is stored in RR
ss = self.rrclient.read(id)
self.assertEqual(ss.entry.event_origin, event_origin)
# Wait until 1 event is published
gevent.sleep((self.interval_timer_interval) + 1)
time_diff = (datetime.datetime.utcnow() - self.interval_timer_sent_time).seconds
timer_counts = math.floor(time_diff/self.interval_timer_interval)
# Validate the number of events generated
self.assertEqual(self.interval_timer_count, timer_counts, "Invalid number of timeouts generated. Number of timeout: %d Expected timeout: %d Timer id: %s " %(self.interval_timer_count, timer_counts, id))
self.ssclient.on_system_restart()
# after system restart, validate the timer is restored
ss = self.rrclient.read(id)
self.assertEqual(ss.entry.event_origin, event_origin)
# Wait until another event is published
start_time = datetime.datetime.utcnow()
gevent.sleep((self.interval_timer_interval * 2) + 1)
time_diff = (datetime.datetime.utcnow() - start_time).seconds
timer_counts = math.floor(time_diff/self.interval_timer_interval)
# Validate the number of events generated
self.assertGreater(self.interval_timer_count, timer_counts)
#Cancle the timer
ss = self.ssclient.cancel_timer(id)
# wait until after next interval to verify that timer was correctly cancelled
gevent.sleep(self.interval_timer_interval)
# Validate the timer correctly cancelled
with self.assertRaises(BadRequest):
self.ssclient.cancel_timer(id)
# Validate the timer is removed from resource regsitry
with self.assertRaises(NotFound):
self.rrclient.read(id)
def on_restart_callback(self, *args, **kwargs):
self.interval_timer_count += 1
log.debug("test_scheduler: on_restart_callback: time: " + str(self.now_utc()) + " count: " + str(self.interval_timer_count))
def test_create_interval_timer_with_end_time(self):
# create the interval timer resource
# create the event listener
# call scheduler to set the timer
# receive a few intervals, validate that arrival time is as expected
# Validate no more events are published after end_time expires
# Validate the timer was canceled after the end_time expires
self.interval_timer_count_2 = 0
self.interval_timer_sent_time_2 = 0
self.interval_timer_received_time_2 = 0
self.interval_timer_interval_2 = 3
event_origin = "Interval_Timer_2"
sub = EventSubscriber(event_type="TimerEvent", callback=self.interval_timer_callback_with_end_time, origin=event_origin)
sub.start()
self.addCleanup(sub.stop)
start_time = self.now_utc()
self.interval_timer_end_time_2 = start_time + 7
id = self.ssclient.create_interval_timer(start_time="now" , interval=self.interval_timer_interval_2,
end_time=self.interval_timer_end_time_2,
event_origin=event_origin, event_subtype="")
self.interval_timer_sent_time_2 = datetime.datetime.utcnow()
self.assertEqual(type(id), str)
# Wait until all events are published
gevent.sleep((self.interval_timer_end_time_2 - start_time) + self.interval_timer_interval_2 + 1)
# Validate the number of events generated
self.assertEqual(self.interval_timer_count_2, 2, "Invalid number of timeouts generated. Number of event: %d Expected: 2 Timer id: %s " %(self.interval_timer_count_2, id))
# Validate the timer was canceled after the end_time is expired
with self.assertRaises(BadRequest):
self.ssclient.cancel_timer(id)
def interval_timer_callback_with_end_time(self, *args, **kwargs):
self.interval_timer_received_time_2 = datetime.datetime.utcnow()
self.interval_timer_count_2 += 1
time_diff = math.fabs( ((self.interval_timer_received_time_2 - self.interval_timer_sent_time_2).total_seconds())
- (self.interval_timer_interval_2 * self.interval_timer_count_2) )
# Assert expire time is within +-10 seconds
self.assertTrue(time_diff <= 10)
log.debug("test_scheduler: interval_timer_callback_with_end_time: time:" + str(self.interval_timer_received_time_2) + " count:" + str(self.interval_timer_count_2))
def interval_timer_callback(self, *args, **kwargs):
self.interval_timer_received_time = datetime.datetime.utcnow()
self.interval_timer_count += 1
time_diff = math.fabs( ((self.interval_timer_received_time - self.interval_timer_sent_time).total_seconds())
- (self.interval_timer_interval * self.interval_timer_count) )
# Assert expire time is within +-10 seconds
self.assertTrue(time_diff <= 10)
log.debug("test_scheduler: interval_timer_callback: time:" + str(self.interval_timer_received_time) + " count:" + str(self.interval_timer_count))
def test_cancel_single_timer(self):
# test creating a new timer that is one-time-only
# create the timer resource
# create the event listener
# call scheduler to set the timer
# create then cancel the timer, verify that event is not received
# create the timer resource
# create the event listener
# call scheduler to set the timer
# call scheduler to cancel the timer
# wait until after expiry to verify that event is not sent
self.single_timer_count = 0
event_origin = "Time_of_Day"
sub = EventSubscriber(event_type="TimerEvent", callback=self.single_timer_callback, origin=event_origin)
sub.start()
self.addCleanup(sub.stop)
now = datetime.datetime.utcnow() + timedelta(seconds=3)
times_of_day =[{'hour': str(now.hour),'minute' : str(now.minute), 'second':str(now.second) }]
id = self.ssclient.create_time_of_day_timer(times_of_day=times_of_day, expires=self.now_utc()+3, event_origin=event_origin, event_subtype="test")
self.assertEqual(type(id), str)
self.ssclient.cancel_timer(id)
gevent.sleep(3)
# Validate the event is not generated
self.assertEqual(self.single_timer_count, 0, "Invalid number of timeouts generated. Number of timeout: %d Expected timeout: 0 Timer id: %s " %(self.single_timer_count, id))
def single_timer_callback (self, *args, **kwargs):
self.single_timer_count =+ 1
log.debug("test_scheduler: single_timer_call_back: time:" + str(self.now_utc()) + " count:" + str(self.single_timer_count))
def test_create_forever_interval_timer(self):
# Test creating interval timer that runs forever
self.interval_timer_count = 0
self.interval_timer_sent_time = 0
self.interval_timer_received_time = 0
self.interval_timer_interval = 3
event_origin = "Interval Timer Forever"
sub = EventSubscriber(event_type="TimerEvent", callback=self.interval_timer_callback, origin=event_origin)
sub.start()
self.addCleanup(sub.stop)
id = self.ssclient.create_interval_timer(start_time= self.now_utc(), interval=self.interval_timer_interval,
end_time=-1,
event_origin=event_origin, event_subtype=event_origin)
self.interval_timer_sent_time = datetime.datetime.utcnow()
self.assertEqual(type(id), str)
# Wait for 4 events to be published
gevent.sleep((self.interval_timer_interval * 4) + 1)
self.ssclient.cancel_timer(id)
time_diff = (datetime.datetime.utcnow() - self.interval_timer_sent_time).seconds
timer_counts = math.floor(time_diff/self.interval_timer_interval)
# Validate the timer id is invalid once it has been canceled
with self.assertRaises(BadRequest):
self.ssclient.cancel_timer(id)
# Validate events are not generated after canceling the timer
self.assertEqual(self.interval_timer_count, timer_counts, "Invalid number of timeouts generated. Number of timeout: %d Expected timeout: %d Timer id: %s " %(self.interval_timer_count, timer_counts, id))
def test_timeoffday_timer(self):
# test creating a new timer that is one-time-only
# create the timer resource
# get the current time, set the timer to several seconds from current time
# create the event listener
# call scheduler to set the timer
# verify that event arrival is within one/two seconds of current time
event_origin = "Time Of Day2"
self.expire_sec_1 = 4
self.expire_sec_2 = 5
self.tod_count = 0
expire1 = datetime.datetime.utcnow() + timedelta(seconds=self.expire_sec_1)
expire2 = datetime.datetime.utcnow() + timedelta(seconds=self.expire_sec_2)
# Create two timers
times_of_day =[{'hour': str(expire1.hour),'minute' : str(expire1.minute), 'second':str(expire1.second) },
{'hour': str(expire2.hour),'minute' : str(expire2.minute), 'second':str(expire2.second)}]
sub = EventSubscriber(event_type="TimerEvent", callback=self.tod_callback, origin=event_origin)
sub.start()
self.addCleanup(sub.stop)
# Expires in one days
expires = time.mktime((datetime.datetime.utcnow() + timedelta(days=2)).timetuple())
self.tod_sent_time = datetime.datetime.utcnow()
id = self.ssclient.create_time_of_day_timer(times_of_day=times_of_day, expires=expires, event_origin=event_origin, event_subtype="")
self.interval_timer_sent_time = datetime.datetime.utcnow()
self.assertEqual(type(id), str)
# Wait until all events are generated
gevent.sleep(9)
time_diff = (datetime.datetime.utcnow() - self.interval_timer_sent_time).seconds
timer_counts = math.floor(time_diff/self.expire_sec_1) + math.floor(time_diff/self.expire_sec_2)
# After waiting, validate only 2 events are generated.
self.assertEqual(self.tod_count, 2, "Invalid number of timeouts generated. Number of timeout: %d Expected timeout: %d Timer id: %s " %(self.tod_count, timer_counts, id))
# Cancel the timer
self.ssclient.cancel_timer(id)
def tod_callback(self, *args, **kwargs):
tod_receive_time = datetime.datetime.utcnow()
self.tod_count += 1
if self.tod_count == 1:
time_diff = math.fabs((tod_receive_time - self.tod_sent_time).total_seconds() - self.expire_sec_1)
self.assertTrue(time_diff <= 2)
elif self.tod_count == 2:
time_diff = math.fabs((tod_receive_time - self.tod_sent_time).total_seconds() - self.expire_sec_2)
self.assertTrue(time_diff <= 2)
log.debug("test_scheduler: tod_callback: time:" + str(tod_receive_time) + " count:" + str(self.tod_count))
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_quit_stops_timers(self):
ar = AsyncResult()
def cb(*args, **kwargs):
ar.set(args)
self.interval_timer_count += 1
event_origin = "test_quitter"
sub = EventSubscriber(event_type="TimerEvent", callback=cb, origin=event_origin)
sub.start()
self.addCleanup(sub.stop)
tid = self.ssclient.create_interval_timer(start_time="now",
interval=1,
event_origin=event_origin)
# wait until at least one scheduled message
ar.get(timeout=5)
# shut it down!
p = self.container.proc_manager.procs_by_name['scheduler']
self.container.terminate_process(p.id)
# assert empty
self.assertEquals(p.schedule_entries, {})
class VizTransformMatplotlibGraphs(TransformStreamPublisher, TransformEventListener, TransformStreamListener):
"""
This class is used for instantiating worker processes that have subscriptions to data streams and convert
incoming data from CDM format to Matplotlib graphs
"""
output_bindings = ['graph_image_param_dict']
event_timer_interval = None
def on_start(self):
#print ">>>>>>>>>>>>>>>>>>>>>> MPL CFG = ", self.CFG
self.pubsub_management = PubsubManagementServiceProcessClient(process=self)
self.ssclient = SchedulerServiceProcessClient(process=self)
self.rrclient = ResourceRegistryServiceProcessClient(process=self)
self.data_retriever_client = DataRetrieverServiceProcessClient(process=self)
self.dsm_client = DatasetManagementServiceProcessClient(process=self)
self.pubsub_client = PubsubManagementServiceProcessClient(process = self)
self.stream_info = self.CFG.get_safe('process.publish_streams',{})
self.stream_names = self.stream_info.keys()
self.stream_ids = self.stream_info.values()
if not self.stream_names:
raise BadRequest('MPL Transform has no output streams.')
graph_time_periods= self.CFG.get_safe('graph_time_periods')
# If this is meant to be an event driven process, schedule an event to be generated every few minutes/hours
self.event_timer_interval = self.CFG.get_safe('graph_gen_interval')
if self.event_timer_interval:
event_origin = "Interval_Timer_Matplotlib"
sub = EventSubscriber(event_type="ResourceEvent", callback=self.interval_timer_callback, origin=event_origin)
sub.start()
self.interval_timer_id = self.ssclient.create_interval_timer(start_time="now" , interval=self._str_to_secs(self.event_timer_interval),
event_origin=event_origin, event_subtype="")
super(VizTransformMatplotlibGraphs,self).on_start()
# when tranform is used as a data process
def recv_packet(self, packet, in_stream_route, in_stream_id):
#Check to see if the class instance was set up as a event triggered transform. If yes, skip the packet
if self.event_timer_interval:
return
log.info('Received packet')
mpl_data_granule = VizTransformMatplotlibGraphsAlgorithm.execute(packet, params=self.get_stream_definition())
for stream_name in self.stream_names:
publisher = getattr(self, stream_name)
publisher.publish(mpl_data_granule)
def get_stream_definition(self):
stream_id = self.stream_ids[0]
stream_def = self.pubsub_management.read_stream_definition(stream_id=stream_id)
return stream_def._id
def process_event(self, msg, headers):
return
def interval_timer_callback(self, *args, **kwargs):
#Find out the input data product to this process
in_dp_id = self.CFG.get_safe('in_dp_id')
print " >>>>>>>>>>>>>> IN DP ID from cfg : ", in_dp_id
# get the dataset_id associated with the data_product. Need it to do the data retrieval
ds_ids,_ = self.rrclient.find_objects(in_dp_id, PRED.hasDataset, RT.Dataset, True)
if ds_ids is None or not ds_ids:
return None
# retrieve data for the specified time interval. Setup up query from pass config first
query = {}
param_list_str = self.CFG.get_safe('parameters')
if param_list_str:
query['parameters'] = param_list_str.split(', ')
# append time if not present in list of parameters
if not 'time' in query['parameters']:
query['parameters'].append('time')
query['start_time'] = query['end_time'] = ntplib.system_to_ntp_time(time.time()) # Now
query['stride_time'] = 1
if self.CFG.get_safe('graph_time_period'):
query['start_time'] = query['end_time'] - self._str_to_secs(self.CFG.get_safe('graph_time_period'))
#print " >>>>>>>>>>>>>> QUERY = ", query
#retrieved_granule = self.data_retriever_client.retrieve(ds_ids[0],{'start_time':start_time,'end_time':end_time})
retrieved_granule = self.data_retriever_client.retrieve(ds_ids[0], query=query)
# add extra parameters to query passed in config that are not needed by data retrieval
if self.CFG.get_safe('resolution'):
query['resolution'] = self.CFG.get_safe('resolution')
# send the granule through the Algorithm code to get the matplotlib graphs
mpl_pdict_id = self.dsm_client.read_parameter_dictionary_by_name('graph_image_param_dict',id_only=True)
mpl_stream_def = self.pubsub_client.create_stream_definition('mpl', parameter_dictionary_id=mpl_pdict_id)
fileName = self.CFG.get_safe('graph_time_period')
mpl_data_granule = VizTransformMatplotlibGraphsAlgorithm.execute(retrieved_granule, config=query, params=mpl_stream_def, fileName=fileName)
if mpl_data_granule == None:
return None
# publish on all specified output streams
for stream_name in self.stream_names:
publisher = getattr(self, stream_name)
publisher.publish(mpl_data_granule)
return
def _str_to_secs(self, time_period):
# this method converts commonly used time periods to its actual seconds counterpart
#separate alpha and numeric parts of the time period
time_n = time_period.lower().rstrip('abcdefghijklmnopqrstuvwxyz ')
time_a = time_period.lower().lstrip('0123456789. ')
# determine if user specified, secs, mins, hours, days, weeks, months, years
factor = None
if time_a == 'sec' or time_a == "secs" or time_a == 'second' or time_a == "seconds":
factor = 1
if time_a == "min" or time_a == "mins" or time_a == "minute" or time_a == "minutes":
factor = 60
if time_a == "hr" or time_a == "hrs" or time_a == "hour" or time_a == "hours":
factor = 60 * 60
if time_a == "day" or time_a == "days":
factor = 60 * 60 * 24
if time_a == "wk" or time_a == "wks" or time_a == "week" or time_a == "weeks":
factor = 60 * 60 * 24 * 7
if time_a == "mon" or time_a == "mons" or time_a == "month" or time_a == "months":
factor = 60 * 60 * 24 * 30
if time_a == "yr" or time_a == "yrs" or time_a == "year" or time_a == "years":
factor = 60 * 60 * 24 * 365
time_period_secs = float(time_n) * factor
return time_period_secs
def on_quit(self):
#Cancel the timer
if hasattr(self, 'interval_timer_id'):
self.ssclient.cancel_timer(self.interval_timer_id)
super(VizTransformMatplotlibGraphs,self).on_quit()
class TestSchedulerService(IonIntegrationTestCase):
def setUp(self):
self.interval_timer_count = 0
self.interval_timer_sent_time = 0
self.interval_timer_received_time = 0
self.interval_timer_interval = 3
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
process = FakeProcess()
self.ssclient = SchedulerServiceProcessClient(node=self.container.node, process=process)
self.rrclient = ResourceRegistryServiceProcessClient(node=self.container.node, process=process)
def tearDown(self):
pass
def now_utc(self):
return time.mktime(datetime.datetime.utcnow().timetuple())
def test_create_interval_timer(self):
# create the interval timer resource
# create the event listener
# call scheduler to set the timer
# receive a few intervals, validate that arrival time is as expected
# cancel the timer
# wait until after next interval to verify that timer was correctly cancelled
self.interval_timer_count = 0
self.interval_timer_sent_time = 0
self.interval_timer_received_time = 0
self.interval_timer_interval = 3
event_origin = "Interval_Timer_233"
sub = EventSubscriber(event_type="ResourceEvent", callback=self.interval_timer_callback, origin=event_origin)
sub.start()
start_time = self.now_utc()
self.interval_timer_end_time = start_time + 10
id = self.ssclient.create_interval_timer(start_time="now" , interval=self.interval_timer_interval,
end_time=self.interval_timer_end_time,
event_origin=event_origin, event_subtype="")
self.interval_timer_sent_time = datetime.datetime.utcnow()
self.assertEqual(type(id), str)
# Validate the timer is stored in RR
ss = self.rrclient.read(id)
self.assertEqual(ss.entry.event_origin, event_origin)
# Wait until two events are published
gevent.sleep((self.interval_timer_interval * 2) + 1)
#Cancle the timer
ss = self.ssclient.cancel_timer(id)
# wait until after next interval to verify that timer was correctly cancelled
gevent.sleep(self.interval_timer_interval)
# Validate the timer correctly cancelled
with self.assertRaises(BadRequest):
self.ssclient.cancel_timer(id)
# Validate the timer is removed from resource regsitry
with self.assertRaises(NotFound):
self.rrclient.read(id)
# Validate only 2 events are published
self.assertEqual(self.interval_timer_count, 2)
def test_system_restart(self):
# create the interval timer resource
# create the event listener
# call scheduler to set the timer
# receive a few intervals, validate that arrival time is as expected
# cancel the timer
# wait until after next interval to verify that timer was correctly cancelled
self.interval_timer_count = 0
self.interval_timer_sent_time = 0
self.interval_timer_received_time = 0
self.interval_timer_interval = 3
event_origin = "Interval_Timer_4444"
sub = EventSubscriber(event_type="ResourceEvent", callback=self.on_restart_callback, origin=event_origin)
sub.start()
start_time = self.now_utc()
self.interval_timer_end_time = start_time + 20
id = self.ssclient.create_interval_timer(start_time="now" , interval=self.interval_timer_interval,
end_time=self.interval_timer_end_time,
event_origin=event_origin, event_subtype="")
self.interval_timer_sent_time = datetime.datetime.utcnow()
self.assertEqual(type(id), str)
# Validate the timer is stored in RR
ss = self.rrclient.read(id)
self.assertEqual(ss.entry.event_origin, event_origin)
# Wait until 1 event is published
gevent.sleep((self.interval_timer_interval) + 1)
# Validate 1 event is published
self.assertEqual(self.interval_timer_count, 1)
self.ssclient.on_system_restart()
# after system restart, validate the timer is restored
ss = self.rrclient.read(id)
self.assertEqual(ss.entry.event_origin, event_origin)
# Wait until another event is published
gevent.sleep((self.interval_timer_interval * 2) + 1)
# Validate 1 event is published
self.assertGreater(self.interval_timer_count, 2)
#Cancle the timer
ss = self.ssclient.cancel_timer(id)
# wait until after next interval to verify that timer was correctly cancelled
gevent.sleep(self.interval_timer_interval)
# Validate the timer correctly cancelled
with self.assertRaises(BadRequest):
self.ssclient.cancel_timer(id)
# Validate the timer is removed from resource regsitry
with self.assertRaises(NotFound):
self.rrclient.read(id)
def on_restart_callback(self, *args, **kwargs):
self.interval_timer_count += 1
def test_create_interval_timer_with_end_time(self):
# create the interval timer resource
# create the event listener
# call scheduler to set the timer
# receive a few intervals, validate that arrival time is as expected
# Validate no more events are published after end_time expires
# Validate the timer was canceled after the end_time expires
self.interval_timer_count = 0
self.interval_timer_sent_time = 0
self.interval_timer_received_time = 0
self.interval_timer_interval = 2
event_origin = "Interval Timer"
sub = EventSubscriber(event_type="ResourceEvent", callback=self.interval_timer_callback, origin=event_origin)
sub.start()
start_time = self.now_utc()
self.interval_timer_end_time = start_time + 5
id = self.ssclient.create_interval_timer(start_time="now" , interval=self.interval_timer_interval,
end_time=self.interval_timer_end_time,
event_origin=event_origin, event_subtype="")
self.interval_timer_sent_time = datetime.datetime.utcnow()
self.assertEqual(type(id), str)
# Wait until all events are published
gevent.sleep((self.interval_timer_end_time - start_time) + self.interval_timer_interval + 1)
# Validate only 2 events are published
self.assertEqual(self.interval_timer_count, 2)
# Validate the timer was canceled after the end_time is expired
with self.assertRaises(BadRequest):
self.ssclient.cancel_timer(id)
def interval_timer_callback(self, *args, **kwargs):
self.interval_timer_received_time = datetime.datetime.utcnow()
self.interval_timer_count += 1
time_diff = math.fabs( ((self.interval_timer_received_time - self.interval_timer_sent_time).total_seconds())
- (self.interval_timer_interval * self.interval_timer_count) )
# Assert expire time is within +-10 seconds
self.assertTrue(time_diff <= 10)
def test_cancel_single_timer(self):
# test creating a new timer that is one-time-only
# create the timer resource
# create the event listener
# call scheduler to set the timer
# create then cancel the timer, verify that event is not received
# create the timer resource
# create the event listener
# call scheduler to set the timer
# call scheduler to cancel the timer
# wait until after expiry to verify that event is not sent
self.single_timer_count = 0
event_origin = "Time_of_Day"
sub = EventSubscriber(event_type="ResourceEvent", callback=self.single_timer_call_back, origin=event_origin)
sub.start()
now = datetime.datetime.utcnow() + timedelta(seconds=3)
times_of_day =[{'hour': str(now.hour),'minute' : str(now.minute), 'second':str(now.second) }]
id = self.ssclient.create_time_of_day_timer(times_of_day=times_of_day, expires=self.now_utc()+3, event_origin=event_origin, event_subtype="test")
self.assertEqual(type(id), str)
self.ssclient.cancel_timer(id)
gevent.sleep(3)
# Validate the event is not generated
self.assertEqual(self.single_timer_count, 0)
def single_timer_call_back (self, *args, **kwargs):
self.single_timer_count =+ 1
def test_create_forever_interval_timer(self):
# Test creating interval timer that runs forever
self.interval_timer_count = 0
self.interval_timer_sent_time = 0
self.interval_timer_received_time = 0
self.interval_timer_interval = 3
event_origin = "Interval Timer Forever"
sub = EventSubscriber(event_type="ResourceEvent", callback=self.interval_timer_callback, origin=event_origin)
sub.start()
id = self.ssclient.create_interval_timer(start_time= self.now_utc(), interval=self.interval_timer_interval,
end_time=-1,
event_origin=event_origin, event_subtype=event_origin)
self.interval_timer_sent_time = datetime.datetime.utcnow()
self.assertEqual(type(id), str)
# Wait for 4 events to be published
gevent.sleep((self.interval_timer_interval * 4) + 1)
self.ssclient.cancel_timer(id)
# Validate the timer id is invalid once it has been canceled
with self.assertRaises(BadRequest):
self.ssclient.cancel_timer(id)
# Validate events are not generated after canceling the timer
self.assertEqual(self.interval_timer_count, 4)
def test_timeoffday_timer(self):
# test creating a new timer that is one-time-only
# create the timer resource
# get the current time, set the timer to several seconds from current time
# create the event listener
# call scheduler to set the timer
# verify that event arrival is within one/two seconds of current time
event_origin = "Time Of Day2"
self.expire_sec_1 = 4
self.expire_sec_2 = 5
self.tod_count = 0
expire1 = datetime.datetime.utcnow() + timedelta(seconds=self.expire_sec_1)
expire2 = datetime.datetime.utcnow() + timedelta(seconds=self.expire_sec_2)
# Create two timers
times_of_day =[{'hour': str(expire1.hour),'minute' : str(expire1.minute), 'second':str(expire1.second) },
{'hour': str(expire2.hour),'minute' : str(expire2.minute), 'second':str(expire2.second)}]
sub = EventSubscriber(event_type="ResourceEvent", callback=self.tod_callback, origin=event_origin)
sub.start()
# Expires in one days
expires = time.mktime((datetime.datetime.utcnow() + timedelta(days=2)).timetuple())
self.tod_sent_time = datetime.datetime.utcnow()
id = self.ssclient.create_time_of_day_timer(times_of_day=times_of_day, expires=expires, event_origin=event_origin, event_subtype="")
self.assertEqual(type(id), str)
# Wait until all events are generated
gevent.sleep(9)
# After waiting for 15 seconds, validate only 2 events are generated.
self.assertTrue(self.tod_count == 2)
# Cancel the timer
self.ssclient.cancel_timer(id)
def tod_callback(self, *args, **kwargs):
tod_receive_time = datetime.datetime.utcnow()
self.tod_count += 1
if self.tod_count == 1:
time_diff = math.fabs((tod_receive_time - self.tod_sent_time).total_seconds() - self.expire_sec_1)
self.assertTrue(time_diff <= 2)
elif self.tod_count == 2:
time_diff = math.fabs((tod_receive_time - self.tod_sent_time).total_seconds() - self.expire_sec_2)
self.assertTrue(time_diff <= 2)
class TestSchedulerService(IonIntegrationTestCase):
def setUp(self):
self.interval_timer_count = 0
self.interval_timer_sent_time = 0
self.interval_timer_received_time = 0
self.interval_timer_interval = 3
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
process = FakeProcess()
self.ssclient = SchedulerServiceProcessClient(node=self.container.node, process=process)
self.rrclient = ResourceRegistryServiceProcessClient(node=self.container.node, process=process)
def tearDown(self):
pass
def now_utc(self):
return time.time()
def test_create_interval_timer(self):
# create the interval timer resource
# create the event listener
# call scheduler to set the timer
# receive a few intervals, validate that arrival time is as expected
# cancel the timer
# wait until after next interval to verify that timer was correctly cancelled
self.interval_timer_count = 0
self.interval_timer_sent_time = 0
self.interval_timer_received_time = 0
self.interval_timer_interval = 3
event_origin = "Interval_Timer_233"
sub = EventSubscriber(event_type="TimerEvent", callback=self.interval_timer_callback, origin=event_origin)
sub.start()
self.addCleanup(sub.stop)
start_time = self.now_utc()
self.interval_timer_end_time = start_time + 10
id = self.ssclient.create_interval_timer(start_time="now" , interval=self.interval_timer_interval,
end_time=self.interval_timer_end_time,
event_origin=event_origin, event_subtype="")
self.interval_timer_sent_time = datetime.datetime.utcnow()
self.assertEqual(type(id), str)
# Validate the timer is stored in RR
ss = self.rrclient.read(id)
self.assertEqual(ss.entry.event_origin, event_origin)
# Wait until two events are published
gevent.sleep((self.interval_timer_interval * 2) + 1)
time_diff = (datetime.datetime.utcnow() - self.interval_timer_sent_time).seconds
timer_counts = math.floor(time_diff/self.interval_timer_interval)
#Cancle the timer
ss = self.ssclient.cancel_timer(id)
# wait until after next interval to verify that timer was correctly cancelled
gevent.sleep(self.interval_timer_interval)
# Validate the timer correctly cancelled
with self.assertRaises(BadRequest):
self.ssclient.cancel_timer(id)
# Validate the timer is removed from resource regsitry
with self.assertRaises(NotFound):
self.rrclient.read(id)
# Validate the number of timer counts
self.assertEqual(self.interval_timer_count, timer_counts, "Invalid number of timeouts generated. Number of timeout: %d Expected timeout: %d Timer id: %s " %(self.interval_timer_count, timer_counts, id))
def test_system_restart(self):
# create the interval timer resource
# create the event listener
# call scheduler to set the timer
# receive a few intervals, validate that arrival time is as expected
# cancel the timer
# wait until after next interval to verify that timer was correctly cancelled
self.interval_timer_count = 0
self.interval_timer_sent_time = 0
self.interval_timer_received_time = 0
self.interval_timer_interval = 3
event_origin = "Interval_Timer_4444"
sub = EventSubscriber(event_type="TimerEvent", callback=self.on_restart_callback, origin=event_origin)
sub.start()
self.addCleanup(sub.stop)
start_time = self.now_utc()
self.interval_timer_end_time = start_time + 20
id = self.ssclient.create_interval_timer(start_time="now", interval=self.interval_timer_interval,
end_time=self.interval_timer_end_time,
event_origin=event_origin, event_subtype="")
self.interval_timer_sent_time = datetime.datetime.utcnow()
self.assertEqual(type(id), str)
# Validate the timer is stored in RR
ss = self.rrclient.read(id)
self.assertEqual(ss.entry.event_origin, event_origin)
# Wait until 1 event is published
gevent.sleep((self.interval_timer_interval) + 1)
time_diff = (datetime.datetime.utcnow() - self.interval_timer_sent_time).seconds
timer_counts = math.floor(time_diff/self.interval_timer_interval)
# Validate the number of events generated
self.assertEqual(self.interval_timer_count, timer_counts, "Invalid number of timeouts generated. Number of timeout: %d Expected timeout: %d Timer id: %s " %(self.interval_timer_count, timer_counts, id))
self.ssclient.on_system_restart()
# after system restart, validate the timer is restored
ss = self.rrclient.read(id)
self.assertEqual(ss.entry.event_origin, event_origin)
# Wait until another event is published
start_time = datetime.datetime.utcnow()
gevent.sleep((self.interval_timer_interval * 2) + 1)
time_diff = (datetime.datetime.utcnow() - start_time).seconds
timer_counts = math.floor(time_diff/self.interval_timer_interval)
# Validate the number of events generated
self.assertGreater(self.interval_timer_count, timer_counts)
#Cancle the timer
ss = self.ssclient.cancel_timer(id)
# wait until after next interval to verify that timer was correctly cancelled
gevent.sleep(self.interval_timer_interval)
# Validate the timer correctly cancelled
with self.assertRaises(BadRequest):
self.ssclient.cancel_timer(id)
# Validate the timer is removed from resource regsitry
with self.assertRaises(NotFound):
self.rrclient.read(id)
def on_restart_callback(self, *args, **kwargs):
self.interval_timer_count += 1
log.debug("test_scheduler: on_restart_callback: time: " + str(self.now_utc()) + " count: " + str(self.interval_timer_count))
def test_create_interval_timer_with_end_time(self):
# create the interval timer resource
# create the event listener
# call scheduler to set the timer
# receive a few intervals, validate that arrival time is as expected
# Validate no more events are published after end_time expires
# Validate the timer was canceled after the end_time expires
self.interval_timer_count_2 = 0
self.interval_timer_sent_time_2 = 0
self.interval_timer_received_time_2 = 0
self.interval_timer_interval_2 = 3
event_origin = "Interval_Timer_2"
sub = EventSubscriber(event_type="TimerEvent", callback=self.interval_timer_callback_with_end_time, origin=event_origin)
sub.start()
self.addCleanup(sub.stop)
start_time = self.now_utc()
self.interval_timer_end_time_2 = start_time + 7
id = self.ssclient.create_interval_timer(start_time="now" , interval=self.interval_timer_interval_2,
end_time=self.interval_timer_end_time_2,
event_origin=event_origin, event_subtype="")
self.interval_timer_sent_time_2 = datetime.datetime.utcnow()
self.assertEqual(type(id), str)
# Wait until all events are published
gevent.sleep((self.interval_timer_end_time_2 - start_time) + self.interval_timer_interval_2 + 1)
# Validate the number of events generated
self.assertEqual(self.interval_timer_count_2, 2, "Invalid number of timeouts generated. Number of event: %d Expected: 2 Timer id: %s " %(self.interval_timer_count_2, id))
# Validate the timer was canceled after the end_time is expired
with self.assertRaises(BadRequest):
self.ssclient.cancel_timer(id)
def interval_timer_callback_with_end_time(self, *args, **kwargs):
self.interval_timer_received_time_2 = datetime.datetime.utcnow()
self.interval_timer_count_2 += 1
time_diff = math.fabs( ((self.interval_timer_received_time_2 - self.interval_timer_sent_time_2).total_seconds())
- (self.interval_timer_interval_2 * self.interval_timer_count_2) )
# Assert expire time is within +-10 seconds
self.assertTrue(time_diff <= 10)
log.debug("test_scheduler: interval_timer_callback_with_end_time: time:" + str(self.interval_timer_received_time_2) + " count:" + str(self.interval_timer_count_2))
def interval_timer_callback(self, *args, **kwargs):
self.interval_timer_received_time = datetime.datetime.utcnow()
self.interval_timer_count += 1
time_diff = math.fabs( ((self.interval_timer_received_time - self.interval_timer_sent_time).total_seconds())
- (self.interval_timer_interval * self.interval_timer_count) )
# Assert expire time is within +-10 seconds
self.assertTrue(time_diff <= 10)
log.debug("test_scheduler: interval_timer_callback: time:" + str(self.interval_timer_received_time) + " count:" + str(self.interval_timer_count))
def test_cancel_single_timer(self):
# test creating a new timer that is one-time-only
# create the timer resource
# create the event listener
# call scheduler to set the timer
# create then cancel the timer, verify that event is not received
# create the timer resource
# create the event listener
# call scheduler to set the timer
# call scheduler to cancel the timer
# wait until after expiry to verify that event is not sent
self.single_timer_count = 0
event_origin = "Time_of_Day"
sub = EventSubscriber(event_type="TimerEvent", callback=self.single_timer_callback, origin=event_origin)
sub.start()
self.addCleanup(sub.stop)
now = datetime.datetime.utcnow() + timedelta(seconds=3)
times_of_day =[{'hour': str(now.hour),'minute' : str(now.minute), 'second':str(now.second) }]
id = self.ssclient.create_time_of_day_timer(times_of_day=times_of_day, expires=self.now_utc()+3, event_origin=event_origin, event_subtype="test")
self.assertEqual(type(id), str)
self.ssclient.cancel_timer(id)
gevent.sleep(3)
# Validate the event is not generated
self.assertEqual(self.single_timer_count, 0, "Invalid number of timeouts generated. Number of timeout: %d Expected timeout: 0 Timer id: %s " %(self.single_timer_count, id))
def single_timer_callback (self, *args, **kwargs):
self.single_timer_count =+ 1
log.debug("test_scheduler: single_timer_call_back: time:" + str(self.now_utc()) + " count:" + str(self.single_timer_count))
def test_create_forever_interval_timer(self):
# Test creating interval timer that runs forever
self.interval_timer_count = 0
self.interval_timer_sent_time = 0
self.interval_timer_received_time = 0
self.interval_timer_interval = 3
event_origin = "Interval Timer Forever"
sub = EventSubscriber(event_type="TimerEvent", callback=self.interval_timer_callback, origin=event_origin)
sub.start()
self.addCleanup(sub.stop)
id = self.ssclient.create_interval_timer(start_time=str(self.now_utc()), interval=self.interval_timer_interval,
end_time="-1",
event_origin=event_origin, event_subtype=event_origin)
self.interval_timer_sent_time = datetime.datetime.utcnow()
self.assertEqual(type(id), str)
# Wait for 4 events to be published
gevent.sleep((self.interval_timer_interval * 4) + 1)
self.ssclient.cancel_timer(id)
time_diff = (datetime.datetime.utcnow() - self.interval_timer_sent_time).seconds
timer_counts = math.floor(time_diff/self.interval_timer_interval)
# Validate the timer id is invalid once it has been canceled
with self.assertRaises(BadRequest):
self.ssclient.cancel_timer(id)
# Validate events are not generated after canceling the timer
self.assertEqual(self.interval_timer_count, timer_counts, "Invalid number of timeouts generated. Number of timeout: %d Expected timeout: %d Timer id: %s " %(self.interval_timer_count, timer_counts, id))
def test_timeoffday_timer(self):
# test creating a new timer that is one-time-only
# create the timer resource
# get the current time, set the timer to several seconds from current time
# create the event listener
# call scheduler to set the timer
# verify that event arrival is within one/two seconds of current time
event_origin = "Time Of Day2"
self.expire_sec_1 = 4
self.expire_sec_2 = 5
self.tod_count = 0
expire1 = datetime.datetime.utcnow() + timedelta(seconds=self.expire_sec_1)
expire2 = datetime.datetime.utcnow() + timedelta(seconds=self.expire_sec_2)
# Create two timers
times_of_day =[{'hour': str(expire1.hour),'minute' : str(expire1.minute), 'second':str(expire1.second) },
{'hour': str(expire2.hour),'minute' : str(expire2.minute), 'second':str(expire2.second)}]
sub = EventSubscriber(event_type="TimerEvent", callback=self.tod_callback, origin=event_origin)
sub.start()
self.addCleanup(sub.stop)
# Expires in one days
expires = calendar.timegm((datetime.datetime.utcnow() + timedelta(days=2)).timetuple())
self.tod_sent_time = datetime.datetime.utcnow()
id = self.ssclient.create_time_of_day_timer(times_of_day=times_of_day, expires=expires, event_origin=event_origin, event_subtype="")
self.interval_timer_sent_time = datetime.datetime.utcnow()
self.assertEqual(type(id), str)
# Wait until all events are generated
gevent.sleep(9)
time_diff = (datetime.datetime.utcnow() - self.interval_timer_sent_time).seconds
timer_counts = math.floor(time_diff/self.expire_sec_1) + math.floor(time_diff/self.expire_sec_2)
# After waiting, validate only 2 events are generated.
self.assertEqual(self.tod_count, 2, "Invalid number of timeouts generated. Number of timeout: %d Expected timeout: %d Timer id: %s " %(self.tod_count, timer_counts, id))
# Cancel the timer
self.ssclient.cancel_timer(id)
def tod_callback(self, *args, **kwargs):
tod_receive_time = datetime.datetime.utcnow()
self.tod_count += 1
if self.tod_count == 1:
time_diff = math.fabs((tod_receive_time - self.tod_sent_time).total_seconds() - self.expire_sec_1)
self.assertTrue(time_diff <= 2)
elif self.tod_count == 2:
time_diff = math.fabs((tod_receive_time - self.tod_sent_time).total_seconds() - self.expire_sec_2)
self.assertTrue(time_diff <= 2)
log.debug("test_scheduler: tod_callback: time:" + str(tod_receive_time) + " count:" + str(self.tod_count))
def test_timeoffday_timer_in_past_seconds(self):
# test creating a new timer that is one-time-only
# create the timer resource
# get the current time, set the timer to several seconds from current time
# create the event listener
# call scheduler to set the timer
# verify that event arrival is within one/two seconds of current time
event_origin = "Time_Of_Day3"
expire_sec = -4
self.tod_count2 = 0
now = datetime.datetime.utcnow()
expire1 = now + timedelta(seconds=expire_sec)
# Create two timers
times_of_day = [{'hour': str(expire1.hour), 'minute': str(expire1.minute), 'second': str(expire1.second)}]
sub = EventSubscriber(event_type="TimerEvent", callback=self.tod_callback2, origin=event_origin)
sub.start()
self.addCleanup(sub.stop)
# Expires in 3 days
expires = calendar.timegm((datetime.datetime.utcnow() + timedelta(days=3)).timetuple())
self.tod_sent_time = datetime.datetime.utcnow()
id = self.ssclient.create_time_of_day_timer(times_of_day=times_of_day, expires=expires, event_origin=event_origin, event_subtype="")
self.interval_timer_sent_time = datetime.datetime.utcnow()
self.assertEqual(type(id), str)
# Wait and see if the any events are generated
gevent.sleep(5)
# After waiting, validate no event is generated
self.assertEqual(self.tod_count2, 0, "Invalid number of timeouts generated. Number of timeout: %d Expected timeout: 0 Timer id: %s " %(self.tod_count2, id))
# Cancel the timer
self.ssclient.cancel_timer(id)
# This is example for the following case
# Example current time is 8:00AM. User setups a timer for 6:00AM. Since it is 8am, it tries to
# setup a timer for tomorrow 6am but the expire time is set at 5AM tomorrow
event_origin = "Time_Of_Day4"
expire_sec = -4
self.tod_count2 = 0
now = datetime.datetime.utcnow()
expire1 = now + timedelta(seconds=expire_sec)
times_of_day = [{'hour': str(expire1.hour), 'minute': str(expire1.minute), 'second': str(expire1.second)}]
sub = EventSubscriber(event_type="TimerEvent", callback=self.tod_callback2, origin=event_origin)
sub.start()
self.addCleanup(sub.stop)
# Expires before the first event
time_delta = timedelta(days=1) + timedelta(seconds=-(abs(expire_sec*2))) # Notice the minus sign. It expires before the first event
expires = calendar.timegm((now + time_delta).timetuple())
self.tod_sent_time = datetime.datetime.utcnow()
with self.assertRaises(BadRequest):
id = self.ssclient.create_time_of_day_timer(times_of_day=times_of_day, expires=expires, event_origin=event_origin, event_subtype="")
def tod_callback2(self, *args, **kwargs):
tod_receive_time = datetime.datetime.utcnow()
self.tod_count2 += 1
log.debug("test_scheduler: tod_callback2: time:")
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_quit_stops_timers(self):
ar = AsyncResult()
def cb(*args, **kwargs):
ar.set(args)
self.interval_timer_count += 1
event_origin = "test_quitter"
sub = EventSubscriber(event_type="TimerEvent", callback=cb, origin=event_origin)
sub.start()
self.addCleanup(sub.stop)
tid = self.ssclient.create_interval_timer(start_time="now",
end_time="-1",
interval=1,
event_origin=event_origin)
# wait until at least one scheduled message
ar.get(timeout=5)
# shut it down!
p = self.container.proc_manager.procs_by_name['scheduler']
self.container.terminate_process(p.id)
# assert empty
self.assertEquals(p.schedule_entries, {})
class VizTransformMatplotlibGraphs(TransformStreamPublisher,
TransformEventListener,
TransformStreamListener):
"""
This class is used for instantiating worker processes that have subscriptions to data streams and convert
incoming data from CDM format to Matplotlib graphs
"""
output_bindings = ['graph_image_param_dict']
event_timer_interval = None
def on_start(self):
#print ">>>>>>>>>>>>>>>>>>>>>> MPL CFG = ", self.CFG
self.pubsub_management = PubsubManagementServiceProcessClient(
process=self)
self.ssclient = SchedulerServiceProcessClient(process=self)
self.rrclient = ResourceRegistryServiceProcessClient(process=self)
self.data_retriever_client = DataRetrieverServiceProcessClient(
process=self)
self.dsm_client = DatasetManagementServiceProcessClient(process=self)
self.pubsub_client = PubsubManagementServiceProcessClient(process=self)
self.stream_info = self.CFG.get_safe('process.publish_streams', {})
self.stream_names = self.stream_info.keys()
self.stream_ids = self.stream_info.values()
if not self.stream_names:
raise BadRequest('MPL Transform has no output streams.')
graph_time_periods = self.CFG.get_safe('graph_time_periods')
# If this is meant to be an event driven process, schedule an event to be generated every few minutes/hours
self.event_timer_interval = self.CFG.get_safe('graph_gen_interval')
if self.event_timer_interval:
event_origin = "Interval_Timer_Matplotlib"
sub = EventSubscriber(event_type="ResourceEvent",
callback=self.interval_timer_callback,
origin=event_origin)
sub.start()
self.interval_timer_id = self.ssclient.create_interval_timer(
start_time="now",
interval=self._str_to_secs(self.event_timer_interval),
event_origin=event_origin,
event_subtype="")
super(VizTransformMatplotlibGraphs, self).on_start()
# when tranform is used as a data process
def recv_packet(self, packet, in_stream_route, in_stream_id):
#Check to see if the class instance was set up as a event triggered transform. If yes, skip the packet
if self.event_timer_interval:
return
log.info('Received packet')
mpl_data_granule = VizTransformMatplotlibGraphsAlgorithm.execute(
packet, params=self.get_stream_definition())
for stream_name in self.stream_names:
publisher = getattr(self, stream_name)
publisher.publish(mpl_data_granule)
def get_stream_definition(self):
stream_id = self.stream_ids[0]
stream_def = self.pubsub_management.read_stream_definition(
stream_id=stream_id)
return stream_def._id
def process_event(self, msg, headers):
return
def interval_timer_callback(self, *args, **kwargs):
#Find out the input data product to this process
in_dp_id = self.CFG.get_safe('in_dp_id')
print " >>>>>>>>>>>>>> IN DP ID from cfg : ", in_dp_id
# get the dataset_id associated with the data_product. Need it to do the data retrieval
ds_ids, _ = self.rrclient.find_objects(in_dp_id, PRED.hasDataset,
RT.Dataset, True)
if ds_ids is None or not ds_ids:
return None
# retrieve data for the specified time interval. Setup up query from pass config first
query = {}
param_list_str = self.CFG.get_safe('parameters')
if param_list_str:
query['parameters'] = param_list_str.split(', ')
# append time if not present in list of parameters
if not 'time' in query['parameters']:
query['parameters'].append('time')
query['start_time'] = query['end_time'] = ntplib.system_to_ntp_time(
time.time()) # Now
query['stride_time'] = 1
if self.CFG.get_safe('graph_time_period'):
query['start_time'] = query['end_time'] - self._str_to_secs(
self.CFG.get_safe('graph_time_period'))
#print " >>>>>>>>>>>>>> QUERY = ", query
#retrieved_granule = self.data_retriever_client.retrieve(ds_ids[0],{'start_time':start_time,'end_time':end_time})
retrieved_granule = self.data_retriever_client.retrieve(ds_ids[0],
query=query)
# add extra parameters to query passed in config that are not needed by data retrieval
if self.CFG.get_safe('resolution'):
query['resolution'] = self.CFG.get_safe('resolution')
# send the granule through the Algorithm code to get the matplotlib graphs
mpl_pdict_id = self.dsm_client.read_parameter_dictionary_by_name(
'graph_image_param_dict', id_only=True)
mpl_stream_def = self.pubsub_client.create_stream_definition(
'mpl', parameter_dictionary_id=mpl_pdict_id)
fileName = self.CFG.get_safe('graph_time_period')
mpl_data_granule = VizTransformMatplotlibGraphsAlgorithm.execute(
retrieved_granule,
config=query,
params=mpl_stream_def,
fileName=fileName)
if mpl_data_granule == None:
return None
# publish on all specified output streams
for stream_name in self.stream_names:
publisher = getattr(self, stream_name)
publisher.publish(mpl_data_granule)
return
def _str_to_secs(self, time_period):
# this method converts commonly used time periods to its actual seconds counterpart
#separate alpha and numeric parts of the time period
time_n = time_period.lower().rstrip('abcdefghijklmnopqrstuvwxyz ')
time_a = time_period.lower().lstrip('0123456789. ')
# determine if user specified, secs, mins, hours, days, weeks, months, years
factor = None
if time_a == 'sec' or time_a == "secs" or time_a == 'second' or time_a == "seconds":
factor = 1
if time_a == "min" or time_a == "mins" or time_a == "minute" or time_a == "minutes":
factor = 60
if time_a == "hr" or time_a == "hrs" or time_a == "hour" or time_a == "hours":
factor = 60 * 60
if time_a == "day" or time_a == "days":
factor = 60 * 60 * 24
if time_a == "wk" or time_a == "wks" or time_a == "week" or time_a == "weeks":
factor = 60 * 60 * 24 * 7
if time_a == "mon" or time_a == "mons" or time_a == "month" or time_a == "months":
factor = 60 * 60 * 24 * 30
if time_a == "yr" or time_a == "yrs" or time_a == "year" or time_a == "years":
factor = 60 * 60 * 24 * 365
time_period_secs = float(time_n) * factor
return time_period_secs
def on_quit(self):
#Cancel the timer
if hasattr(self, 'interval_timer_id'):
self.ssclient.cancel_timer(self.interval_timer_id)
super(VizTransformMatplotlibGraphs, self).on_quit()
class VizTransformMatplotlibGraphs(TransformDataProcess, TransformEventListener):
"""
This class is used for instantiating worker processes that have subscriptions to data streams and convert
incoming data from CDM format to Matplotlib graphs
"""
output_bindings = ['graph_image_param_dict']
def on_start(self):
self.pubsub_management = PubsubManagementServiceProcessClient(process=self)
self.ssclient = SchedulerServiceProcessClient(node=self.container.node, process=self)
self.rrclient = ResourceRegistryServiceProcessClient(node=self.container.node, process=self)
self.stream_info = self.CFG.get_safe('process.publish_streams',{})
self.stream_names = self.stream_info.keys()
self.stream_ids = self.stream_info.values()
if not self.stream_names:
raise BadRequest('MPL Transform has no output streams.')
graph_time_periods= self.CFG.get_safe('graph_time_periods')
# If this is meant to be an event driven process, schedule an event to be generated every few minutes/hours
event_timer_interval = self.CFG.get_safe('event_timer_interval')
if event_timer_interval:
event_origin = "Interval_Timer_Matplotlib"
sub = EventSubscriber(event_type="ResourceEvent", callback=self.interval_timer_callback, origin=event_origin)
sub.start()
self.interval_timer_id = self.ssclient.create_interval_timer(start_time="now" , interval=event_timer_interval,
event_origin=event_origin, event_subtype="")
super(VizTransformMatplotlibGraphs,self).on_start()
def recv_packet(self, packet, in_stream_route, in_stream_id):
log.info('Received packet')
outgoing = VizTransformMatplotlibGraphsAlgorithm.execute(packet, params=self.get_stream_definition())
for stream_name in self.stream_names:
publisher = getattr(self, stream_name)
publisher.publish(outgoing)
def get_stream_definition(self):
stream_id = self.stream_ids[0]
stream_def = self.pubsub_management.read_stream_definition(stream_id=stream_id)
return stream_def._id
def process_event(self, msg, headers):
return
def interval_timer_callback(self, *args, **kwargs):
#Find out the input data product to this process
# retrieve data for every case of the output graph
return
def on_quit(self):
#Cancel the timer
if hasattr(self, 'interval_timer_id'):
self.ssclient.cancel_timer(self.interval_timer_id)
super(VizTransformMatplotlibGraphs,self).on_quit()