def test_unsubscribe_scan(self):
controller.subscribe(self.long_pvs[0], SubscriptionMode.Scan(period=1))
#check that the system has registered the subscription
apv = controller.get_apv(self.long_pvs[0])
self.assertEqual( apv.name, self.long_pvs[0] )
# wait for a while so that we gather some data
time.sleep(3)
# request unsubscription
unsub_req = controller.unsubscribe(self.long_pvs[0])
self.assertTrue(unsub_req)
unsub_req.wait()
before = controller.get_values(self.long_pvs[0])
# verify that the system has removed the pv
apv = controller.get_apv(self.long_pvs[0])
self.assertFalse(apv.subscribed)
# wait to see if we are still receiving data
time.sleep(3)
after = controller.get_values(self.long_pvs[0])
# we shouldn't have received any data during the last sleep period
self.assertEqual(len(before), len(after))
# and the scan task's timer should have been cleaned up
self.assertEqual(controller.timers.num_active_tasks, 0)
def test_get_values_throttled(self):
# give it time to gather some data
wait_for = 5
ioc_freq = 10 # Hz
max_archiving_freq = 5 # Hz
mode = SubscriptionMode.Monitor(delta=0.01,max_freq=max_archiving_freq)
self.test_subscribe(mode)
time.sleep(wait_for)
no_values = controller.get_values('test:doesntexist')
self.assertEqual(no_values['rows'], [])
for pvset in (self.long_pvs, self.double_pvs):
for pv in pvset:
values = controller.get_values(pv)
logger.info("Received %d values for '%s' after %d seconds at %d Hz", \
len(values['rows']), pv, wait_for, max_archiving_freq)
# diferences in timestamp shouldn't exceed 1/max_archiving_freq (period)
max_period = 1/max_archiving_freq
for i in range(len(values['rows'])-1):
ith = values['rows'][i]['timestamp']
ip1th = values['rows'][i+1]['timestamp']
self.assertGreaterEqual( ith-ip1th, max_period)
def test_unsubscribe_monitor(self):
pvname = self.long_pvs[0]
controller.subscribe(pvname, SubscriptionMode.Monitor(delta=0.01))
#check that the system has registered the subscription
apv = controller.get_apv(pvname)
self.assertEqual( apv.name, self.long_pvs[0] )
# wait for a while so that we gather some data
time.sleep(2)
# request unsubscription
unsub_req = controller.unsubscribe(self.long_pvs[0])
self.assertTrue(unsub_req)
unsub_req.wait()
# take note of the data present
before = controller.get_values(self.long_pvs[0])['rows']
# verify that the system has removed the pv
apv = controller.get_apv(self.long_pvs[0])
self.assertFalse(apv.subscribed)
# wait to see if we are still receiving data
time.sleep(2)
after = controller.get_values(self.long_pvs[0])['rows']
# we shouldn't have received any data during the last sleep period
self.assertEqual(len(before), len(after))
def test_get_values_by_date(self):
self.test_subscribe()
wait_for = 5
start = datetime.datetime.now()
halftime = start + datetime.timedelta(seconds=wait_for/2.0)
halftime_ts = time.mktime( halftime.timetuple())
time.sleep(wait_for)
# test "up to" semantics, open interval on the left
for pvset in (self.long_pvs, self.double_pvs):
for pv in pvset:
all_values = controller.get_values(pv, limit=2**30)
upto_values = controller.get_values(pv, to_date=halftime, limit=2**30) # first half
from_values = controller.get_values(pv, from_date=halftime, limit=2**30) # second half
ts = time.mktime( halftime.timetuple())
for v in upto_values['rows']:
self.assertLessEqual(v['timestamp'], ts)
for v in from_values['rows']:
self.assertGreaterEqual(v['timestamp'], ts)
# it may happen that new values arrive between the gathering of values and the
# counting colums operation
self.assertLessEqual(len(all_values['rows']), all_values['meta']['total'])
self.assertLessEqual(len(upto_values['rows']), upto_values['meta']['total'])
self.assertLessEqual(upto_values['meta']['last_ts'], halftime_ts)
self.assertGreaterEqual(from_values['meta']['first_ts'], halftime_ts)
self.assertLessEqual(len(from_values['rows']), from_values['meta']['total'])
# it's >= because there may be some overlap, due to the loss of precision in the
# timestamp we use in the get_values call
self.assertGreaterEqual( len(upto_values['rows'])+len(from_values['rows']), len(all_values['rows']) )
def test_subscribe_scan(self):
for pv in self.pvs[:4]:
self.assertEqual(len(controller.get_values(pv)['rows']), 0)
periods = (1,1.5, 2, 2.1)
for pv,p in zip(self.pvs[:4], periods):
controller.subscribe(pv, SubscriptionMode.Scan(period=p))
sleep_for = 10
time.sleep(sleep_for)
for pv,p in zip(self.pvs[:4], periods):
res = controller.get_values(pv)['rows']
self.assertAlmostEqual(sleep_for/p, len(res), delta=1)
def test_get_values(self):
self.test_subscribe()
# give it time to gather some data
wait_for = 10
ioc_freq = 10 # Hz
time.sleep(wait_for) # ioc generates values at 1 Hz
expected_num_values = wait_for * ioc_freq
# doesn't exist AND never subscribed
no_values = controller.get_values('test:doesntexist')
self.assertEqual(no_values['rows'], [])
self.assertEqual(no_values['meta']['total'], 0)
for longpv in self.long_pvs:
values = controller.get_values(longpv, limit=expected_num_values)
rows = values['rows']
logger.info("Received %d values after %d seconds at %d Hz", len(rows), wait_for, ioc_freq)
self.assertEqual(len(rows), expected_num_values)
# we don't really know how for from a change the PV is. We
# can only guarantee that we'll get expected_num_values
# updates during our sleep period
first_value = int(rows[0]['value'])
for (i,value) in enumerate(rows):
self.assertEqual(longpv, value['pvname'])
self.assertEqual('long', value['type'])
self.assertEqual(first_value-i, int(value['value']))
for doublepv in self.double_pvs:
values = controller.get_values(doublepv, limit=expected_num_values)
rows = values['rows']
logger.info("Received %d values after %d seconds at %d Hz", len(rows), wait_for, ioc_freq)
self.assertEqual(len(rows), expected_num_values)
first_value = float(rows[0]['value'])
print(first_value)
for (i,value) in enumerate(rows):
self.assertEqual(doublepv, value['pvname'])
self.assertEqual('double', value['type'])
self.assertEqual(first_value-i, float(value['value']))
for fakepv in self.fake_pvs:
values = controller.get_values(fakepv)
# they don't exist but we subscribed to them. Expect entries with null values
rows = values['rows']
for row in rows:
self.assertIsNone(row['value'])
def test_get_values_disconnected(self):
conn_ts = datastore._get_timestamp_ms()
futures = [ controller.subscribe(pv, SubscriptionMode.Monitor(delta=0.01)) \
for pv in self.existing_pvs ]
results = [ fut.get() for fut in futures ]
time.sleep(2)
# when a pv gets disconnected, NaN should be stored as its value.
# moreover, the pv's status should reflect the disconnection at that
# point in time
for pv in self.existing_pvs:
statuses = controller.get_statuses(pv)
self.assertTrue(statuses)
status = statuses[0]
self.assertTrue(status['connected'])
last_values = controller.get_values(pv)
self.assertTrue(last_values['rows'])
last_value = last_values['rows'][0]
self.assertGreater(last_value['archived_at_ts'], conn_ts)
self.assertGreater(last_value['value'], 1)
predisconn_ts = datastore._get_timestamp_ms()
_block_ioc(True)
time.sleep(35)
postdisconn_ts = datastore._get_timestamp_ms()
for pv in self.existing_pvs:
statuses = controller.get_statuses(pv)
self.assertTrue(statuses)
status = statuses[0]
self.assertFalse(status['connected'])
last_values = controller.get_values(pv)
self.assertTrue(last_values['rows'])
last_value = last_values['rows'][0]
self.assertGreater(last_value['archived_at_ts'], predisconn_ts)
self.assertGreater(postdisconn_ts, last_value['archived_at_ts'])
self.assertIsNone(last_value['value'])
_block_ioc(False)
def test_constant_value_scan(self):
pv = self.nonchanging_pvs[0]
controller.subscribe(pv, SubscriptionMode.Scan(period=1))
time.sleep(4)
values = controller.get_values(self.nonchanging_pvs[0])
rows = values['rows']
self.assertGreaterEqual(len(rows), 3)
self.assertLessEqual(len(rows), 4)
last_value = rows[-1] # ie, oldest
for i,value in enumerate(reversed(rows[:-1])):
self.assertAlmostEqual(value['archived_at_ts']-(i+1)*1e6, last_value['archived_at_ts'], delta=0.1*1e6)
self.assertEqual(value['timestamp'], last_value['timestamp'])
def test_get_values_delta(self):
# give it time to gather some data
wait_for = 10
ioc_freq = 10 # Hz
delta = 2
mode = SubscriptionMode.Monitor(delta=delta)
self.test_subscribe(mode)
time.sleep(wait_for)
no_values = controller.get_values('test:doesntexist')
self.assertEqual(no_values['rows'], [])
for pvset in (self.long_pvs, self.double_pvs):
for pv in pvset:
values = controller.get_values(pv)
rows = values['rows']
logger.info("Received %d values for '%s' after %d seconds", \
len(rows), pv, wait_for)
for i in range(len(rows)-1):
ith = rows[i]['value']
ip1th = rows[i+1]['value']
self.assertGreaterEqual( ith-ip1th, delta)
def get(self, pvname):
fields = self.get_arguments("field")
limit = int(self.get_argument("limit", default=10))
from_ts = self.get_argument("from_ts", default=None)
to_ts = self.get_argument("to_ts", default=None)
nextpage = self.get_argument("nextpage", None)
prevpage = self.get_argument("prevpage", None)
reverse = True
if nextpage:
from_ts = nextpage
if prevpage:
from_ts = prevpage
reverse = False
next_url = None
prev_url = None
at_first_page = False
rows = controller.get_values(pvname, fields, limit + 1, from_ts, to_ts, reverse)
count_total = controller.get_count(pvname)
count_remaining = controller.get_count(pvname, from_ts)
if prevpage: # we come from a prevpage link
# if a prevpage has been requested while already at the first page,
# a single row is returned, that with the prevpage's pointer value
# in its archived_at_ts field
at_first_page = len(rows) == 1
if not at_first_page:
rows.reverse()
if rows:
# check if we are at the end
if len(rows) > limit or at_first_page: # not at the end
last = rows.pop()
next_ts = last["archived_at_ts"]
else: # at the end
next_ts = None
current_url = self.request.full_url()
url_parts = urlparse.urlparse(current_url)
qargs = urlparse.parse_qsl(url_parts.query)
qargs[:] = [qarg for qarg in qargs if (qarg[0] not in ("nextpage", "prevpage"))]
if next_ts:
qargs.append(("nextpage", next_ts))
qs = urllib.urlencode(qargs)
next_url = urlparse.urlunparse(url_parts._replace(query=qs))
qargs.pop() # restore, get ready for prevpage
if not at_first_page:
curr_hdr_st = rows[0]["archived_at_ts"]
qargs.append(("prevpage", curr_hdr_st))
qs = urllib.urlencode(qargs)
prev_url = urlparse.urlunparse(url_parts._replace(query=qs))
self.win(
{
"rows": rows,
"count_total": count_total,
"count_remaining": count_remaining,
"nextpage": next_url,
"prevpage": prev_url,
}
)