def test_get_data_json(self):
now = datetime.datetime.now()
end = now - datetime.timedelta(minutes=1)
start = end - datetime.timedelta(minutes=1)
query = util.construct_data_query(
channels=['SIN-CVME-TIFGUN-EVR0:BUNCH-1-OK'], start=start, end=end)
data = client.get_data_json(query)
value = data[0]["data"][0]
logger.info(value)
self.assertTrue("value" in value)
self.assertTrue("time" in value)
self.assertTrue("pulseId" in value)
query = util.construct_data_query(
channels=['SIN-CVME-TIFGUN-EVR0:BUNCH-1-OK'],
start=start,
end=end,
value_mapping=util.construct_value_mapping(incomplete="fill-null"))
data = client.get_data_json(query)
value = data[0][0] # first row first column
logger.info(value)
self.assertTrue("value" in value)
self.assertTrue("time" in value)
self.assertTrue("pulseId" in value)
def get_pulse_id_from_timestamp(global_timestamp=None, mapping_channel="SIN-CVME-TIFGUN-EVR0:BEAMOK",
base_url=default_base_url):
"""
Retrieve pulse_id for given timestamp
:param global_timestamp: timestamp to retrieve pulseid for - if no timestamp is specified take current time
:param mapping_channel: Channel used to determine timestamp <> pulse-id mapping
:param base_url:
:return: pulse-id for timestamp
"""
# Use current time if no timestamp is specified
if not global_timestamp:
global_timestamp = datetime.now()
_start = global_timestamp - timedelta(seconds=1)
_end = global_timestamp + timedelta(milliseconds=10)
# retrieve raw data - data object needs to contain one object for the channel with one data element
query = util.construct_data_query(mapping_channel, start=_start, end=_end)
data = get_data_json(query, base_url=base_url)
if not data[0]["data"]:
raise ValueError("Requested timestamp not in data buffer. Cannot determine pulse_id.")
pulse_id = data[0]["data"][-1]["pulseId"]
# TODO Need to check whether actually does match (check pulse before/after)
# pulse_id = data[0]["data"][-1]["globalDate"]
return pulse_id
def get_timestamp_from_pulse_id(pulse_ids, mapping_channel="SIN-CVME-TIFGUN-EVR0:BEAMOK", base_url=None):
"""
Get global data for a given pulse-id
:param pulse_ids: list of pulse-ids to retrieve global date for
:param mapping_channel: channel that is used to determine pulse-id<>timestamp mapping
:param base_url:
:return: list of corresponding global timestamps
"""
if not isinstance(pulse_ids, list):
pulse_ids = [pulse_ids]
dates = []
for pulse_id in pulse_ids:
# retrieve raw data - data object needs to contain one object for the channel with one data element
query = util.construct_data_query(mapping_channel, start=pulse_id, range_type="pulseId", event_fields=["pulseId", "time"])
data = get_data_json(query, base_url=base_url)
if not pulse_id == data[0]["data"][0]["pulseId"]:
raise RuntimeError('Unable to retrieve mapping')
dates.append(util.convert_date(data[0]["data"][0]["time"]))
if len(pulse_ids) != len(dates):
raise RuntimeError("Unable to retrieve mapping")
return dates
def test_get_data_idread_json_compare(self):
"""
Test if get_data_json returns same data as get_data_idread
"""
now = datetime.datetime.now()
end = now - datetime.timedelta(minutes=1)
start = end - datetime.timedelta(minutes=1)
query = util.construct_data_query(
channels=[
'SIN-CVME-TIFGUN-EVR0:BUNCH-1-OK',
# 'SINEG01-RCIR-PUP10:SIG-AMPLT',
# 'sf-databuffer/SINEG01-RCIR-PUP10:SIG-AMPLT-MAX'
],
start=start,
end=end)
json_data = client.get_data_json(query)
print(json_data[0]["data"][:10])
idread_data = client.get_data_idread(query)
print(idread_data[0]["data"][:10])
for i in range(len(json_data[0]["data"])):
json_value = json_data[0]["data"][i]
idread_value = idread_data[0]["data"][i]
self.assertEqual(json_value["pulseId"], idread_value["pulseId"])
self.assertEqual(json_value["value"], idread_value["value"])
def test_get_data_idread_json_compare_image(self):
"""
Test if get_data_json returns same data as get_data_idread
"""
query = util.construct_data_query(
channels=[
'SF-IMAGEBUFFER/SLAAR21-LCAM-C511:FPICTURE',
# 'SINEG01-RCIR-PUP10:SIG-AMPLT',
# 'sf-databuffer/SINEG01-RCIR-PUP10:SIG-AMPLT-MAX'
],
start=7928427268,
end=7928427268)
json_data = client.get_data_json(query)
# print(json_data[0]["data"][:10])
idread_data = client.get_data_idread(query)
# print(idread_data[0]["data"][:10])
for i in range(len(json_data[0]["data"])):
self.assertTrue(json_data[0]["data"][i]["pulseId"] ==
idread_data[0]["data"][i]["pulseId"])
self.assertTrue(
numpy.array_equal(json_data[0]["data"][i]["value"],
idread_data[0]["data"][i]["value"]))
def save(args):
"""CLI Action save"""
channels = args.channels.split(',')
# Figure out wihch channels have pulse ids
db_channels = []
for channel in channels:
res = api.search(channel, backends=["sf-databuffer"])
if res['sf-databuffer']:
db_channels.append(channel)
aa_channels = []
for channel in channels:
if channel not in db_channels:
aa_channels.append(channel)
if args.from_pulse != -1 and args.to_pulse != -1:
if aa_channels:
logger.error(
"Cannot search archiver appliance channels with pulse "
"ids. The following channels were not found in data buffer: "
"%s", aa_channels)
# If range is pulse ids
query = util.construct_data_query(
channels=db_channels,
start=args.from_pulse,
end=args.to_pulse,
range_type='pulseId',
event_fields=["value", "pulseId", "timeRaw"])
res = api.get_data_idread(query)
else:
# If range is time
query = util.construct_data_query(channels=aa_channels,
start=args.from_time,
end=args.to_time,
event_fields=["value", "timeRaw"])
res = api.get_data_idread(query)
# If range is time
query = util.construct_data_query(
channels=db_channels,
start=args.from_time,
end=args.to_time,
event_fields=["value", "pulseId", "timeRaw"])
res += api.get_data_idread(query)
to_hdf5(res, args.filename, overwrite=args.overwrite)
return 0
def test_construct_data_query(self):
raised = False
try:
util.construct_data_query("CHANNEL_A")
except ValueError:
raised = True
self.assertTrue(raised)
query = util.construct_data_query("CHANNEL_A", start=10)
self.assertEqual(len(query["channels"]), 1)
self.assertEqual(query["channels"][0]["name"], "CHANNEL_A")
self.assertNotIn("backend", query["channels"][0])
self.assertNotIn("ordering", query)
logger.info(query)
query = util.construct_data_query("backend01/CHANNEL_A", start=10)
self.assertEqual(len(query["channels"]), 1)
self.assertEqual(query["channels"][0]["name"], "CHANNEL_A")
self.assertEqual(query["channels"][0]["backend"], "backend01")
self.assertNotIn("ordering", query)
logger.info(query)
# Ordering checks
query = util.construct_data_query("backend01/CHANNEL_A", start=10, ordering='asc')
self.assertEqual(query["ordering"], "asc")
raised = False
try:
util.construct_data_query("backend01/CHANNEL_A", start=10, ordering='bla')
except ValueError:
raised = True
self.assertTrue(raised)
def test_get_data_aggregation(self):
# If test fails check whether channel currently has data
now = datetime.datetime.now() - datetime.timedelta(hours=10)
query = util.construct_data_query(["SIN-CVME-TIFGUN-EVR0:BEAMOK"], start=now, delta_range=100,
aggregation=util.construct_aggregation(nr_of_bins=100))
data = client.get_data(query)
logger.info(data)
self.assertEqual(len(data[0]["data"]), 100)
def test_get_data_idread(self):
now = datetime.datetime.now()
end = now - datetime.timedelta(minutes=1)
start = end - datetime.timedelta(minutes=1)
query = util.construct_data_query(
channels=['SIN-CVME-TIFGUN-EVR0:BUNCH-1-OK'], start=start, end=end)
data = client.get_data_idread(query)
logger.info(data[0]["data"][0])
self.assertTrue(True)
def test_local_get_data(self): # Only works if the testserver.py server is running
now = datetime.datetime.now()
end = now
start = end - datetime.timedelta(minutes=10)
query = util.construct_data_query(channels=["A", "B"], start=start, end=end)
data = client.get_data(query, base_url="http://localhost:8080/archivertestdata")
# Test function returns 10 datapoints with values from 0 to 9
self.assertEqual(len(data[0]["data"]), 10)
for i in range(10):
self.assertEqual(data[0]["data"][i]["value"], i)
def test_get_data_long_timerange(self):
# If this test fails check whether the used channels are currently available in the databuffer / archiver
now = datetime.datetime.now()
end = now - datetime.timedelta(minutes=1)
start = end - datetime.timedelta(hours=12)
query = util.construct_data_query(channels=['sf-archiverappliance/S10CB02-CVME-ILK:CENTRAL-CORETEMP',
'sf-archiverappliance/S10CB02-CVME-ILK:CENTRAL-CORETEMP2'],
start=start, end=end)
data = client.get_data_json(query)
logger.info(data[0])
self.assertTrue(True)
def test_request(self):
# This test will fail if the production backend is not available or there is no data for the requested channel
base_url = "https://data-api.psi.ch/sf"
end = datetime.datetime.now()
start = end - datetime.timedelta(minutes=10)
query = util.construct_data_query(
channels=["SINEG01-RCIR-PUP10:SIG-AMPLT"],
start=start,
end=end,
response=util.construct_response(format="rawevent"))
with requests.post(base_url + '/query', json=query,
stream=True) as response:
idread_util.decode(response.raw)
self.assertTrue(True)
def test_get_data_server_side_mapping(self):
# When server side mapping is enabled the returned dictionary is different!
# If this test fails check whether the used channels are currently available in the databuffer / archiver
now = datetime.datetime.now()
end = now - datetime.timedelta(minutes=1)
start = end - datetime.timedelta(minutes=1)
query = util.construct_data_query(channels=['SIN-CVME-TIFGUN-EVR0:BEAMOK',
'sf-archiverappliance/S10CB01-CVME-ILK:P2020-CORETEMP'],
start=start, end=end,
# value_mapping=util.construct_value_mapping(incomplete="fill-null"))
# value_mapping=util.construct_value_mapping(incomplete="drop"))
value_mapping=util.construct_value_mapping(incomplete="provide-as-is"))
json_data = client.get_data_json(query)
idread_data = client.get_data_idread(query)
for i in range(len(json_data)):
# logger.info(data['data'][i])
json_value = json_data[i]
idread_value = idread_data[i]
# self.assertTrue(len(json_value) == len(idread_value))
if json_value[0] is None:
self.assertEqual(json_value[0], idread_value[0], "failed on index: %d" % i)
else:
self.assertTrue(json_value[0]["channel"] == "SIN-CVME-TIFGUN-EVR0:BEAMOK", "failed on index: %d" % i)
self.assertTrue(idread_value[0]["channel"] == "SIN-CVME-TIFGUN-EVR0:BEAMOK", "failed on index: %d" % i)
self.assertEqual(json_value[0]["value"], idread_value[0]["value"], "failed on index: %d" % i)
if json_value[1] is None:
self.assertEqual(json_value[1], idread_value[1], "failed on index: %d" % i)
else:
self.assertTrue(json_value[1]["channel"] == "S10CB01-CVME-ILK:P2020-CORETEMP", "failed on index: %d" % i)
self.assertTrue(idread_value[1]["channel"] == "S10CB01-CVME-ILK:P2020-CORETEMP", "failed on index: %d" % i)
self.assertEqual(json_value[1]["value"], idread_value[1]["value"], "failed on index: %d" % i)
