def test_get_data_json(self): now = datetime.datetime.now() end = now - datetime.timedelta(minutes=1) start = end - datetime.timedelta(minutes=1) query = util.construct_data_query( channels=['SIN-CVME-TIFGUN-EVR0:BUNCH-1-OK'], start=start, end=end) data = client.get_data_json(query) value = data[0]["data"][0] logger.info(value) self.assertTrue("value" in value) self.assertTrue("time" in value) self.assertTrue("pulseId" in value) query = util.construct_data_query( channels=['SIN-CVME-TIFGUN-EVR0:BUNCH-1-OK'], start=start, end=end, value_mapping=util.construct_value_mapping(incomplete="fill-null")) data = client.get_data_json(query) value = data[0][0] # first row first column logger.info(value) self.assertTrue("value" in value) self.assertTrue("time" in value) self.assertTrue("pulseId" in value)
def get_pulse_id_from_timestamp(global_timestamp=None, mapping_channel="SIN-CVME-TIFGUN-EVR0:BEAMOK", base_url=default_base_url): """ Retrieve pulse_id for given timestamp :param global_timestamp: timestamp to retrieve pulseid for - if no timestamp is specified take current time :param mapping_channel: Channel used to determine timestamp <> pulse-id mapping :param base_url: :return: pulse-id for timestamp """ # Use current time if no timestamp is specified if not global_timestamp: global_timestamp = datetime.now() _start = global_timestamp - timedelta(seconds=1) _end = global_timestamp + timedelta(milliseconds=10) # retrieve raw data - data object needs to contain one object for the channel with one data element query = util.construct_data_query(mapping_channel, start=_start, end=_end) data = get_data_json(query, base_url=base_url) if not data[0]["data"]: raise ValueError("Requested timestamp not in data buffer. Cannot determine pulse_id.") pulse_id = data[0]["data"][-1]["pulseId"] # TODO Need to check whether actually does match (check pulse before/after) # pulse_id = data[0]["data"][-1]["globalDate"] return pulse_id
def get_timestamp_from_pulse_id(pulse_ids, mapping_channel="SIN-CVME-TIFGUN-EVR0:BEAMOK", base_url=None): """ Get global data for a given pulse-id :param pulse_ids: list of pulse-ids to retrieve global date for :param mapping_channel: channel that is used to determine pulse-id<>timestamp mapping :param base_url: :return: list of corresponding global timestamps """ if not isinstance(pulse_ids, list): pulse_ids = [pulse_ids] dates = [] for pulse_id in pulse_ids: # retrieve raw data - data object needs to contain one object for the channel with one data element query = util.construct_data_query(mapping_channel, start=pulse_id, range_type="pulseId", event_fields=["pulseId", "time"]) data = get_data_json(query, base_url=base_url) if not pulse_id == data[0]["data"][0]["pulseId"]: raise RuntimeError('Unable to retrieve mapping') dates.append(util.convert_date(data[0]["data"][0]["time"])) if len(pulse_ids) != len(dates): raise RuntimeError("Unable to retrieve mapping") return dates
def test_get_data_idread_json_compare(self): """ Test if get_data_json returns same data as get_data_idread """ now = datetime.datetime.now() end = now - datetime.timedelta(minutes=1) start = end - datetime.timedelta(minutes=1) query = util.construct_data_query( channels=[ 'SIN-CVME-TIFGUN-EVR0:BUNCH-1-OK', # 'SINEG01-RCIR-PUP10:SIG-AMPLT', # 'sf-databuffer/SINEG01-RCIR-PUP10:SIG-AMPLT-MAX' ], start=start, end=end) json_data = client.get_data_json(query) print(json_data[0]["data"][:10]) idread_data = client.get_data_idread(query) print(idread_data[0]["data"][:10]) for i in range(len(json_data[0]["data"])): json_value = json_data[0]["data"][i] idread_value = idread_data[0]["data"][i] self.assertEqual(json_value["pulseId"], idread_value["pulseId"]) self.assertEqual(json_value["value"], idread_value["value"])
def test_get_data_idread_json_compare_image(self): """ Test if get_data_json returns same data as get_data_idread """ query = util.construct_data_query( channels=[ 'SF-IMAGEBUFFER/SLAAR21-LCAM-C511:FPICTURE', # 'SINEG01-RCIR-PUP10:SIG-AMPLT', # 'sf-databuffer/SINEG01-RCIR-PUP10:SIG-AMPLT-MAX' ], start=7928427268, end=7928427268) json_data = client.get_data_json(query) # print(json_data[0]["data"][:10]) idread_data = client.get_data_idread(query) # print(idread_data[0]["data"][:10]) for i in range(len(json_data[0]["data"])): self.assertTrue(json_data[0]["data"][i]["pulseId"] == idread_data[0]["data"][i]["pulseId"]) self.assertTrue( numpy.array_equal(json_data[0]["data"][i]["value"], idread_data[0]["data"][i]["value"]))
def save(args): """CLI Action save""" channels = args.channels.split(',') # Figure out wihch channels have pulse ids db_channels = [] for channel in channels: res = api.search(channel, backends=["sf-databuffer"]) if res['sf-databuffer']: db_channels.append(channel) aa_channels = [] for channel in channels: if channel not in db_channels: aa_channels.append(channel) if args.from_pulse != -1 and args.to_pulse != -1: if aa_channels: logger.error( "Cannot search archiver appliance channels with pulse " "ids. The following channels were not found in data buffer: " "%s", aa_channels) # If range is pulse ids query = util.construct_data_query( channels=db_channels, start=args.from_pulse, end=args.to_pulse, range_type='pulseId', event_fields=["value", "pulseId", "timeRaw"]) res = api.get_data_idread(query) else: # If range is time query = util.construct_data_query(channels=aa_channels, start=args.from_time, end=args.to_time, event_fields=["value", "timeRaw"]) res = api.get_data_idread(query) # If range is time query = util.construct_data_query( channels=db_channels, start=args.from_time, end=args.to_time, event_fields=["value", "pulseId", "timeRaw"]) res += api.get_data_idread(query) to_hdf5(res, args.filename, overwrite=args.overwrite) return 0
def test_construct_data_query(self): raised = False try: util.construct_data_query("CHANNEL_A") except ValueError: raised = True self.assertTrue(raised) query = util.construct_data_query("CHANNEL_A", start=10) self.assertEqual(len(query["channels"]), 1) self.assertEqual(query["channels"][0]["name"], "CHANNEL_A") self.assertNotIn("backend", query["channels"][0]) self.assertNotIn("ordering", query) logger.info(query) query = util.construct_data_query("backend01/CHANNEL_A", start=10) self.assertEqual(len(query["channels"]), 1) self.assertEqual(query["channels"][0]["name"], "CHANNEL_A") self.assertEqual(query["channels"][0]["backend"], "backend01") self.assertNotIn("ordering", query) logger.info(query) # Ordering checks query = util.construct_data_query("backend01/CHANNEL_A", start=10, ordering='asc') self.assertEqual(query["ordering"], "asc") raised = False try: util.construct_data_query("backend01/CHANNEL_A", start=10, ordering='bla') except ValueError: raised = True self.assertTrue(raised)
def test_get_data_aggregation(self): # If test fails check whether channel currently has data now = datetime.datetime.now() - datetime.timedelta(hours=10) query = util.construct_data_query(["SIN-CVME-TIFGUN-EVR0:BEAMOK"], start=now, delta_range=100, aggregation=util.construct_aggregation(nr_of_bins=100)) data = client.get_data(query) logger.info(data) self.assertEqual(len(data[0]["data"]), 100)
def test_get_data_idread(self): now = datetime.datetime.now() end = now - datetime.timedelta(minutes=1) start = end - datetime.timedelta(minutes=1) query = util.construct_data_query( channels=['SIN-CVME-TIFGUN-EVR0:BUNCH-1-OK'], start=start, end=end) data = client.get_data_idread(query) logger.info(data[0]["data"][0]) self.assertTrue(True)
def test_local_get_data(self): # Only works if the testserver.py server is running now = datetime.datetime.now() end = now start = end - datetime.timedelta(minutes=10) query = util.construct_data_query(channels=["A", "B"], start=start, end=end) data = client.get_data(query, base_url="http://localhost:8080/archivertestdata") # Test function returns 10 datapoints with values from 0 to 9 self.assertEqual(len(data[0]["data"]), 10) for i in range(10): self.assertEqual(data[0]["data"][i]["value"], i)
def test_get_data_long_timerange(self): # If this test fails check whether the used channels are currently available in the databuffer / archiver now = datetime.datetime.now() end = now - datetime.timedelta(minutes=1) start = end - datetime.timedelta(hours=12) query = util.construct_data_query(channels=['sf-archiverappliance/S10CB02-CVME-ILK:CENTRAL-CORETEMP', 'sf-archiverappliance/S10CB02-CVME-ILK:CENTRAL-CORETEMP2'], start=start, end=end) data = client.get_data_json(query) logger.info(data[0]) self.assertTrue(True)
def test_request(self): # This test will fail if the production backend is not available or there is no data for the requested channel base_url = "https://data-api.psi.ch/sf" end = datetime.datetime.now() start = end - datetime.timedelta(minutes=10) query = util.construct_data_query( channels=["SINEG01-RCIR-PUP10:SIG-AMPLT"], start=start, end=end, response=util.construct_response(format="rawevent")) with requests.post(base_url + '/query', json=query, stream=True) as response: idread_util.decode(response.raw) self.assertTrue(True)
def test_get_data_server_side_mapping(self): # When server side mapping is enabled the returned dictionary is different! # If this test fails check whether the used channels are currently available in the databuffer / archiver now = datetime.datetime.now() end = now - datetime.timedelta(minutes=1) start = end - datetime.timedelta(minutes=1) query = util.construct_data_query(channels=['SIN-CVME-TIFGUN-EVR0:BEAMOK', 'sf-archiverappliance/S10CB01-CVME-ILK:P2020-CORETEMP'], start=start, end=end, # value_mapping=util.construct_value_mapping(incomplete="fill-null")) # value_mapping=util.construct_value_mapping(incomplete="drop")) value_mapping=util.construct_value_mapping(incomplete="provide-as-is")) json_data = client.get_data_json(query) idread_data = client.get_data_idread(query) for i in range(len(json_data)): # logger.info(data['data'][i]) json_value = json_data[i] idread_value = idread_data[i] # self.assertTrue(len(json_value) == len(idread_value)) if json_value[0] is None: self.assertEqual(json_value[0], idread_value[0], "failed on index: %d" % i) else: self.assertTrue(json_value[0]["channel"] == "SIN-CVME-TIFGUN-EVR0:BEAMOK", "failed on index: %d" % i) self.assertTrue(idread_value[0]["channel"] == "SIN-CVME-TIFGUN-EVR0:BEAMOK", "failed on index: %d" % i) self.assertEqual(json_value[0]["value"], idread_value[0]["value"], "failed on index: %d" % i) if json_value[1] is None: self.assertEqual(json_value[1], idread_value[1], "failed on index: %d" % i) else: self.assertTrue(json_value[1]["channel"] == "S10CB01-CVME-ILK:P2020-CORETEMP", "failed on index: %d" % i) self.assertTrue(idread_value[1]["channel"] == "S10CB01-CVME-ILK:P2020-CORETEMP", "failed on index: %d" % i) self.assertEqual(json_value[1]["value"], idread_value[1]["value"], "failed on index: %d" % i)