def test_transform(self, input_filter_string, input_reading_block,
expected_return):
jqfilter_instance = JQFilter()
with patch.object(pyjq, "all",
return_value=expected_return) as mock_pyjq:
ret = jqfilter_instance.transform(input_filter_string,
input_reading_block)
assert ret == expected_return
mock_pyjq.assert_called_once_with(input_reading_block,
input_filter_string)
def test_transform_exceptions(self, input_filter_string,
input_reading_block, expected_error,
expected_log):
jqfilter_instance = JQFilter()
with patch.object(pyjq, "all",
side_effect=expected_error) as mock_pyjq:
with patch.object(jqfilter_instance._logger, "error") as log:
with pytest.raises(expected_error):
jqfilter_instance.transform(input_filter_string,
input_reading_block)
mock_pyjq.assert_called_once_with(input_reading_block,
input_filter_string)
log.assert_called_once_with(expected_log, '')
def _send_data_block(self, stream_id):
""" Sends a block of data to the destination using the configured plugin
Args:
Returns:
Raises:
Todo:
"""
data_sent = False
SendingProcess._logger.debug("{0} - ".format("_send_data_block"))
try:
last_object_id = self._last_object_id_read(stream_id)
data_to_send = self._load_data_into_memory(last_object_id)
if data_to_send:
if self._config_from_manager['applyFilter']["value"].upper() == "TRUE":
jqfilter = JQFilter()
# Steps needed to proper format the data generated by the JQFilter to the one expected by the SP
data_to_send_2 = jqfilter.transform(data_to_send, self._config_from_manager['filterRule']["value"])
data_to_send_3 = json.dumps(data_to_send_2)
del data_to_send_2
data_to_send_4 = eval(data_to_send_3)
del data_to_send_3
data_to_send = data_to_send_4[0]
del data_to_send_4
data_sent, new_last_object_id, num_sent = self._plugin.plugin_send(self._plugin_handle, data_to_send, stream_id)
if data_sent:
# Updates reached position, statistics and logs the operation within the Storage Layer
self._last_object_id_update(new_last_object_id, stream_id)
self._update_statistics(num_sent, stream_id)
loop = asyncio.get_event_loop()
loop.run_until_complete(self._audit.information(self._AUDIT_CODE, {"sentRows": num_sent}))
except Exception:
_message = _MESSAGES_LIST["e000006"]
SendingProcess._logger.error(_message)
raise
return data_sent
async def _task_fetch_data(self):
""" Read data from the Storage Layer into a memory structure"""
try:
last_object_id = await self._last_object_id_read()
self._memory_buffer_fetch_idx = 0
sleep_time = self.TASK_FETCH_SLEEP
sleep_num_increments = 1
while self._task_fetch_data_run:
slept = False
if self._memory_buffer_fetch_idx < self._config['memory_buffer_size']:
# Checks if there is enough space to load a new block of data
if self._memory_buffer[self._memory_buffer_fetch_idx] is None:
try:
data_to_send = await self._load_data_into_memory(last_object_id)
except Exception as ex:
_message = _MESSAGES_LIST["e000028"].format(ex)
SendingProcess._logger.error(_message)
await self._audit.failure(self._AUDIT_CODE, {"error - on _task_fetch_data": _message})
data_to_send = False
slept = True
await asyncio.sleep(sleep_time)
if data_to_send:
# Handles the JQFilter functionality
if self._config_from_manager['applyFilter']["value"].upper() == "TRUE":
jqfilter = JQFilter()
# Steps needed to proper format the data generated by the JQFilter
# to the one expected by the SP
data_to_send_2 = jqfilter.transform(data_to_send,
self._config_from_manager['filterRule']["value"])
data_to_send_3 = json.dumps(data_to_send_2)
del data_to_send_2
data_to_send_4 = eval(data_to_send_3)
del data_to_send_3
data_to_send = data_to_send_4[0]
del data_to_send_4
# Loads the block of data into the in memory buffer
self._memory_buffer[self._memory_buffer_fetch_idx] = data_to_send
last_position = len(data_to_send) - 1
last_object_id = data_to_send[last_position]['id']
self._memory_buffer_fetch_idx += 1
self._task_fetch_data_sem.release()
self.performance_track("task _task_fetch_data")
else:
# There is no more data to load
slept = True
await asyncio.sleep(sleep_time)
else:
# There is no more space in the in memory buffer
await self._task_send_data_sem.acquire()
else:
self._memory_buffer_fetch_idx = 0
# Handles the sleep time, it is doubled every time up to a limit
if slept:
sleep_num_increments += 1
sleep_time *= 2
if sleep_num_increments > self.TASK_SLEEP_MAX_INCREMENTS:
sleep_time = self.TASK_FETCH_SLEEP
sleep_num_increments = 1
except Exception as ex:
_message = _MESSAGES_LIST["e000028"].format(ex)
SendingProcess._logger.error(_message)
await self._audit.failure(self._AUDIT_CODE, {"error - on _task_fetch_data": _message})
raise
class TestJQFilter:
"""
JQ Filter Tests
- Test that north plugins can load and apply JQ filter
- Test that correct results are returned after applying JQ filter
"""
_name = "JQFilter"
# TODO: How to eliminate manual intervention as below when tests will run unattended at CI?
_core_management_port = pytest.test_env.core_mgmt_port
_core_management_host = "localhost"
_storage_client = StorageClient("localhost", _core_management_port)
_readings = ReadingsStorageClient("localhost", _core_management_port)
_cfg_manager = ConfigurationManager(_storage_client)
# Configuration related to JQ Filter
_CONFIG_CATEGORY_NAME ="JQ_FILTER"
_CONFIG_CATEGORY_DESCRIPTION = "JQ configuration"
_DEFAULT_FILTER_CONFIG = {
"applyFilter": {
"description": "Whether to apply filter before processing the data",
"type": "boolean",
"default": "False"
},
"filterRule": {
"description": "JQ formatted filter to apply (applicable if applyFilter is True)",
"type": "string",
"default": ".[]"
}
}
_first_read_id = None
_raw_data = None
_jqfilter = JQFilter()
@classmethod
def set_configuration(cls):
"""" set the default configuration for plugin
:return:
Configuration information that will be set for any north plugin
"""
event_loop = asyncio.get_event_loop()
event_loop.run_until_complete(cls._cfg_manager.create_category(cls._CONFIG_CATEGORY_NAME,
cls._DEFAULT_FILTER_CONFIG,
cls._CONFIG_CATEGORY_DESCRIPTION))
return event_loop.run_until_complete(cls._cfg_manager.get_category_all_items(cls._CONFIG_CATEGORY_NAME))
@classmethod
@pytest.fixture(scope="class", autouse=True)
def init_test(cls):
"""Setup and Cleanup method, executed once for the entire test class"""
cls.set_configuration()
cls._first_read_id = cls._insert_readings_data()
cls._insert_readings_data()
payload = PayloadBuilder()\
.WHERE(['id', '>=', cls._first_read_id]) \
.ORDER_BY(['id', 'ASC']) \
.payload()
readings = cls._readings.query(payload)
cls._raw_data = readings['rows']
yield
# Delete all test data from readings and configuration
cls._storage_client.delete_from_tbl("readings", {})
payload = PayloadBuilder().WHERE(["key", "=", cls._CONFIG_CATEGORY_NAME]).payload()
cls._storage_client.delete_from_tbl("configuration", payload)
@classmethod
def _insert_readings_data(cls):
"""Insert reads in readings table
args:
:return:
The id of inserted row
"""
readings = []
read = dict()
read["asset_code"] = "TEST_JQ"
read["read_key"] = str(uuid.uuid4())
read['reading'] = dict()
read['reading']['rate'] = random.randint(1, 100)
ts = str(datetime.now(tz=timezone.utc))
read["user_ts"] = ts
readings.append(read)
payload = dict()
payload['readings'] = readings
cls._readings.append(json.dumps(payload))
payload = PayloadBuilder().AGGREGATE(["max", "id"]).payload()
result = cls._storage_client.query_tbl_with_payload("readings", payload)
return int(result["rows"][0]["max_id"])
async def test_default_filter_configuration(self):
"""Test that filter is not applied when testing with default configuration"""
apply_filter = await self._cfg_manager.get_category_item_value_entry(self._CONFIG_CATEGORY_NAME, 'applyFilter')
jq_rule = await self._cfg_manager.get_category_item_value_entry(self._CONFIG_CATEGORY_NAME, 'filterRule')
if apply_filter.upper() == "TRUE":
transformed_data = self._jqfilter.transform(self._raw_data, jq_rule)
assert transformed_data is None
else:
assert True
async def test_default_filterRule(self):
"""Test that filter is applied and returns readings block unaltered with default configuration of filterRule"""
await self._cfg_manager.set_category_item_value_entry(self._CONFIG_CATEGORY_NAME, 'applyFilter', "True")
apply_filter = await self._cfg_manager.get_category_item_value_entry(self._CONFIG_CATEGORY_NAME, 'applyFilter')
jq_rule = await self._cfg_manager.get_category_item_value_entry(self._CONFIG_CATEGORY_NAME, 'filterRule')
if apply_filter.upper() == "TRUE":
transformed_data = self._jqfilter.transform(self._raw_data, jq_rule)
assert transformed_data == self._raw_data
else:
assert False
async def test_custom_filter_configuration(self):
"""Test with supplied filterRule"""
await self._cfg_manager.set_category_item_value_entry(self._CONFIG_CATEGORY_NAME, 'applyFilter', "True")
await self._cfg_manager.set_category_item_value_entry(self._CONFIG_CATEGORY_NAME,
'filterRule', ".[0]|{Measurement_id: .id}")
apply_filter = await self._cfg_manager.get_category_item_value_entry(self._CONFIG_CATEGORY_NAME, 'applyFilter')
jq_rule = await self._cfg_manager.get_category_item_value_entry(self._CONFIG_CATEGORY_NAME, 'filterRule')
transformed_data = self._jqfilter.transform(self._raw_data, jq_rule)
if apply_filter.upper() == "TRUE":
assert transformed_data == [{"Measurement_id": self._first_read_id}]
else:
assert False
async def test_invalid_filter_configuration(self):
"""Test with invalid filterRule"""
await self._cfg_manager.set_category_item_value_entry(self._CONFIG_CATEGORY_NAME, 'filterRule', "|")
jq_rule = await self._cfg_manager.get_category_item_value_entry(self._CONFIG_CATEGORY_NAME, 'filterRule')
with pytest.raises(ValueError) as ex:
self._jqfilter.transform(self._raw_data, jq_rule)
assert "jq: error: syntax error, unexpected '|'" in str(ex)
def test_init(self):
with patch.object(logger, "setup") as log:
jqfilter_instance = JQFilter()
assert isinstance(jqfilter_instance, JQFilter)
log.assert_called_once_with("JQFilter")