def __init__(self):
""" All processes must have these three command line arguments passed:
--address [core microservice management host]
--port [core microservice management port]
--name [process name]
"""
self._start_time = time.time()
try:
self._core_management_host = self.get_arg_value("--address")
self._core_management_port = self.get_arg_value("--port")
self._name = self.get_arg_value("--name")
except ArgumentParserError:
raise
if self._core_management_host is None:
raise ValueError("--address is not specified")
elif self._core_management_port is None:
raise ValueError("--port is not specified")
elif self._name is None:
raise ValueError("--name is not specified")
self._core_microservice_management_client = MicroserviceManagementClient(
self._core_management_host, self._core_management_port)
self._readings_storage_async = ReadingsStorageClientAsync(
self._core_management_host, self._core_management_port)
self._storage_async = StorageClientAsync(self._core_management_host,
self._core_management_port)
def test_init(self):
mockServiceRecord = MagicMock(ServiceRecord)
mockServiceRecord._address = HOST
mockServiceRecord._type = "Storage"
mockServiceRecord._port = PORT
mockServiceRecord._management_port = 2000
rsc = ReadingsStorageClientAsync(1, 2, mockServiceRecord)
assert "{}:{}".format(HOST, PORT) == rsc.base_url
async def test_fetch(self, event_loop):
# GET, '/storage/reading?id={}&count={}'
fake_storage_srvr = FakeFoglampStorageSrvr(loop=event_loop)
await fake_storage_srvr.start()
mockServiceRecord = MagicMock(ServiceRecord)
mockServiceRecord._address = HOST
mockServiceRecord._type = "Storage"
mockServiceRecord._port = PORT
mockServiceRecord._management_port = 2000
rsc = ReadingsStorageClientAsync(1, 2, mockServiceRecord)
assert "{}:{}".format(HOST, PORT) == rsc.base_url
with pytest.raises(Exception) as excinfo:
args = None, 3
await rsc.fetch(*args)
assert excinfo.type is ValueError
assert "first reading id to retrieve the readings block is required" in str(excinfo.value)
with pytest.raises(Exception) as excinfo:
args = 2, None
await rsc.fetch(*args)
assert excinfo.type is ValueError
assert "count is required to retrieve the readings block" in str(excinfo.value)
with pytest.raises(Exception) as excinfo:
args = 2, "1s"
await rsc.fetch(*args)
assert excinfo.type is ValueError
assert "invalid literal for int() with base 10" in str(excinfo.value)
with pytest.raises(Exception) as excinfo:
with patch.object(_LOGGER, "error") as log_e:
args = "bad_data", 3
await rsc.fetch(*args)
log_e.assert_called_once_with('GET url: %s, Error code: %d, reason: %s, details: %s',
'/storage/reading?id=bad_data&count=3', 400, 'bad data', {"key": "value"})
with pytest.raises(Exception) as excinfo:
with patch.object(_LOGGER, "error") as log_e:
args = "internal_server_err", 3
await rsc.fetch(*args)
log_e.assert_called_once_with('GET url: %s, Error code: %d, reason: %s, details: %s',
'/storage/reading?id=internal_server_err&count=3', 500, 'something wrong', {"key": "value"})
assert excinfo.type is aiohttp.client_exceptions.ContentTypeError
args = 2, 3
response = await rsc.fetch(*args)
assert {'readings': [], 'start': '2', 'count': '3'} == response
await fake_storage_srvr.stop()
def get_readings_async():
""" Storage Object """
try:
services = ServiceRegistry.get(name="FogLAMP Storage")
storage_svc = services[0]
_readings = ReadingsStorageClientAsync(core_mgt_host=None,
core_mgt_port=None,
svc=storage_svc)
# _logger.info(type(_storage))
except Exception as ex:
_logger.exception(str(ex))
raise
return _readings
async def test_append(self, event_loop):
# 'POST', '/storage/reading', readings
fake_storage_srvr = FakeFoglampStorageSrvr(loop=event_loop)
await fake_storage_srvr.start()
mockServiceRecord = MagicMock(ServiceRecord)
mockServiceRecord._address = HOST
mockServiceRecord._type = "Storage"
mockServiceRecord._port = PORT
mockServiceRecord._management_port = 2000
rsc = ReadingsStorageClientAsync(1, 2, mockServiceRecord)
assert "{}:{}".format(HOST, PORT) == rsc.base_url
with pytest.raises(Exception) as excinfo:
await rsc.append(None)
assert excinfo.type is ValueError
assert "Readings payload is missing" in str(excinfo.value)
with pytest.raises(Exception) as excinfo:
await rsc.append("blah")
assert excinfo.type is TypeError
assert "Readings payload must be a valid JSON" in str(excinfo.value)
with pytest.raises(Exception) as excinfo:
with patch.object(_LOGGER, "error") as log_e:
readings_bad_payload = json.dumps({"Xreadings": []})
await rsc.append(readings_bad_payload)
log_e.assert_called_once_with("POST url %s with payload: %s, Error code: %d, reason: %s, details: %s",
'/storage/reading', '{"Xreadings": []}', 400, 'bad data', {"key": "value"})
assert excinfo.type is aiohttp.client_exceptions.ContentTypeError
with pytest.raises(Exception) as excinfo:
with patch.object(_LOGGER, "error") as log_e:
r = '{"readings": [], "internal_server_err": 1}'
await rsc.append(r)
log_e.assert_called_once_with("POST url %s with payload: %s, Error code: %d, reason: %s, details: %s",
'/storage/reading', '{"readings": [], "internal_server_err": 1}',
500, 'something wrong', {"key": "value"})
assert excinfo.type is aiohttp.client_exceptions.ContentTypeError
readings = json.dumps({"readings": []})
response = await rsc.append(readings)
assert {'readings': []} == response['appended']
await fake_storage_srvr.stop()
async def test_query(self, event_loop):
# 'PUT', '/storage/reading/query' query_payload
fake_storage_srvr = FakeFoglampStorageSrvr(loop=event_loop)
await fake_storage_srvr.start()
mockServiceRecord = MagicMock(ServiceRecord)
mockServiceRecord._address = HOST
mockServiceRecord._type = "Storage"
mockServiceRecord._port = PORT
mockServiceRecord._management_port = 2000
rsc = ReadingsStorageClientAsync(1, 2, mockServiceRecord)
assert "{}:{}".format(HOST, PORT) == rsc.base_url
with pytest.raises(Exception) as excinfo:
await rsc.query(None)
assert excinfo.type is ValueError
assert "Query payload is missing" in str(excinfo.value)
with pytest.raises(Exception) as excinfo:
await rsc.query("blah")
assert excinfo.type is TypeError
assert "Query payload must be a valid JSON" in str(excinfo.value)
response = await rsc.query(json.dumps({"k": "v"}))
assert {"k": "v"} == response["called"]
with pytest.raises(Exception) as excinfo:
with patch.object(_LOGGER, "error") as log_e:
await rsc.query(json.dumps({"bad_request": "v"}))
log_e.assert_called_once_with("PUT url %s with query payload: %s, Error code: %d, reason: %s, details: %s",
'/storage/reading/query', '{"bad_request": "v"}', 400, 'bad data', {"key": "value"})
assert excinfo.type is aiohttp.client_exceptions.ContentTypeError
with pytest.raises(Exception) as excinfo:
with patch.object(_LOGGER, "error") as log_e:
await rsc.query(json.dumps({"internal_server_err": "v"}))
log_e.assert_called_once_with("PUT url %s with query payload: %s, Error code: %d, reason: %s, details: %s",
'/storage/reading/query', '{"internal_server_err": "v"}', 500, 'something wrong', {"key": "value"})
assert excinfo.type is aiohttp.client_exceptions.ContentTypeError
await fake_storage_srvr.stop()
async def run(self):
global _log_performance
global _LOGGER
# Setups signals handlers, to properly handle the termination
# a) SIGTERM - 15 : kill or system shutdown
signal.signal(signal.SIGTERM, SendingProcess._signal_handler)
# Command line parameter handling
self._log_performance, self._debug_level = handling_input_parameters()
_log_performance = self._log_performance
try:
self._storage_async = StorageClientAsync(
self._core_management_host, self._core_management_port)
self._readings = ReadingsStorageClientAsync(
self._core_management_host, self._core_management_port)
self._audit = AuditLogger(self._storage_async)
except Exception as ex:
SendingProcess._logger.exception(_MESSAGES_LIST["e000023"].format(
str(ex)))
sys.exit(1)
else:
SendingProcess._logger.removeHandler(SendingProcess._logger.handle)
logger_name = _MODULE_NAME + "_" + self._name
SendingProcess._logger = logger.setup(
logger_name,
level=logging.INFO
if self._debug_level in [None, 0, 1] else logging.DEBUG)
_LOGGER = SendingProcess._logger
try:
is_started = await self._start()
if is_started:
await self.send_data()
self.stop()
SendingProcess._logger.info("Execution completed.")
sys.exit(0)
except (ValueError, Exception) as ex:
SendingProcess._logger.exception(
_MESSAGES_LIST["e000002"].format(str(ex)))
sys.exit(1)
def __init__(self):
""" All processes must have these three command line arguments passed:
--address [core microservice management host]
--port [core microservice management port]
--name [process name]
"""
self._start_time = time.time()
try:
parser = SilentArgParse()
parser.add_argument("--name", required=True)
parser.add_argument("--address", required=True)
parser.add_argument("--port", required=True, type=int)
namespace, args = parser.parse_known_args()
self._name = getattr(namespace, 'name')
self._core_management_host = getattr(namespace, 'address')
self._core_management_port = getattr(namespace, 'port')
r = range(1, 65536)
if self._core_management_port not in r:
raise ArgumentParserError("Invalid Port: {}".format(self._core_management_port))
for item in args:
if item.startswith('--'):
kv = item.split('=')
if len(kv) == 2:
if len(kv[1].strip()) == 0:
raise ArgumentParserError("Invalid value {} for optional arg {}".format(kv[1], kv[0]))
except ArgumentParserError as ex:
_logger.error("Arg parser error: %s", str(ex))
raise
self._core_microservice_management_client = MicroserviceManagementClient(self._core_management_host,
self._core_management_port)
self._readings_storage_async = ReadingsStorageClientAsync(self._core_management_host,
self._core_management_port)
self._storage_async = StorageClientAsync(self._core_management_host, self._core_management_port)
async def test_purge(self, event_loop):
# 'PUT', url=put_url, /storage/reading/purge?age=&sent=&flags
fake_storage_srvr = FakeFoglampStorageSrvr(loop=event_loop)
await fake_storage_srvr.start()
mockServiceRecord = MagicMock(ServiceRecord)
mockServiceRecord._address = HOST
mockServiceRecord._type = "Storage"
mockServiceRecord._port = PORT
mockServiceRecord._management_port = 2000
rsc = ReadingsStorageClientAsync(1, 2, mockServiceRecord)
assert "{}:{}".format(HOST, PORT) == rsc.base_url
with pytest.raises(Exception) as excinfo:
kwargs = dict(flag='blah', age=1, sent_id=0, size=None)
await rsc.purge(**kwargs)
assert excinfo.type is InvalidReadingsPurgeFlagParameters
assert "Purge flag valid options are retain or purge only" in str(excinfo.value)
with pytest.raises(Exception) as excinfo:
kwargs = dict(age=1, sent_id=0, size=1, flag='retain')
await rsc.purge(**kwargs)
assert excinfo.type is PurgeOnlyOneOfAgeAndSize
assert "Purge must specify only one of age or size" in str(excinfo.value)
with pytest.raises(Exception) as excinfo:
kwargs = dict(age=None, sent_id=0, size=None, flag='retain')
await rsc.purge(**kwargs)
assert excinfo.type is PurgeOneOfAgeAndSize
assert "Purge must specify one of age or size" in str(excinfo.value)
with pytest.raises(Exception) as excinfo:
kwargs = dict(age=0, sent_id=0, size=0, flag='retain')
await rsc.purge(**kwargs)
assert excinfo.type is PurgeOneOfAgeAndSize
assert "Purge must specify one of age or size" in str(excinfo.value)
# age int
with pytest.raises(Exception) as excinfo:
kwargs = dict(age="1b", sent_id=0, size=None, flag='retain')
await rsc.purge(**kwargs)
assert excinfo.type is ValueError
assert "invalid literal for int() with base 10" in str(excinfo.value)
# size int
with pytest.raises(Exception) as excinfo:
kwargs = dict(age=None, sent_id=0, size="1b", flag='retain')
await rsc.purge(**kwargs)
assert excinfo.type is ValueError
assert "invalid literal for int() with base 10" in str(excinfo.value)
# sent_id int
with pytest.raises(Exception) as excinfo:
kwargs = dict(age=1, sent_id="1b", size=None, flag='retain')
await rsc.purge(**kwargs)
assert excinfo.type is ValueError
assert "invalid literal for int() with base 10" in str(excinfo.value)
with pytest.raises(Exception) as excinfo:
with patch.object(_LOGGER, "error") as log_e:
kwargs = dict(age=-1, sent_id=1, size=None, flag='retain')
await rsc.purge(**kwargs)
log_e.assert_called_once_with('PUT url %s, Error code: %d, reason: %s, details: %s',
'/storage/reading/purge?age=-1&sent=1&flags=retain', 400, 'age should not be less than 0', {"key": "value"})
assert excinfo.type is aiohttp.client_exceptions.ContentTypeError
with pytest.raises(Exception) as excinfo:
with patch.object(_LOGGER, "error") as log_e:
kwargs = dict(age=None, sent_id=1, size=4294967296, flag='retain')
await rsc.purge(**kwargs)
log_e.assert_called_once_with('PUT url %s, Error code: %d, reason: %s, details: %s',
'/storage/reading/purge?size=4294967296&sent=1&flags=retain', 500, 'unsigned int range', {"key": "value"})
assert excinfo.type is aiohttp.client_exceptions.ContentTypeError
kwargs = dict(age=1, sent_id=1, size=0, flag='retain')
response = await rsc.purge(**kwargs)
assert 1 == response["called"]
kwargs = dict(age=0, sent_id=1, size=1, flag='retain')
await rsc.purge(**kwargs)
assert 1 == response["called"]
kwargs = dict(age=1, sent_id=1, size=None, flag='retain')
await rsc.purge(**kwargs)
assert 1 == response["called"]
kwargs = dict(age=None, sent_id=1, size=1, flag='retain')
await rsc.purge(**kwargs)
assert 1 == response["called"]
await fake_storage_srvr.stop()