def test_init_with_invalid_storage_service(self):
svc = {"id": 1, "name": "foo", "address": "local", "service_port": 1000, "management_port": 2000,
"type": "xStorage", "protocol": "http"}
with pytest.raises(Exception) as excinfo:
with patch.object(StorageClientAsync, '_get_storage_service', return_value=svc):
sc = StorageClientAsync(1, 2)
assert excinfo.type is InvalidServiceInstance
def __init__(self):
""" All processes must have these three command line arguments passed:
--address [core microservice management host]
--port [core microservice management port]
--name [process name]
"""
self._start_time = time.time()
try:
self._core_management_host = self.get_arg_value("--address")
self._core_management_port = self.get_arg_value("--port")
self._name = self.get_arg_value("--name")
except ArgumentParserError:
raise
if self._core_management_host is None:
raise ValueError("--address is not specified")
elif self._core_management_port is None:
raise ValueError("--port is not specified")
elif self._name is None:
raise ValueError("--name is not specified")
self._core_microservice_management_client = MicroserviceManagementClient(
self._core_management_host, self._core_management_port)
self._readings_storage_async = ReadingsStorageClientAsync(
self._core_management_host, self._core_management_port)
self._storage_async = StorageClientAsync(self._core_management_host,
self._core_management_port)
def test_init(self):
svc = {"id": 1, "name": "foo", "address": "local", "service_port": 1000, "management_port": 2000,
"type": "Storage", "protocol": "http"}
with patch.object(StorageClientAsync, '_get_storage_service', return_value=svc):
sc = StorageClientAsync(1, 2)
assert "local:1000" == sc.base_url
assert "local:2000" == sc.management_api_url
async def test_query_tbl_with_payload(self, event_loop):
# 'PUT', '/storage/table/{tbl_name}/query', query_payload
fake_storage_srvr = FakeFoglampStorageSrvr(loop=event_loop)
await fake_storage_srvr.start()
mockServiceRecord = MagicMock(ServiceRecord)
mockServiceRecord._address = HOST
mockServiceRecord._type = "Storage"
mockServiceRecord._port = PORT
mockServiceRecord._management_port = 2000
sc = StorageClientAsync(1, 2, mockServiceRecord)
assert "{}:{}".format(HOST, PORT) == sc.base_url
with pytest.raises(Exception) as excinfo:
args = None, json.dumps({"k": "v"})
await sc.query_tbl_with_payload(*args)
assert excinfo.type is ValueError
assert "Table name is missing" in str(excinfo.value)
with pytest.raises(Exception) as excinfo:
args = "aTable", None
await sc.query_tbl_with_payload(*args)
assert excinfo.type is ValueError
assert "Query payload is missing" in str(excinfo.value)
with pytest.raises(Exception) as excinfo:
args = "aTable", {"k": "v"}
await sc.query_tbl_with_payload(*args)
assert excinfo.type is TypeError
assert "Query payload must be a valid JSON" in str(excinfo.value)
args = "aTable", json.dumps({"k": "v"})
response = await sc.query_tbl_with_payload(*args)
assert {"k": "v"} == response["called"]
with pytest.raises(Exception) as excinfo:
with patch.object(_LOGGER, "error") as log_e:
with patch.object(_LOGGER, "info") as log_i:
args = "aTable", json.dumps({"bad_request": "v"})
await sc.query_tbl_with_payload(*args)
log_i.assert_called_once_with("PUT %s, with query payload: %s", '/storage/table/aTable/query',
'{"bad_request": "v"}')
log_e.assert_called_once_with("Error code: %d, reason: %s, details: %s", 400, 'bad data', {'key': 'value'})
assert excinfo.type is aiohttp.client_exceptions.ContentTypeError
with pytest.raises(Exception) as excinfo:
with patch.object(_LOGGER, "error") as log_e:
with patch.object(_LOGGER, "info") as log_i:
args = "aTable", json.dumps({"internal_server_err": "v"})
await sc.query_tbl_with_payload(*args)
log_i.assert_called_once_with("PUT %s, with query payload: %s", '/storage/table/aTable/query',
'{"internal_server_err": "v"}')
log_e.assert_called_once_with("Error code: %d, reason: %s, details: %s", 500, 'something wrong', {'key': 'value'})
assert excinfo.type is aiohttp.client_exceptions.ContentTypeError
await fake_storage_srvr.stop()
def test_init_with_service_record(self):
mockServiceRecord = MagicMock(ServiceRecord)
mockServiceRecord._address = "local"
mockServiceRecord._type = "Storage"
mockServiceRecord._port = 1000
mockServiceRecord._management_port = 2000
sc = StorageClientAsync(1, 2, mockServiceRecord)
assert "local:1000" == sc.base_url
assert "local:2000" == sc.management_api_url
def test_init_with_service_record_non_storage_type(self):
mockServiceRecord = MagicMock(ServiceRecord)
mockServiceRecord._address = "local"
mockServiceRecord._type = "xStorage"
mockServiceRecord._port = 1000
mockServiceRecord._management_port = 2000
with pytest.raises(Exception) as excinfo:
with patch.object(_LOGGER, "warning") as log:
sc = StorageClientAsync(1, 2, mockServiceRecord)
log.assert_called_once_with("Storage should be a valid *Storage* micro-service instance")
assert excinfo.type is InvalidServiceInstance
async def _get_storage_client(cls):
storage_service = None
while storage_service is None and cls._storage_client_async is None:
try:
found_services = ServiceRegistry.get(name="FogLAMP Storage")
storage_service = found_services[0]
cls._storage_client_async = StorageClientAsync(
cls._host, cls.core_management_port, svc=storage_service)
except (service_registry_exceptions.DoesNotExist,
InvalidServiceInstance, StorageServiceUnavailable,
Exception) as ex:
await asyncio.sleep(5)
def get_storage_async():
""" Storage Object """
try:
services = ServiceRegistry.get(name="FogLAMP Storage")
storage_svc = services[0]
_storage = StorageClientAsync(core_management_host=None,
core_management_port=None,
svc=storage_svc)
# _logger.info(type(_storage))
except Exception as ex:
_logger.exception(str(ex))
raise
return _storage
async def test_query_tbl(self, event_loop):
# 'GET', '/storage/table/{tbl_name}', *allows query params
fake_storage_srvr = FakeFoglampStorageSrvr(loop=event_loop)
await fake_storage_srvr.start()
mockServiceRecord = MagicMock(ServiceRecord)
mockServiceRecord._address = HOST
mockServiceRecord._type = "Storage"
mockServiceRecord._port = PORT
mockServiceRecord._management_port = 2000
sc = StorageClientAsync(1, 2, mockServiceRecord)
assert "{}:{}".format(HOST, PORT) == sc.base_url
with pytest.raises(Exception) as excinfo:
await sc.query_tbl()
assert excinfo.type is TypeError
assert "query_tbl() missing 1 required positional argument: 'tbl_name'" in str(excinfo.value)
args = "aTable", None # query_tbl without query param is == SELECT *
response = await sc.query_tbl(*args)
assert 1 == response["called"]
args = "aTable", 'foo=v1&bar=v2'
response = await sc.query_tbl(*args)
assert 'foo passed' == response["called"]
with pytest.raises(Exception) as excinfo:
with patch.object(_LOGGER, "error") as log_e:
with patch.object(_LOGGER, "info") as log_i:
args = "aTable", 'bad_foo=1'
await sc.query_tbl(*args)
log_i.assert_called_once_with("GET %s", '/storage/table/aTable?bad_foo=1')
log_e.assert_called_once_with("Error code: %d, reason: %s, details: %s", 400, 'bad data', {'key': 'value'})
assert excinfo.type is aiohttp.client_exceptions.ContentTypeError
with pytest.raises(Exception) as excinfo:
with patch.object(_LOGGER, "error") as log_e:
with patch.object(_LOGGER, "info") as log_i:
args = "aTable", 'internal_server_err_foo=1'
await sc.query_tbl(*args)
log_i.assert_called_once_with("GET %s", '/storage/table/aTable?internal_server_err_foo=1')
log_e.assert_called_once_with("Error code: %d, reason: %s, details: %s", 500, 'something wrong', {'key': 'value'})
assert excinfo.type is aiohttp.client_exceptions.ContentTypeError
await fake_storage_srvr.stop()
async def modify_process_name(service_name, core_management_host,
core_management_port):
storage = StorageClientAsync(core_management_host, core_management_port)
try:
payload = PayloadBuilder().SELECT("id").WHERE(
['schedule_name', '=', service_name]).payload()
result = await storage.query_tbl_with_payload('schedules', payload)
except Exception:
raise
if int(result['count']):
sch_id = result['rows'][0]['id']
else:
_LOGGER.error('No schedule id found for %s. Exiting...', service_name)
raise RuntimeError('No schedule id found for %s. Exiting...',
service_name)
# Modify process name
try:
put_url = "{}://{}:{}/foglamp/schedule/{}".format(
_HTTP, _HOST, _PORT, sch_id)
data = '{"process_name": "south_c"}'
verify_ssl = False if _HTTP == 'http' else True
connector = aiohttp.TCPConnector(verify_ssl=verify_ssl)
async with aiohttp.ClientSession(connector=connector) as session:
async with session.put(put_url, data=data) as resp:
status_code = resp.status
jdoc = await resp.text()
if status_code not in range(200, 209):
_LOGGER.error(
"Error code: %d, reason: %s, details: %s, url: %s",
resp.status, resp.reason, jdoc, put_url)
raise StorageServerError(code=resp.status,
reason=resp.reason,
error=jdoc)
except Exception:
raise
else:
_LOGGER.info(
'Modified process_name from "south" to "south_c" for Python South service [%s]: %s',
service_name, jdoc)
async def run(self):
global _log_performance
global _LOGGER
# Setups signals handlers, to properly handle the termination
# a) SIGTERM - 15 : kill or system shutdown
signal.signal(signal.SIGTERM, SendingProcess._signal_handler)
# Command line parameter handling
self._log_performance, self._debug_level = handling_input_parameters()
_log_performance = self._log_performance
try:
self._storage_async = StorageClientAsync(
self._core_management_host, self._core_management_port)
self._readings = ReadingsStorageClientAsync(
self._core_management_host, self._core_management_port)
self._audit = AuditLogger(self._storage_async)
except Exception as ex:
SendingProcess._logger.exception(_MESSAGES_LIST["e000023"].format(
str(ex)))
sys.exit(1)
else:
SendingProcess._logger.removeHandler(SendingProcess._logger.handle)
logger_name = _MODULE_NAME + "_" + self._name
SendingProcess._logger = logger.setup(
logger_name,
level=logging.INFO
if self._debug_level in [None, 0, 1] else logging.DEBUG)
_LOGGER = SendingProcess._logger
try:
is_started = await self._start()
if is_started:
await self.send_data()
self.stop()
SendingProcess._logger.info("Execution completed.")
sys.exit(0)
except (ValueError, Exception) as ex:
SendingProcess._logger.exception(
_MESSAGES_LIST["e000002"].format(str(ex)))
sys.exit(1)
def __init__(self):
""" All processes must have these three command line arguments passed:
--address [core microservice management host]
--port [core microservice management port]
--name [process name]
"""
self._start_time = time.time()
try:
parser = SilentArgParse()
parser.add_argument("--name", required=True)
parser.add_argument("--address", required=True)
parser.add_argument("--port", required=True, type=int)
namespace, args = parser.parse_known_args()
self._name = getattr(namespace, 'name')
self._core_management_host = getattr(namespace, 'address')
self._core_management_port = getattr(namespace, 'port')
r = range(1, 65536)
if self._core_management_port not in r:
raise ArgumentParserError("Invalid Port: {}".format(self._core_management_port))
for item in args:
if item.startswith('--'):
kv = item.split('=')
if len(kv) == 2:
if len(kv[1].strip()) == 0:
raise ArgumentParserError("Invalid value {} for optional arg {}".format(kv[1], kv[0]))
except ArgumentParserError as ex:
_logger.error("Arg parser error: %s", str(ex))
raise
self._core_microservice_management_client = MicroserviceManagementClient(self._core_management_host,
self._core_management_port)
self._readings_storage_async = ReadingsStorageClientAsync(self._core_management_host,
self._core_management_port)
self._storage_async = StorageClientAsync(self._core_management_host, self._core_management_port)
async def change_to_south_c(service_name, microservice_management_host,
core_management_host, core_management_port,
microservice_id):
global _HTTP, _HOST, _PORT
storage = StorageClientAsync(core_management_host, core_management_port)
configuration_manager = ConfigurationManager(storage)
config = await configuration_manager.get_category_all_items('rest_api')
is_rest_server_http_enabled = False if config['enableHttp'][
'value'] == 'false' else True
port_from_config = config['httpPort']['value'] if is_rest_server_http_enabled \
else config['httpsPort']['value']
_HTTP = 'http' if is_rest_server_http_enabled else 'https'
_HOST = microservice_management_host
_PORT = int(port_from_config)
await disable_service(service_name)
await unregister_service(service_name, core_management_port,
microservice_id)
await modify_process_name(service_name, core_management_host,
core_management_port)
await reenable_service(service_name)
def test_init_with_invalid_service_record(self):
with pytest.raises(Exception) as excinfo:
with patch.object(_LOGGER, "warning") as log:
sc = StorageClientAsync(1, 2, "blah")
log.assert_called_once_with("Storage should be a valid FogLAMP micro-service instance")
assert excinfo.type is InvalidServiceInstance
cls._pid_file_create(full_path, pid)
@classmethod
def set_as_completed(cls, file_name):
""" Sets a job as completed
Args:
file_name: semaphore file either for backup or restore operations
Returns:
Raises:
"""
_logger.debug("{func}".format(func="set_as_completed"))
full_path = JOB_SEM_FILE_PATH + "/" + file_name
if os.path.exists(full_path):
os.remove(full_path)
if __name__ == "__main__":
message = _MESSAGES_LIST["e000003"]
print(message)
if False:
# Used to assign the proper objects type without actually executing them
_storage = StorageClientAsync("127.0.0.1", "0")
_logger = logger.setup(_MODULE_NAME)