def get_backup_list(self, limit, skip, status):
""" Retrieves backups information
Args:
limit: maximum number of backups information to retrieve
skip: TBD - # FIXME:
status: BACKUP_STATUS_UNDEFINED= retrieves the information for all the backups state
or for a specific state only
Returns:
Raises:
"""
if status == lib.BACKUP_STATUS_UNDEFINED:
payload = payload_builder.PayloadBuilder() \
.LIMIT(limit) \
.ORDER_BY(['ts', 'ASC']) \
.payload()
else:
payload = payload_builder.PayloadBuilder() \
.WHERE(['state', '=', status]) \
.LIMIT(limit) \
.ORDER_BY(['ts', 'ASC']) \
.payload()
backups_from_storage = self._storage.query_tbl_with_payload(
lib.STORAGE_TABLE_BACKUPS, payload)
backups_information = backups_from_storage['rows']
return backups_information
def get_backup_details(self, _id):
""" Retrieves information for a specific backup
Args:
_id: Backup id to retrieve
Returns:
backup_information: information related to the requested backup
Raises:
exceptions.DoesNotExist
"""
payload = payload_builder.PayloadBuilder() \
.WHERE(['id', '=', _id]) \
.payload()
backup_from_storage = self._storage.query_tbl_with_payload(
lib.STORAGE_TABLE_BACKUPS, payload)
if backup_from_storage['count'] == 0:
raise exceptions.DoesNotExist
elif backup_from_storage['count'] == 1:
backup_information = backup_from_storage['rows'][0]
else:
raise exceptions.NotUniqueBackup
return backup_information
def sl_get_backup_details(self, backup_id: int) -> dict:
""" Returns the details of a backup
Args:
backup_id: int - the id of the backup to return
Returns:
backup_information: all the information available related to the requested backup_id
Raises:
exceptions.DoesNotExist
exceptions.NotUniqueBackup
"""
payload = payload_builder.PayloadBuilder() \
.WHERE(['id', '=', backup_id]) \
.payload()
backup_from_storage = self._storage.query_tbl_with_payload(self.STORAGE_TABLE_BACKUPS, payload)
if backup_from_storage['count'] == 0:
raise exceptions.DoesNotExist
elif backup_from_storage['count'] == 1:
backup_information = backup_from_storage['rows'][0]
else:
raise exceptions.NotUniqueBackup
return backup_information
async def _retrieve_omf_types_already_created(self, configuration_key,
type_id):
""" Retrieves the list of OMF types already defined/sent to the PICROMF
Args:
configuration_key - part of the key to identify the type
type_id - part of the key to identify the type
Returns:
List of Asset code already defined into the PI Server
Raises:
"""
payload = payload_builder.PayloadBuilder() \
.WHERE(['configuration_key', '=', configuration_key]) \
.AND_WHERE(['type_id', '=', type_id]) \
.payload()
omf_created_objects = await self._sending_process_instance._storage_async.query_tbl_with_payload(
'omf_created_objects', payload)
self._logger.debug("{func} - omf_created_objects {item} ".format(
func="_retrieve_omf_types_already_created",
item=omf_created_objects))
# Extracts only the asset_code column
rows = []
for row in omf_created_objects['rows']:
rows.append(row['asset_code'])
return rows
async def add_statistics(key, description):
payload = payload_builder.PayloadBuilder() \
.INSERT(key=key, description=description) \
.payload()
await self._storage_async.insert_into_tbl("statistics", payload)
rows = await get_rows(key=key)
return rows[0]['key']
async def get_all_backups(
self,
limit: int,
skip: int,
status: [lib.BackupStatus, None],
sort_order: lib.SortOrder = lib.SortOrder.DESC) -> list:
""" Returns a list of backups is returned sorted in chronological order with the most recent backup first.
Args:
limit: int - limit the number of backups returned to the number given
skip: int - skip the number of backups specified before returning backups-
this, in conjunction with the limit option, allows for a paged interface to be built
status: lib.BackupStatus - limit the returned backups to those only with the specified status,
None = retrieves information for all the backups
sort_order: lib.SortOrder - Defines the order used to present information, DESC by default
Returns:
backups_information: all the information available related to the requested backups
Raises:
"""
payload = payload_builder.PayloadBuilder().SELECT("id", "status", "ts", "file_name", "type") \
.ALIAS("return", ("ts", 'ts')).FORMAT("return", ("ts", "YYYY-MM-DD HH24:MI:SS.MS"))
if status:
payload.WHERE(['status', '=', status])
backups_from_storage = await self._storage.query_tbl_with_payload(
self._backup_lib.STORAGE_TABLE_BACKUPS, payload.payload())
backups_information = backups_from_storage['rows']
return backups_information
async def get_rows_from_name(description):
payload = payload_builder.PayloadBuilder() \
.SELECT("id", "description", "active") \
.WHERE(['description', '=', description]) \
.payload()
streams = await self._storage_async.query_tbl_with_payload("streams", payload)
return streams['rows']
def sl_get_backup_details(self, backup_id: int) -> dict:
""" Returns the details of a backup
Args:
backup_id: int - the id of the backup to return
Returns:
backup_information: all the information available related to the requested backup_id
Raises:
exceptions.DoesNotExist
exceptions.NotUniqueBackup
"""
payload = payload_builder.PayloadBuilder().SELECT("id", "status", "ts", "file_name", "type")\
.ALIAS("return", ("ts", 'ts')).FORMAT("return", ("ts", "YYYY-MM-DD HH24:MI:SS.MS"))\
.WHERE(['id', '=', backup_id]).payload()
backup_from_storage = self._storage.query_tbl_with_payload(
self.STORAGE_TABLE_BACKUPS, payload)
if backup_from_storage['count'] == 0:
raise exceptions.DoesNotExist
elif backup_from_storage['count'] == 1:
backup_information = backup_from_storage['rows'][0]
else:
raise exceptions.NotUniqueBackup
return backup_information
def sl_get_backup_details_from_file_name(self, _file_name):
""" Retrieves backup information from file name
Args:
_file_name: file name to search in the Storage layer
Returns:
backup_information: Backup information related to the file name
Raises:
exceptions.DoesNotExist
exceptions.NotUniqueBackup
"""
payload = payload_builder.PayloadBuilder() \
.WHERE(['file_name', '=', _file_name]) \
.payload()
backups_from_storage = self._storage.query_tbl_with_payload(
self.STORAGE_TABLE_BACKUPS, payload)
if backups_from_storage['count'] == 1:
backup_information = backups_from_storage['rows'][0]
elif backups_from_storage['count'] == 0:
raise exceptions.DoesNotExist
else:
raise exceptions.NotUniqueBackup
return backup_information
def _load_data_into_memory_statistics(self, last_object_id):
""" Extracts statistics data from the DB Layer, converts it into the proper format
loading into a memory structure
Args:
last_object_id: last row_id already handled
Returns:
converted_data: data extracted from the DB Layer and converted in the proper format
Raises:
Todo:
"""
SendingProcess._logger.debug("{0} - position |{1}| ".format("_load_data_into_memory_statistics", last_object_id))
raw_data = None
try:
payload = payload_builder.PayloadBuilder() \
.SELECT("id", "key", '{"column": "ts", "timezone": "UTC"}', "value", "history_ts")\
.WHERE(['id', '>', last_object_id]) \
.LIMIT(self._config['blockSize']) \
.ORDER_BY(['id', 'ASC']) \
.payload()
statistics_history = self._storage.query_tbl_with_payload('statistics_history', payload)
raw_data = statistics_history['rows']
converted_data = self._transform_in_memory_data_statistics(raw_data)
except Exception:
_message = _MESSAGES_LIST["e000009"]
SendingProcess._logger.error(_message)
raise
return converted_data
def _load_data_into_memory_readings(self, last_object_id):
""" Extracts from the DB Layer data related to the readings loading into a memory structure
Args:
last_object_id: last value already handled
Returns:
raw_data: data extracted from the DB Layer
Raises:
Todo:
"""
SendingProcess._logger.debug("{0} - position {1} ".format(
"_load_data_into_memory_readings", last_object_id))
raw_data = None
try:
# Loads data
payload = payload_builder.PayloadBuilder() \
.WHERE(['id', '>', last_object_id]) \
.LIMIT(self._config['blockSize']) \
.ORDER_BY(['id', 'ASC']) \
.payload()
readings = self._readings.query(payload)
raw_data = readings['rows']
except Exception as _ex:
_message = _MESSAGES_LIST["e000009"].format(str(_ex))
SendingProcess._logger.error(_message)
raise
return raw_data
async def get_rows(key):
payload = payload_builder.PayloadBuilder() \
.SELECT("key", "description") \
.WHERE(['key', '=', key]) \
.LIMIT(1) \
.payload()
statistics = await self._storage_async.query_tbl_with_payload("statistics", payload)
return statistics['rows']
def get_all_backups(
self,
limit: int,
skip: int,
status: [lib.BackupStatus, None],
sort_order: lib.SortOrder = lib.SortOrder.DESC) -> list:
""" Returns a list of backups is returned sorted in chronological order with the most recent backup first.
Args:
limit: int - limit the number of backups returned to the number given
skip: int - skip the number of backups specified before returning backups-
this, in conjunction with the limit option, allows for a paged interface to be built
status: lib.BackupStatus - limit the returned backups to those only with the specified status,
None = retrieves information for all the backups
sort_order: lib.SortOrder - Defines the order used to present information, DESC by default
Returns:
backups_information: all the information available related to the requested backups
Raises:
"""
if status is None:
payload = payload_builder.PayloadBuilder() \
.LIMIT(limit) \
.SKIP(skip) \
.ORDER_BY(['ts', sort_order]) \
.payload()
else:
payload = payload_builder.PayloadBuilder() \
.WHERE(['status', '=', status]) \
.LIMIT(limit) \
.SKIP(skip) \
.ORDER_BY(['ts', sort_order]) \
.payload()
backups_from_storage = self._storage.query_tbl_with_payload(
self._backup_lib.STORAGE_TABLE_BACKUPS, payload)
backups_information = backups_from_storage['rows']
return backups_information
async def _last_object_id_update(self, new_last_object_id):
""" Updates reached position"""
try:
payload = payload_builder.PayloadBuilder() \
.SET(last_object=new_last_object_id, ts='now()') \
.WHERE(['id', '=', self._stream_id]) \
.payload()
await self._storage_async.update_tbl("streams", payload)
except Exception as _ex:
SendingProcess._logger.error(_MESSAGES_LIST["e000020"].format(_ex))
raise
async def add_stream(config_stream_id, description):
if config_stream_id:
payload = payload_builder.PayloadBuilder() \
.INSERT(id=config_stream_id,
description=description) \
.payload()
await self._storage_async.insert_into_tbl("streams", payload)
rows = await get_rows_from_stream_id(stream_id=config_stream_id)
else:
# If an user is upgrading FogLamp, then it has got existing data in streams table but
# no entry in configuration for streams_id for this schedule name. Hence it must
# check if an entry is already there for this schedule name in streams table.
rows = await get_rows_from_name(description=self._name)
if len(rows) == 0:
payload = payload_builder.PayloadBuilder() \
.INSERT(description=description) \
.payload()
await self._storage_async.insert_into_tbl("streams", payload)
rows = await get_rows_from_name(description=self._name)
return rows[0]['id'], rows[0]['active']
async def _delete_backup_information(self, _id):
""" Deletes backup information from the Storage layer
Args:
_id: Backup id to delete
Returns:
Raises:
"""
payload = payload_builder.PayloadBuilder() \
.WHERE(['id', '=', _id]) \
.payload()
await self._storage.delete_from_tbl(
self._backup_lib.STORAGE_TABLE_BACKUPS, payload)
def _deleted_omf_types_already_created(self, config_category_name, type_id):
""" Deletes OMF types/objects tracked as already created, it is used to force the recreation of the types
Args:
config_category_name: used to identify OMF objects already created
type_id: used to identify OMF objects already created
Returns:
Raises:
Todo:
"""
payload = payload_builder.PayloadBuilder() \
.WHERE(['configuration_key', '=', config_category_name]) \
.AND_WHERE(['type_id', '=', type_id]) \
.payload()
self._sending_process_instance._storage.delete_from_tbl("omf_created_objects", payload)
def _flag_created_omf_type(self, configuration_key, type_id, asset_code):
""" Stores into the Storage layer the successfully creation of the type into PICROMF.
Args:
configuration_key - part of the key to identify the type
type_id - part of the key to identify the type
asset_code - asset code defined into PICROMF
Returns:
Raises:
Todo:
"""
payload = payload_builder.PayloadBuilder()\
.INSERT(configuration_key=configuration_key,
asset_code=asset_code,
type_id=type_id)\
.payload()
self._sending_process_instance._storage.insert_into_tbl("omf_created_objects", payload)
async def _load_data_into_memory_statistics(self, last_object_id):
""" Extracts statistics data from the DB Layer, converts it into the proper format"""
raw_data = None
try:
payload = payload_builder.PayloadBuilder() \
.SELECT("id", "key", '{"column": "ts", "timezone": "UTC"}', "value", "history_ts") \
.WHERE(['id', '>', last_object_id]) \
.LIMIT(self._config['blockSize']) \
.ORDER_BY(['id', 'ASC']) \
.payload()
statistics_history = await self._storage_async.query_tbl_with_payload('statistics_history', payload)
raw_data = statistics_history['rows']
converted_data = self._transform_in_memory_data_statistics(raw_data)
except Exception:
SendingProcess._logger.error(_MESSAGES_LIST["e000009"])
raise
return converted_data
def write(self, level, log):
""" Logs an operation in the Storage layer
Args:
level: {SUCCESS|FAILURE|WARNING|INFO}
log: message to log as a dict
Returns:
Raises:
Logs in the syslog in case of an error but the exception is not propagated
"""
try:
payload = payload_builder.PayloadBuilder() \
.INSERT(code=LogStorage.LOG_CODE,
level=level,
log=log) \
.payload()
self._storage.insert_into_tbl("log", payload)
except Exception as _ex:
_message = _MESSAGES_LIST["e000024"].format(_ex)
_LOGGER.error(_message)
def _last_object_id_update(self, new_last_object_id, stream_id):
""" Updates reached position
Args:
new_last_object_id: Last row id already sent
stream_id: Managed stream id
Todo:
it should evolve using the DB layer
"""
try:
SendingProcess._logger.debug("Last position, sent |{0}| ".format(str(new_last_object_id)))
# TODO : FOGL-623 - avoid the update of the field ts when it will be managed by the DB itself
#
payload = payload_builder.PayloadBuilder() \
.SET(last_object=new_last_object_id, ts='now()') \
.WHERE(['id', '=', stream_id]) \
.payload()
self._storage.update_tbl("streams", payload)
except Exception as _ex:
_message = _MESSAGES_LIST["e000020"].format(_ex)
SendingProcess._logger.error(_message)
raise
def sl_backup_status_update(self, _id, _status, _exit_code):
""" Updates the status of the backup using the Storage layer
Args:
_id: Backup's Id to update
_status: status of the backup {BackupStatus.SUCCESSFUL|BackupStatus.RESTORED}
_exit_code: exit status of the backup/restore execution
Returns:
Raises:
"""
_logger.debug("{func} - id |{file}| ".format(func="sl_backup_status_update", file=_id))
payload = payload_builder.PayloadBuilder() \
.SET(status=_status,
ts="now()",
exit_code=_exit_code) \
.WHERE(['id', '=', _id]) \
.payload()
self._storage.update_tbl(self.STORAGE_TABLE_BACKUPS, payload)
def _delete_backup_information(self, _id, _file_name):
""" Deletes backup information from the Storage layer
Args:
_id: Backup id to delete
_file_name: file name to delete
Returns:
Raises:
"""
try:
payload = payload_builder.PayloadBuilder() \
.WHERE(['id', '=', _id]) \
.payload()
self._storage.delete_from_tbl(lib.STORAGE_TABLE_BACKUPS, payload)
except Exception as _ex:
_message = self._MESSAGES_LIST["e000005"].format(
_id, _file_name, _ex)
self._logger.warning(_message)
def sl_backup_status_create(self, _file_name, _type, _status):
""" Logs the creation of the backup in the Storage layer
Args:
_file_name: file_name used for the backup as a full path
_type: backup type {BackupType.FULL|BackupType.INCREMENTAL}
_status: backup status, usually BackupStatus.RUNNING
Returns:
Raises:
"""
_logger.debug("{func} - file name |{file}| ".format(func="sl_backup_status_create", file=_file_name))
payload = payload_builder.PayloadBuilder() \
.INSERT(file_name=_file_name,
ts="now()",
type=_type,
status=_status,
exit_code=0) \
.payload()
self._storage.insert_into_tbl(self.STORAGE_TABLE_BACKUPS, payload)
def _start_core(cls, loop=None):
_logger.info("start core")
try:
host = cls._host
cls.core_app = cls._make_core_app()
cls.core_server, cls.core_server_handler = cls._start_app(
loop, cls.core_app, host, 0)
address, cls.core_management_port = cls.core_server.sockets[
0].getsockname()
_logger.info('Management API started on http://%s:%s', address,
cls.core_management_port)
# see http://<core_mgt_host>:<core_mgt_port>/foglamp/service for registered services
# start storage
loop.run_until_complete(cls._start_storage(loop))
# get storage client
loop.run_until_complete(cls._get_storage_client())
# If readings table is empty, set last_object of all streams to 0
total_count_payload = payload_builder.PayloadBuilder().AGGREGATE(
["count", "*"]).ALIAS("aggregate",
("*", "count", "count")).payload()
result = loop.run_until_complete(
cls._storage_client_async.query_tbl_with_payload(
'readings', total_count_payload))
total_count = result['rows'][0]['count']
if (total_count == 0):
_logger.info(
"'foglamp.readings' table is empty, force reset of 'foglamp.streams' last_objects"
)
payload = payload_builder.PayloadBuilder().SET(
last_object=0, ts='now()').payload()
loop.run_until_complete(
cls._storage_client_async.update_tbl("streams", payload))
else:
_logger.info(
"'foglamp.readings' has " + str(total_count) +
" rows, 'foglamp.streams' last_objects reset is not required"
)
# obtain configuration manager and interest registry
cls._configuration_manager = ConfigurationManager(
cls._storage_client_async)
cls._interest_registry = InterestRegistry(
cls._configuration_manager)
# start scheduler
# see scheduler.py start def FIXME
# scheduler on start will wait for storage service registration
loop.run_until_complete(cls._start_scheduler())
# start monitor
loop.run_until_complete(cls._start_service_monitor())
loop.run_until_complete(cls.rest_api_config())
cls.service_app = cls._make_app(auth_required=cls.is_auth_required)
# ssl context
ssl_ctx = None
if not cls.is_rest_server_http_enabled:
# ensure TLS 1.2 and SHA-256
# handle expiry?
ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
cert, key = cls.get_certificates()
_logger.info('Loading certificates %s and key %s', cert, key)
ssl_ctx.load_cert_chain(cert, key)
# Get the service data and advertise the management port of the core
# to allow other microservices to find FogLAMP
loop.run_until_complete(cls.service_config())
_logger.info('Announce management API service')
cls.management_announcer = ServiceAnnouncer(
'core.{}'.format(cls._service_name), cls._MANAGEMENT_SERVICE,
cls.core_management_port, ['The FogLAMP Core REST API'])
cls.service_server, cls.service_server_handler = cls._start_app(
loop,
cls.service_app,
host,
cls.rest_server_port,
ssl_ctx=ssl_ctx)
address, service_server_port = cls.service_server.sockets[
0].getsockname()
# Write PID file with REST API details
cls._write_pid(address, service_server_port)
_logger.info(
'REST API Server started on %s://%s:%s',
'http' if cls.is_rest_server_http_enabled else 'https',
address, service_server_port)
# All services are up so now we can advertise the Admin and User REST API's
cls.admin_announcer = ServiceAnnouncer(cls._service_name,
cls._ADMIN_API_SERVICE,
service_server_port,
[cls._service_description])
cls.user_announcer = ServiceAnnouncer(cls._service_name,
cls._USER_API_SERVICE,
service_server_port,
[cls._service_description])
# register core
# a service with 2 web server instance,
# registering now only when service_port is ready to listen the request
# TODO: if ssl then register with protocol https
cls._register_core(host, cls.core_management_port,
service_server_port)
# Everything is complete in the startup sequence, write the audit log entry
cls._audit = AuditLogger(cls._storage_client_async)
loop.run_until_complete(cls._audit.information('START', None))
loop.run_forever()
except (OSError, RuntimeError, TimeoutError) as e:
sys.stderr.write('Error: ' + format(str(e)) + "\n")
sys.exit(1)
except Exception as e:
sys.stderr.write('Error: ' + format(str(e)) + "\n")
sys.exit(1)
