async def _retrieve_omf_types_already_created(self, configuration_key, type_id): """ Retrieves the list of OMF types already defined/sent to the PICROMF Args: configuration_key - part of the key to identify the type type_id - part of the key to identify the type Returns: List of Asset code already defined into the PI Server Raises: """ payload = payload_builder.PayloadBuilder() \ .WHERE(['configuration_key', '=', configuration_key]) \ .AND_WHERE(['type_id', '=', type_id]) \ .payload() omf_created_objects = await self._sending_process_instance._storage_async.query_tbl_with_payload( 'omf_created_objects', payload) self._logger.debug("{func} - omf_created_objects {item} ".format( func="_retrieve_omf_types_already_created", item=omf_created_objects)) # Extracts only the asset_code column rows = [] for row in omf_created_objects['rows']: rows.append(row['asset_code']) return rows
async def get_all_backups( self, limit: int, skip: int, status: [lib.BackupStatus, None], sort_order: lib.SortOrder = lib.SortOrder.DESC) -> list: """ Returns a list of backups is returned sorted in chronological order with the most recent backup first. Args: limit: int - limit the number of backups returned to the number given skip: int - skip the number of backups specified before returning backups- this, in conjunction with the limit option, allows for a paged interface to be built status: lib.BackupStatus - limit the returned backups to those only with the specified status, None = retrieves information for all the backups sort_order: lib.SortOrder - Defines the order used to present information, DESC by default Returns: backups_information: all the information available related to the requested backups Raises: """ payload = payload_builder.PayloadBuilder().SELECT("id", "status", "ts", "file_name", "type") \ .ALIAS("return", ("ts", 'ts')).FORMAT("return", ("ts", "YYYY-MM-DD HH24:MI:SS.MS")) if status: payload.WHERE(['status', '=', status]) backups_from_storage = await self._storage.query_tbl_with_payload( self._backup_lib.STORAGE_TABLE_BACKUPS, payload.payload()) backups_information = backups_from_storage['rows'] return backups_information
async def add_statistics(key, description): payload = payload_builder.PayloadBuilder() \ .INSERT(key=key, description=description) \ .payload() await self._storage_async.insert_into_tbl("statistics", payload) rows = await get_rows(key=key) return rows[0]['key']
async def get_rows_from_stream_id(stream_id): payload = payload_builder.PayloadBuilder() \ .SELECT("id", "description", "active") \ .WHERE(['id', '=', stream_id]) \ .payload() streams = await self._storage_async.query_tbl_with_payload("streams", payload) return streams['rows']
def sl_get_backup_details(self, backup_id: int) -> dict: """ Returns the details of a backup Args: backup_id: int - the id of the backup to return Returns: backup_information: all the information available related to the requested backup_id Raises: exceptions.DoesNotExist exceptions.NotUniqueBackup """ payload = payload_builder.PayloadBuilder().SELECT("id", "status", "ts", "file_name", "type")\ .ALIAS("return", ("ts", 'ts')).FORMAT("return", ("ts", "YYYY-MM-DD HH24:MI:SS.MS"))\ .WHERE(['id', '=', backup_id]).payload() backup_from_storage = asyncio.get_event_loop().run_until_complete( self._storage.query_tbl_with_payload(self.STORAGE_TABLE_BACKUPS, payload)) if backup_from_storage['count'] == 0: raise exceptions.DoesNotExist elif backup_from_storage['count'] == 1: backup_information = backup_from_storage['rows'][0] else: raise exceptions.NotUniqueBackup return backup_information
def sl_get_backup_details_from_file_name(self, _file_name): """ Retrieves backup information from file name Args: _file_name: file name to search in the Storage layer Returns: backup_information: Backup information related to the file name Raises: exceptions.DoesNotExist exceptions.NotUniqueBackup """ payload = payload_builder.PayloadBuilder() \ .WHERE(['file_name', '=', _file_name]) \ .payload() backups_from_storage = asyncio.get_event_loop().run_until_complete( self._storage.query_tbl_with_payload(self.STORAGE_TABLE_BACKUPS, payload)) if backups_from_storage['count'] == 1: backup_information = backups_from_storage['rows'][0] elif backups_from_storage['count'] == 0: raise exceptions.DoesNotExist else: raise exceptions.NotUniqueBackup return backup_information
async def get_rows(key): payload = payload_builder.PayloadBuilder() \ .SELECT("key", "description") \ .WHERE(['key', '=', key]) \ .LIMIT(1) \ .payload() statistics = await self._storage_async.query_tbl_with_payload("statistics", payload) return statistics['rows']
async def add_table_statistics_history(self, pyz, file_spec): # The contents of the statistics history from the storage layer temp_file = self._interim_file_path + "/" + "statistics-history-{}".format(file_spec) payload = payload_builder.PayloadBuilder() \ .LIMIT(1000) \ .ORDER_BY(['history_ts', 'DESC']) \ .payload() data = await self._storage.query_tbl_with_payload("statistics_history", payload) self.write_to_tar(pyz, temp_file, data)
async def add_stream(config_stream_id, description): if config_stream_id: payload = payload_builder.PayloadBuilder() \ .INSERT(id=config_stream_id, description=description) \ .payload() await self._storage_async.insert_into_tbl("streams", payload) rows = await get_rows_from_stream_id(stream_id=config_stream_id) else: # If an user is upgrading Fledge, then it has got existing data in streams table but # no entry in configuration for streams_id for this schedule name. Hence it must # check if an entry is already there for this schedule name in streams table. rows = await get_rows_from_name(description=self._name) if len(rows) == 0: payload = payload_builder.PayloadBuilder() \ .INSERT(description=description) \ .payload() await self._storage_async.insert_into_tbl("streams", payload) rows = await get_rows_from_name(description=self._name) return rows[0]['id'], rows[0]['active']
async def _last_object_id_update(self, new_last_object_id): """ Updates reached position""" try: payload = payload_builder.PayloadBuilder() \ .SET(last_object=new_last_object_id, ts='now()') \ .WHERE(['id', '=', self._stream_id]) \ .payload() await self._storage_async.update_tbl("streams", payload) except Exception as _ex: SendingProcess._logger.error(_MESSAGES_LIST["e000020"].format(_ex)) raise
async def add_table_plugin_data(self, pyz, file_spec): # The contents of the plugin_data from the storage layer temp_file = self._interim_file_path + "/" + "plugin-data-{}".format( file_spec) payload = payload_builder.PayloadBuilder() \ .LIMIT(1000) \ .ORDER_BY(['key', 'ASC']) \ .payload() data = await self._storage.query_tbl_with_payload( "plugin_data", payload) self.write_to_tar(pyz, temp_file, data)
async def _delete_backup_information(self, _id): """ Deletes backup information from the Storage layer Args: _id: Backup id to delete Returns: Raises: """ payload = payload_builder.PayloadBuilder() \ .WHERE(['id', '=', _id]) \ .payload() await self._storage.delete_from_tbl( self._backup_lib.STORAGE_TABLE_BACKUPS, payload)
async def deleted_omf_types_already_created(self, config_category_name, type_id): """ Deletes OMF types/objects tracked as already created, it is used to force the recreation of the types Args: config_category_name: used to identify OMF objects already created type_id: used to identify OMF objects already created Returns: Raises: """ payload = payload_builder.PayloadBuilder() \ .WHERE(['configuration_key', '=', config_category_name]) \ .AND_WHERE(['type_id', '=', type_id]) \ .payload() await self._sending_process_instance._storage_async.delete_from_tbl( "omf_created_objects", payload)
async def _load_data_into_memory_statistics(self, last_object_id): """ Extracts statistics data from the DB Layer, converts it into the proper format""" raw_data = None try: payload = payload_builder.PayloadBuilder() \ .SELECT("id", "key", '{"column": "ts", "timezone": "UTC"}', "value", "history_ts") \ .WHERE(['id', '>', last_object_id]) \ .LIMIT(self._config['blockSize']) \ .ORDER_BY(['id', 'ASC']) \ .payload() statistics_history = await self._storage_async.query_tbl_with_payload('statistics_history', payload) raw_data = statistics_history['rows'] converted_data = self._transform_in_memory_data_statistics(raw_data) except Exception: SendingProcess._logger.error(_MESSAGES_LIST["e000009"]) raise return converted_data
async def _flag_created_omf_type(self, configuration_key, type_id, asset_code): """ Stores into the Storage layer the successfully creation of the type into PICROMF. Args: configuration_key - part of the key to identify the type type_id - part of the key to identify the type asset_code - asset code defined into PICROMF Returns: Raises: """ payload = payload_builder.PayloadBuilder()\ .INSERT(configuration_key=configuration_key, asset_code=asset_code, type_id=type_id)\ .payload() await self._sending_process_instance._storage_async.insert_into_tbl( "omf_created_objects", payload)
def sl_backup_status_update(self, _id, _status, _exit_code): """ Updates the status of the backup using the Storage layer Args: _id: Backup's Id to update _status: status of the backup {BackupStatus.SUCCESSFUL|BackupStatus.RESTORED} _exit_code: exit status of the backup/restore execution Returns: Raises: """ _logger.debug("{func} - id |{file}| ".format( func="sl_backup_status_update", file=_id)) payload = payload_builder.PayloadBuilder() \ .SET(status=_status, ts="now()", exit_code=_exit_code) \ .WHERE(['id', '=', _id]) \ .payload() asyncio.get_event_loop().run_until_complete( self._storage.update_tbl(self.STORAGE_TABLE_BACKUPS, payload))
def sl_backup_status_create(self, _file_name, _type, _status): """ Logs the creation of the backup in the Storage layer Args: _file_name: file_name used for the backup as a full path _type: backup type {BackupType.FULL|BackupType.INCREMENTAL} _status: backup status, usually BackupStatus.RUNNING Returns: Raises: """ _logger.debug("{func} - file name |{file}| ".format( func="sl_backup_status_create", file=_file_name)) payload = payload_builder.PayloadBuilder() \ .INSERT(file_name=_file_name, ts="now()", type=_type, status=_status, exit_code=0) \ .payload() asyncio.get_event_loop().run_until_complete( self._storage.insert_into_tbl(self.STORAGE_TABLE_BACKUPS, payload))