def plugin_init(data): """ Initializes the OMF plugin for the sending of blocks of readings to the PI Connector. Args: Returns: Raises: PluginInitializeFailed Todo: """ global _config global _config_omf_types global _config_omf_types_from_manager global _logger global _recreate_omf_objects try: # note : _module_name is used as __name__ refers to the Sending Process logger_name = _MODULE_NAME + "_" + str(data['stream_id']['value']) _logger = logger.setup(logger_name) if _log_debug_level == 0 else \ logger.setup(logger_name, level=logging.INFO if _log_debug_level == 1 else logging.DEBUG) except Exception as ex: _logger.error("{0} - ERROR - {1}".format(time.strftime("%Y-%m-%d %H:%M:%S:"), plugin_common.MESSAGES_LIST["e000012"].format(str(ex)))) raise ex _logger.debug("{0} - ".format("plugin_info")) # Retrieves the configurations and apply the related conversions _config['_CONFIG_CATEGORY_NAME'] = data['_CONFIG_CATEGORY_NAME'] _config['URL'] = data['URL']['value'] _config['producerToken'] = data['producerToken']['value'] _config['OMFMaxRetry'] = int(data['OMFMaxRetry']['value']) _config['OMFRetrySleepTime'] = int(data['OMFRetrySleepTime']['value']) _config['OMFHttpTimeout'] = int(data['OMFHttpTimeout']['value']) _config['StaticData'] = ast.literal_eval(data['StaticData']['value']) # TODO: compare instance fetching via inspect vs as param passing # import inspect # _config['sending_process_instance'] = inspect.currentframe().f_back.f_locals['self'] _config['sending_process_instance'] = data['sending_process_instance'] # _config_omf_types = json.loads(data['omf_types']['value']) _config_omf_types = _config['sending_process_instance']._fetch_configuration(cat_name=_CONFIG_CATEGORY_OMF_TYPES_NAME, cat_desc=_CONFIG_CATEGORY_OMF_TYPES_DESCRIPTION, cat_config=_CONFIG_DEFAULT_OMF_TYPES, cat_keep_original=True) # Converts the value field from str to a dict for item in _config_omf_types: if _config_omf_types[item]['type'] == 'JSON': # The conversion from a dict to str changes the case and it should be fixed before the conversion value = _config_omf_types[item]['value'].replace("true", "True") new_value = ast.literal_eval(value) _config_omf_types[item]['value'] = new_value _logger.debug("{0} - URL {1}".format("plugin_init", _config['URL'])) try: _recreate_omf_objects = True except Exception as ex: _logger.error(plugin_common.MESSAGES_LIST["e000011"].format(ex)) raise plugin_exceptions.PluginInitializeFailed(ex) return _config
def test_logger_destination_error(self): """ Test Error gets returned when destination isn't 0 or 1 :assert: Assert ValueError is returned when destination=2 """ with pytest.raises(ValueError) as error_exec: logger.setup(__name__, destination=2) assert "ValueError: Invalid destination 2" in str(error_exec)
def __init__(self, _storage): self._storage = _storage if not Restore._logger: Restore._logger = logger.setup(self._MODULE_NAME, destination=_LOGGER_DESTINATION, level=_LOGGER_LEVEL)
def __init__(self): super().__init__() if not self._logger: self._logger = logger.setup(self._MODULE_NAME, destination=_LOGGER_DESTINATION, level=_LOGGER_LEVEL) # Handled Restore command line parameters try: self._backup_id = super().get_arg_value("--backup-id") self._file_name = super().get_arg_value("--file") except Exception as _ex: _message = _MESSAGES_LIST["e000003"].format(_ex) _logger.exception(_message) raise exceptions.ArgumentParserError(_message) self._restore_lib = lib.BackupRestoreLib(self._storage, self._logger) self._job = lib.Job() self._force_restore = True """ Restore a backup doesn't exist in the backups table """ # Creates the objects references used by the library lib._logger = self._logger lib._storage = self._storage
def test_logger_type(self): """ Test the logger type being returned at setup :assert: Assert that setup returns value of type logger.Logger """ assert isinstance(logger.setup(__name__), logging.Logger) is True
async def get_backups(request): """ Returns a list of all backups :Example: curl -X GET http://localhost:8082/foglamp/backup :Example: curl -X GET http://localhost:8082/foglamp/backup?limit=2&skip=1&status=complete """ try: limit = int( request.query['limit']) if 'limit' in request.query else None skip = int(request.query['skip']) if 'skip' in request.query else None status = request.query['status'] if 'status' in request.query else None # TODO : Fix after actual implementation Backup.get_backup_list.return_value = [{ 'id': 28, 'date': '2017-08-30 04:05:10.382', 'status': 'running' }, { 'id': 27, 'date': '2017-08-29 04:05:13.392', 'status': 'failed' }, { 'id': 26, 'date': '2017-08-28 04:05:08.201', 'status': 'complete' }] # backup_json = [{"id": b[0], "date": b[1], "status": b[2]} # for b in Backup.get_backup_list(limit=limit, skip=skip, status=status)] backup_json = Backup.get_backup_list(limit=limit, skip=skip, status=status) # ## Test #########################################################################################: _logger = logger.setup("BACKUP-API-TEST", destination=logger.SYSLOG, level=logging.DEBUG) _logger.info("get_backups - START 3 ") _storage = connect.get_storage() backup = backup_postgres.Backup(_storage) backup_json = backup.get_all_backups(999, 0, None) _logger.debug("get_backups - END ") _logger.handlers = [] _logger.removeHandler(_logger.handle) _logger = None del backup # ## ##########################################################+###############################: except Backup.DoesNotExist: raise web.HTTPNotFound( reason='No backups found for queried parameters') return web.json_response({"backups": backup_json})
def test_destination_consule(self): """ Test the logger type being returned when destination=1 :assert: Assert that the setup returns value of type logging.Logger """ assert isinstance(logger.setup(__name__, destination=1), logging.Logger) is True
def test_logger_handler(self): """ Test that handler has been added :assert: Assert hasHandler returns True """ output = logger.setup(__name__) assert output.hasHandlers() is True
def __init__(self, _storage): self._storage = _storage if not Backup._logger: Backup._logger = logger.setup(self._MODULE_NAME, destination=_LOGGER_DESTINATION, level=_LOGGER_LEVEL) self._backup_lib = lib.BackupRestoreLib(self._storage, self._logger)
def __init__(self): self._logger = logger.setup("SMNTR", level=logging.INFO) self._monitor_loop_task = None # type: asyncio.Task """Task for :meth:`_monitor_loop`, to ensure it has finished""" self._sleep_interval = None # type: int """The time (in seconds) to sleep between health checks""" self._ping_timeout = None # type: int """Timeout for a response from any given micro-service"""
def start(self): # Command line parameter handling global _log_performance global _LOGGER # Setups signals handlers, to properly handle the termination # a) SIGTERM - 15 : kill or system shutdown signal.signal(signal.SIGTERM, SendingProcess._signal_handler) try: self._mgt_name, self._mgt_port, self._mgt_address, self.input_stream_id, self._log_performance, self._log_debug_level = \ handling_input_parameters() _log_performance = self._log_performance except Exception as ex: message = _MESSAGES_LIST["e000017"].format(str(ex)) SendingProcess._logger.exception(message) sys.exit(1) try: self._storage = StorageClient(self._mgt_address, self._mgt_port) self._readings = ReadingsStorageClient(self._mgt_address, self._mgt_port) self._audit = AuditLogger(self._storage) except Exception as ex: message = _MESSAGES_LIST["e000023"].format(str(ex)) SendingProcess._logger.exception(message) sys.exit(1) else: # Reconfigures the logger using the Stream ID to differentiates # logging from different processes SendingProcess._logger.removeHandler(SendingProcess._logger.handle) logger_name = _MODULE_NAME + "_" + str(self.input_stream_id) SendingProcess._logger = logger.setup(logger_name, destination=_LOGGER_DESTINATION, level=_LOGGER_LEVEL) try: # Set the debug level if self._log_debug_level == 1: SendingProcess._logger.setLevel(logging.INFO) elif self._log_debug_level >= 2: SendingProcess._logger.setLevel(logging.DEBUG) # Sets the reconfigured logger _LOGGER = SendingProcess._logger # Start sending if self._start(self.input_stream_id): self.send_data(self.input_stream_id) # Stop Sending self.stop() SendingProcess._logger.info(_MESSAGES_LIST["i000002"]) sys.exit(0) except Exception as ex: message = _MESSAGES_LIST["e000002"].format(str(ex)) SendingProcess._logger.exception(message) sys.exit(1)
def __init__(self): self._logger = logger.setup(__name__, level=20) self._monitor_loop_task = None # type: asyncio.Task """Task for :meth:`_monitor_loop`, to ensure it has finished""" self._sleep_interval = None # type: int """The time (in seconds) to sleep between health checks""" self._ping_timeout = None # type: int """Timeout for a response from any given micro-service""" self._max_attempts = None # type: int """Number of max attempts for finding a heartbeat of service"""
def test_logger_level(self): """ Test logger level gets updated :assert: Assert that unless i==0, output.getEffectiveLevel() == i """ for i in range(0, 70, 10): output = logger.setup(__name__, level=i) if i == 0: assert output.getEffectiveLevel() == 30 else: assert output.getEffectiveLevel() == i
def test_logger_level(self): """ Test logger level gets updated :assert: Assert that unless i==0, output.getEffectiveLevel() == i """ for i in range(0, 60, 10): output = logger.setup(__name__, level=i) if i == 0: # Level NOTSET (0) so inherits level WARNING (30) assert logging.WARNING == output.getEffectiveLevel() else: assert i == output.getEffectiveLevel()
def test_logger_instance(self): """ Test the logger type being returned at setup :assert: Assert that setup returns instance of type logger.Logger Assert instance name Assert instance hasHandler Assert instance default log level WARNING """ instance = logger.setup(__name__) assert isinstance(instance, logging.Logger) assert "test_logger" == instance.name assert instance.hasHandlers() assert logging.WARNING == instance.getEffectiveLevel()
def __init__(self, core_management_host=None, core_management_port=None): """Constructor""" cls = Scheduler # Initialize class attributes if not cls._logger: cls._logger = logger.setup(__name__, level=20) # cls._logger = logger.setup(__name__, destination=logger.CONSOLE, level=logging.DEBUG) # cls._logger = logger.setup(__name__, level=logging.DEBUG) if not cls._core_management_port: cls._core_management_port = core_management_port if not cls._core_management_host: cls._core_management_host = core_management_host # Instance attributes self._storage = None self._ready = False """True when the scheduler is ready to accept API calls""" self._start_time = None # type: int """When the scheduler started""" self._max_running_tasks = None # type: int """Maximum number of tasks that can execute at any given time""" self._paused = False """When True, the scheduler will not start any new tasks""" self._process_scripts = dict() """Dictionary of scheduled_processes.name to script""" self._schedules = dict() """Dictionary of schedules.id to _ScheduleRow""" self._schedule_executions = dict() """Dictionary of schedules.id to _ScheduleExecution""" self._task_processes = dict() """Dictionary of tasks.id to _TaskProcess""" self._check_processes_pending = False """bool: True when request to run check_processes""" self._scheduler_loop_task = None # type: asyncio.Task """Task for :meth:`_scheduler_loop`, to ensure it has finished""" self._scheduler_loop_sleep_task = None # type: asyncio.Task """Task for asyncio.sleep used by :meth:`_scheduler_loop`""" self.current_time = None # type: int """Time to use when determining when to start tasks, for testing""" self._last_task_purge_time = None # type: int """When the tasks table was last purged""" self._max_completed_task_age = None # type: datetime.timedelta """Delete finished task rows when they become this old""" self._purge_tasks_task = None # type: asyncio.Task """asynico task for :meth:`purge_tasks`, if scheduled to run"""
def __init__(self): """Constructor""" cls = Monitor # Initialize class attributes if not cls._logger: cls._logger = logger.setup(__name__) self._monitor_loop_task = None # type: asyncio.Task """Task for :meth:`_monitor_loop`, to ensure it has finished""" self._sleep_interval = None # type: int """The time (in seconds) to sleep between health checks""" self._ping_timeout = None # type: int """Timeout for a response from any given microservice"""
def test_compare_setup(self): """ Test that logger.setup() generates the same value as logging for level - 10 to 50 propagate: True or False :assert: Assert logging.getLogger() and logger.setup return the same value(s) """ for name in (__name__, 'aaa'): log = logging.getLogger(name) for level in range(10, 60, 10): for propagate in (True, False): log.setLevel(level) log.propagate = propagate assert log is logger.setup( name, propagate=propagate, level=level)
def __init__(self): super().__init__() if not self._logger: self._logger = logger.setup(self._MODULE_NAME, destination=_LOGGER_DESTINATION, level=_LOGGER_LEVEL) self._backup = Backup(self._storage) self._backup_lib = lib.BackupRestoreLib(self._storage, self._logger) self._job = lib.Job() # Creates the objects references used by the library lib._logger = self._logger lib._storage = self._storage
async def run(self): global _log_performance global _LOGGER # Setups signals handlers, to properly handle the termination # a) SIGTERM - 15 : kill or system shutdown signal.signal(signal.SIGTERM, SendingProcess._signal_handler) # Command line parameter handling self._log_performance, self._debug_level = handling_input_parameters() _log_performance = self._log_performance try: self._storage_async = StorageClientAsync( self._core_management_host, self._core_management_port) self._readings = ReadingsStorageClientAsync( self._core_management_host, self._core_management_port) self._audit = AuditLogger(self._storage_async) except Exception as ex: SendingProcess._logger.exception(_MESSAGES_LIST["e000023"].format( str(ex))) sys.exit(1) else: SendingProcess._logger.removeHandler(SendingProcess._logger.handle) logger_name = _MODULE_NAME + "_" + self._name SendingProcess._logger = logger.setup( logger_name, level=logging.INFO if self._debug_level in [None, 0, 1] else logging.DEBUG) _LOGGER = SendingProcess._logger try: is_started = await self._start() if is_started: await self.send_data() self.stop() SendingProcess._logger.info("Execution completed.") sys.exit(0) except (ValueError, Exception) as ex: SendingProcess._logger.exception( _MESSAGES_LIST["e000002"].format(str(ex))) sys.exit(1)
def start(self): # Command line parameter handling try: self._mgt_name, self._mgt_port, self._mgt_address, self.input_stream_id, self._log_performance, self._log_debug_level = \ handling_input_parameters() except Exception as ex: message = _MESSAGES_LIST["e000017"].format(str(ex)) SendingProcess._logger.exception(message) sys.exit(1) try: self._storage = StorageClient(self._mgt_address, self._mgt_port) self._readings = ReadingsStorageClient(self._mgt_address, self._mgt_port) self._log_storage = LogStorage(self._storage) except Exception as ex: message = _MESSAGES_LIST["e000023"].format(str(ex)) SendingProcess._logger.exception(message) sys.exit(1) else: # Reconfigures the logger using the Stream ID to differentiates # logging from different processes SendingProcess._logger.removeHandler(SendingProcess._logger.handle) logger_name = _MODULE_NAME + "_" + str(self.input_stream_id) SendingProcess._logger = logger.setup(logger_name) try: # Set the debug level if self._log_debug_level == 1: SendingProcess._logger.setLevel(logging.INFO) elif self._log_debug_level >= 2: SendingProcess._logger.setLevel(logging.DEBUG) # Start sending if self._start(self.input_stream_id): self.send_data(self.input_stream_id) # Stop Sending self.stop() SendingProcess._logger.info(_MESSAGES_LIST["i000002"]) sys.exit(0) except Exception as ex: message = _MESSAGES_LIST["e000002"].format(str(ex)) SendingProcess._logger.exception(message) sys.exit(1)
async def create_backup(request): """ Creates a backup :Example: curl -X POST http://localhost:8082/foglamp/backup """ # TODO : Fix after actual implementation Backup.create_backup.return_value = "running" status = Backup.create_backup() # ## Test #########################################################################################: _logger = logger.setup("BACKUP-API-TEST", destination=logger.SYSLOG, level=logging.DEBUG) _logger.info( "=== START ===========================================================================================" ) _storage = connect.get_storage() backup = backup_postgres.Backup(_storage) try: await backup.create_backup() status = "running" except Exception as _ex: status = "failed" _logger.info( "==== END ==========================================================================================" ) _logger.handlers = [] ### #########################################################################################: return web.json_response({"status": status})
def __init__(self): super().__init__() try: # FIXME: # self._logger = logger.setup(self._MODULE_NAME) self._logger = logger.setup( self._MODULE_NAME, # destination=logger.CONSOLE, level=logging.DEBUG) except Exception as _ex: _message = self._MESSAGES_LIST["e000001"].format(str(_ex)) _current_time = time.strftime("%Y-%m-%d %H:%M:%S") print("[FOGLAMP] {0} - ERROR - {1}".format(_current_time, _message), file=sys.stderr) sys.exit(1) self._config_from_manager = {} self._config = {} self._job = lib.Job() self._event_loop = asyncio.get_event_loop() self._foglamp_backup = "" self._foglamp_data = "" self._foglamp_data_etc = "" self._foglamp_root = "" self._backups_dir = "" self._semaphores_dir = "" # Creates the objects references used by the library lib._logger = self._logger lib._storage = self._storage
}, 'noOfAssets': { 'description': 'No. of assets to generate', 'type': 'integer', 'default': '2500', 'order': '2' }, 'dataPointsPerSec': { 'description': 'Data points per second', 'type': 'integer', 'default': '5000', 'order': '3' } } _LOGGER = logger.setup(__name__, level=logging.INFO) _task = None no_of_assets = 0 asset_srl = 0 cn = 0 cn_time = None def plugin_info(): """ Returns information about the plugin. Args: Returns: dict: plugin information Raises: """ return {
import datetime import time import uuid from typing import List, Union import json from foglamp.common import logger from foglamp.common.statistics import Statistics from foglamp.common.storage_client.storage_client import ReadingsStorageClient, StorageClient from foglamp.common.storage_client.exceptions import StorageServerError __author__ = "Terris Linenbach" __copyright__ = "Copyright (c) 2017 OSIsoft, LLC" __license__ = "Apache 2.0" __version__ = "${VERSION}" _LOGGER = logger.setup(__name__) # type: logging.Logger _MAX_ATTEMPTS = 2 # _LOGGER = logger.setup(__name__, level=logging.DEBUG) # type: logging.Logger # _LOGGER = logger.setup(__name__, destination=logger.CONSOLE, level=logging.DEBUG) class Ingest(object): """Adds sensor readings to FogLAMP Also tracks readings-related statistics. Readings are added to a configurable list. Configurable batches of inserts are sent to storage """ # Class attributes
from importlib import import_module import copy import json from collections import OrderedDict from foglamp.common.storage_client.payload_builder import PayloadBuilder from foglamp.common.storage_client.storage_client import StorageClient from foglamp.common import logger __author__ = "Ashwin Gopalakrishnan, Ashish Jabble" __copyright__ = "Copyright (c) 2017 OSIsoft, LLC" __license__ = "Apache 2.0" __version__ = "${VERSION}" _logger = logger.setup(__name__) # MAKE UPPER_CASE _valid_type_strings = [ 'boolean', 'integer', 'string', 'IPv4', 'IPv6', 'X509 certificate', 'password', 'JSON' ] class ConfigurationManager(object): """ Configuration Manager General naming convention: category(s) category_name - string
'default': 'cc2650poll' }, 'pollInterval': { 'description': 'The interval between poll calls to the device poll routine expressed in milliseconds.', 'type': 'integer', 'default': '500' }, 'bluetoothAddress': { 'description': 'Bluetooth MAC address', 'type': 'string', 'default': 'B0:91:22:EA:79:04' } } _LOGGER = logger.setup(__name__, level=20) sensortag_characteristics = characteristics def plugin_info(): """ Returns information about the plugin. Args: Returns: dict: plugin information Raises: """ return { 'name': 'TI SensorTag CC2650 Poll plugin',
"e000025": "Required argument '--name' is missing - command line |{0}|", "e000026": "Required argument '--port' is missing - command line |{0}|", "e000027": "Required argument '--address' is missing - command line |{0}|", } """ Messages used for Information, Warning and Error notice """ # LOG configuration _LOG_LEVEL_DEBUG = 10 _LOG_LEVEL_INFO = 20 _LOG_LEVEL_WARNING = 30 _LOGGER_LEVEL = _LOG_LEVEL_WARNING _LOGGER_DESTINATION = logger.SYSLOG _LOGGER = logger.setup(__name__, destination=_LOGGER_DESTINATION, level=_LOGGER_LEVEL) _event_loop = "" _log_performance = False """ Enable/Disable performance logging, enabled using a command line parameter""" class PluginInitialiseFailed(RuntimeError): """ PluginInitializeFailed """ pass class UnknownDataSource(RuntimeError): """ the data source could be only one among: readings, statistics or audit """ pass
"e000023": "cannot complete the initialization - error details |{0}|", "e000024": "unable to log the operation in the Storage Layer - error details |{0}|", "e000025": "Required argument '--name' is missing - command line |{0}|", "e000026": "Required argument '--port' is missing - command line |{0}|", "e000027": "Required argument '--address' is missing - command line |{0}|", "e000028": "cannot complete the fetch operation - error details |{0}|", "e000029": "an error occurred during the teardown operation - error details |{0}|", "e000030": "unable to create parent configuration category", "e000031": "unable to convert in memory data structure related to the readings data " "- error details |{0}| - row |{1}|", "e000032": "asset code not defined - row |{0}|", } """ Messages used for Information, Warning and Error notice """ _LOGGER = logger.setup(__name__) _event_loop = "" _log_performance = False """ Enable/Disable performance logging, enabled using a command line parameter""" class PluginInitialiseFailed(RuntimeError): """ PluginInitializeFailed """ pass class UnknownDataSource(RuntimeError): """ the data source could be only one among: readings or statistics""" pass
#!/usr/bin/env python3 # -*- coding: utf-8 -*- # FOGLAMP_BEGIN # See: http://foglamp.readthedocs.io/ # FOGLAMP_END """Statistics history process starter""" import asyncio from foglamp.tasks.statistics.statistics_history import StatisticsHistory from foglamp.common import logger __author__ = "Terris Linenbach, Vaibhav Singhal" __copyright__ = "Copyright (c) 2017 OSIsoft, LLC" __license__ = "Apache 2.0" __version__ = "${VERSION}" if __name__ == '__main__': _logger = logger.setup("StatisticsHistory") statistics_history_process = StatisticsHistory() loop = asyncio.get_event_loop() loop.run_until_complete(statistics_history_process.run())