def test_conversion(self):
""" Test if the conversion from old dict-based config to new list based is correct. """
old_path = resource_filename(
'intelmq', 'tests/bots/experts/modify/old_format.conf')
old_config = load_configuration(old_path)
new_path = resource_filename(
'intelmq', 'tests/bots/experts/modify/new_format.conf')
new_config = load_configuration(new_path)
self.assertDictEqual(convert_config(old_config)[0], new_config[0])
def test_conversion(self):
""" Test if the conversion from old dict-based config to new list based is correct. """
old_path = resource_filename('intelmq',
'tests/bots/experts/modify/old_format.conf')
old_config = load_configuration(old_path)
new_path = resource_filename('intelmq',
'tests/bots/experts/modify/new_format.conf')
new_config = load_configuration(new_path)
self.assertDictEqual(convert_config(old_config)[0],
new_config[0])
def __init__(self, message=(), auto=False, harmonization=None):
try:
classname = message['__type'].lower()
del message['__type']
except (KeyError, TypeError):
classname = self.__class__.__name__.lower()
if harmonization is None:
harmonization = utils.load_configuration(HARMONIZATION_CONF_FILE)
try:
self.harmonization_config = harmonization[classname]
except KeyError:
raise exceptions.InvalidArgument('__type',
got=classname,
expected=VALID_MESSSAGE_TYPES,
docs=HARMONIZATION_CONF_FILE)
if classname == 'event' and self.harmonization_config['extra']['type'] == 'JSON':
warnings.warn("Assuming harmonization type 'JSONDict' for harmonization field 'extra'. "
"This assumption will be removed in version 2.0.", DeprecationWarning)
self.harmonization_config['extra']['type'] = 'JSONDict'
super(Message, self).__init__()
if isinstance(message, dict):
iterable = message.items()
elif isinstance(message, tuple):
iterable = message
for key, value in iterable:
if not self.add(key, value, sanitize=False, raise_failure=False):
self.add(key, value, sanitize=True)
def harmonization(defaults, runtime, harmonization, dry_run):
"""
Checks if all harmonization fields and types are correct
"""
changed = None
original = load_configuration(resource_filename('intelmq',
'etc/harmonization.conf'))
for msg_type, msg in original.items():
if msg_type not in harmonization:
harmonization[msg_type] = msg
changed = True
continue
for fieldname, field in msg.items():
if fieldname not in harmonization[msg_type]:
harmonization[msg_type][fieldname] = field
changed = True
continue
if harmonization[msg_type][fieldname]['type'] != original[msg_type][fieldname]['type']:
harmonization[msg_type][fieldname]['type'] = original[msg_type][fieldname]['type']
changed = True
installed_regex = harmonization[msg_type][fieldname].get('regex')
original_regex = original[msg_type][fieldname].get('regex')
if original_regex and original_regex != installed_regex:
harmonization[msg_type][fieldname]['regex'] = original[msg_type][fieldname]['regex']
changed = True
installed_regex = harmonization[msg_type][fieldname].get('iregex')
original_regex = original[msg_type][fieldname].get('iregex')
if original_regex and original_regex != installed_regex:
harmonization[msg_type][fieldname]['iregex'] = original[msg_type][fieldname]['iregex']
changed = True
return changed, defaults, runtime, harmonization
def init(self):
if not SieveExpertBot.harmonization:
harmonization_config = utils.load_configuration(HARMONIZATION_CONF_FILE)
SieveExpertBot.harmonization = harmonization_config['event']
self.metamodel = SieveExpertBot.init_metamodel()
self.sieve = SieveExpertBot.read_sieve_file(self.parameters.file, self.metamodel)
def v300_pipeline_file_removal(configuration, harmonization, dry_run, **kwargs):
"""
Remove the pipeline.conf file
"""
changed = None
messages = []
pipeline_file = Path(CONFIG_DIR) / "pipeline.conf"
if pipeline_file.exists():
pipelines = load_configuration(pipeline_file)
for bot in configuration:
if bot == 'global':
continue
if bot in pipelines:
if 'destination-queues' in pipelines[bot]:
destination_queues = pipelines[bot]['destination-queues']
if isinstance(destination_queues, dict):
configuration[bot]['parameters']['destination_queues'] = destination_queues
if isinstance(destination_queues, list):
configuration[bot]['parameters']['destination_queues'] = {'_default': destination_queues}
if isinstance(destination_queues, str):
configuration[bot]['parameters']['destination_queues'] = {'_default': [destination_queues]}
if 'source-queue' in pipelines[bot]:
if pipelines[bot]['source-queue'] != f"{bot}-queue":
configuration[bot]['parameters']['source_queue'] = pipelines[bot]['source-queue']
if dry_run:
print(f'Would now remove file {pipeline_file!r}.')
else:
pipeline_file.unlink()
changed = True
messages = ' '.join(messages)
return messages if messages else changed, configuration, harmonization
def init(self):
config = load_configuration(self.parameters.configuration_path)
if type(config) is dict:
self.logger.warning('Support for dict-based configuration will be '
'removed in version 3.0. Have a look at the '
'NEWS file section 1.0.0.dev7.')
config = modify_expert_convert_config(config)
if getattr(self.parameters, 'case_sensitive', True):
self.re_kwargs = {}
else:
self.re_kwargs = {'flags': re.IGNORECASE}
if not hasattr(self.parameters, 'overwrite'):
self.logger.warning("Parameter 'overwrite' is not given, assuming 'True'. "
"Please set it explicitly, default will change to "
"'False' in version 3.0.0'.")
self.overwrite = getattr(self.parameters, 'overwrite', True)
# regex compilation
self.config = []
for rule in config:
self.config.append(rule)
for field, expression in rule["if"].items():
if isinstance(expression, str) and expression != '':
self.config[-1]["if"][field] = re.compile(expression, **self.re_kwargs)
def __init__(self, message=(), auto=False, harmonization=None):
try:
classname = message['__type'].lower()
del message['__type']
except (KeyError, TypeError):
classname = self.__class__.__name__.lower()
if harmonization is None:
harmonization = utils.load_configuration(HARMONIZATION_CONF_FILE)
try:
self.harmonization_config = harmonization[classname]
except KeyError:
raise exceptions.InvalidArgument('__type',
got=classname,
expected=VALID_MESSSAGE_TYPES,
docs=HARMONIZATION_CONF_FILE)
super(Message, self).__init__()
if isinstance(message, dict):
iterable = message.items()
elif isinstance(message, tuple):
iterable = message
for key, value in iterable:
if not self.add(key, value, sanitize=False, raise_failure=False):
self.add(key, value, sanitize=True)
def __load_pipeline_configuration(self):
self.logger.debug("Loading pipeline configuration from %r." %
PIPELINE_CONF_FILE)
config = utils.load_configuration(PIPELINE_CONF_FILE)
self.__source_queues = None
self.__destination_queues = None
if self.__bot_id in list(config.keys()):
if 'source-queue' in config[self.__bot_id].keys():
self.__source_queues = config[self.__bot_id]['source-queue']
self.logger.debug("Pipeline configuration: parameter "
"'source-queue' loaded with the value {!r}."
"".format(self.__source_queues))
if 'destination-queues' in config[self.__bot_id].keys():
self.__destination_queues = config[
self.__bot_id]['destination-queues']
self.logger.debug("Pipeline configuration: parameter"
"'destination-queues' loaded with the value "
"{!r}.".format(", ".join(
self.__destination_queues)))
else:
self.logger.error("Pipeline configuration: no key "
"{!r}.".format(self.__bot_id))
self.stop()
def check(parameters):
try:
harmonization_config = utils.load_configuration(HARMONIZATION_CONF_FILE)
SieveExpertBot._harmonization = harmonization_config['event']
grammarfile = os.path.join(os.path.dirname(__file__), 'sieve.tx')
if not os.path.exists(grammarfile):
raise FileExistsError(f'Sieve grammar file not found: {grammarfile!r}.')
metamodel = None
try:
metamodel = metamodel_from_file(grammarfile)
except TextXError as e:
raise ValueError(f'Could not process sieve grammar file. Error in ({e.line}, {e.col}).')
if not os.path.exists(parameters['file']):
raise ValueError(f'File does not exist: {parameters["file"]!r}')
try:
metamodel.model_from_file(parameters['file'])
except TextXError as e:
raise ValueError(f'Could not process sieve file {parameters["file"]!r}. Error in ({e.line}, {e.col}).')
except Exception:
return [['error', f'Validation of Sieve file failed with the following traceback: {traceback.format_exc()!r}']]
def init(self):
self.config = load_configuration(self.parameters.configuration_path)
self.logger.debug("Connecting to PostgreSQL.")
try:
if hasattr(self.parameters, 'connect_timeout'):
connect_timeout = self.parameters.connect_timeout
else:
connect_timeout = 5
self.con = psycopg2.connect(database=self.parameters.database,
user=self.parameters.user,
password=self.parameters.password,
host=self.parameters.host,
port=self.parameters.port,
sslmode=self.parameters.sslmode,
connect_timeout=connect_timeout,
)
self.cur = self.con.cursor()
global SELECT_QUERY
SELECT_QUERY = SELECT_QUERY.format(table=self.parameters.table)
except:
self.logger.exception('Failed to connect to database.')
self.stop()
self.logger.info("Connected to PostgreSQL.")
def __load_pipeline_configuration(self):
self.logger.debug("Loading pipeline configuration")
config = utils.load_configuration(PIPELINE_CONF_FILE)
self.__source_queues = None
self.__destination_queues = None
if self.__bot_id in list(config.keys()):
if "source-queue" in config[self.__bot_id].keys():
self.__source_queues = config[self.__bot_id]["source-queue"]
self.logger.debug(
"Pipeline configuration: parameter "
"'source-queue' loaded with the value {!r}."
"".format(self.__source_queues)
)
if "destination-queues" in config[self.__bot_id].keys():
self.__destination_queues = config[self.__bot_id]["destination-queues"]
self.logger.debug(
"Pipeline configuration: parameter"
"'destination-queues' loaded with the value "
"{!r}.".format(", ".join(self.__destination_queues))
)
else:
self.logger.error("Pipeline configuration: no key " "{!r}.".format(self.__bot_id))
self.stop()
def set_bot(cls):
cls.bot_reference = SquelcherExpertBot
cls.default_input_message = INPUT1
try:
cls.sysconfig = (utils.load_configuration(RUNTIME_CONF_FILE)
['Expert']['Squelcher'])
except:
cls.sysconfig = {"configuration_path": "/opt/intelmq/etc/"
"squelcher.conf",
"host": "localhost",
"port": 5432,
"database": "intelmq",
"user": "******",
"password": "******",
"sslmode": "require",
"table": "tests",
}
cls.con = psycopg2.connect(database=cls.sysconfig['database'],
user=cls.sysconfig['user'],
password=cls.sysconfig['password'],
host=cls.sysconfig['host'],
port=cls.sysconfig['port'],
sslmode=cls.sysconfig['sslmode'],
)
cls.con.autocommit = True
cls.cur = cls.con.cursor()
cls.cur.execute("TRUNCATE TABLE {}".format(cls.sysconfig['table']))
global INSERT_QUERY
INSERT_QUERY = INSERT_QUERY.format(table=cls.sysconfig['table'])
def load_pipeline_configuration(self):
self.logger.debug("Loading pipeline configuration")
config = utils.load_configuration(PIPELINE_CONF_FILE)
self.source_queues = None
self.destination_queues = None
if self.bot_id in list(config.keys()):
if 'source-queue' in config[self.bot_id].keys():
self.source_queues = config[self.bot_id]['source-queue']
self.logger.debug("Pipeline configuration: parameter "
"'source-queue' loaded with the value '%s'."
% self.source_queues)
if 'destination-queues' in config[self.bot_id].keys():
self.destination_queues = config[
self.bot_id]['destination-queues']
self.logger.debug("Pipeline configuration: parameter"
"'destination-queues' loaded with the value "
"'%s'." % ", ".join(self.destination_queues))
else:
self.logger.error("Pipeline configuration: no key "
"'{}'.".format(self.bot_id))
self.stop()
def v100_dev7_modify_syntax(defaults, runtime, harmonization, dry_run):
"""
Migrate modify bot configuration format
"""
changed = None
for bot_id, bot in runtime.items():
if bot["module"] == "intelmq.bots.experts.modify.expert":
if "configuration_path" in bot["parameters"]:
config = load_configuration(bot["parameters"]["configuration_path"])
if type(config) is dict:
new_config = modify_expert_convert_config(config)
if len(config) != len(new_config):
return 'Error converting modify expert syntax. Different size of configurations. Please report this.'
changed = True
if dry_run:
print('Would now convert file %r syntax.',
bot["parameters"]["configuration_path"])
continue
try:
write_configuration(bot["parameters"]["configuration_path"],
new_config)
except PermissionError:
return ('Can\'t update %s\'s configuration: Permission denied.' % bot_id,
defaults, runtime, harmonization)
return changed, defaults, runtime, harmonization
def init(self):
if not SieveExpertBot.harmonization:
harmonization_config = utils.load_configuration(HARMONIZATION_CONF_FILE)
SieveExpertBot.harmonization = harmonization_config['event']
self.metamodel = SieveExpertBot.init_metamodel()
self.sieve = SieveExpertBot.read_sieve_file(self.parameters.file, self.metamodel)
def mocked(conf_file):
if conf_file == PIPELINE_CONF_FILE:
return {
bot_id: {
"source-queue": src_name,
"destination-queues": dst_names
},
}
elif conf_file == RUNTIME_CONF_FILE:
conf = BOT_CONFIG.copy()
conf.update(sysconfig)
return {
bot_id: {
'description':
'Instance of a bot for automated unit tests.',
'group': group,
'module': module,
'name': 'Test Bot',
'parameters': conf,
}
}
elif conf_file.startswith(CONFIG_DIR):
confname = os.path.join('etc/', os.path.split(conf_file)[-1])
fname = pkg_resources.resource_filename('intelmq', confname)
with open(fname, 'rt') as fpconfig:
return json.load(fpconfig)
else:
return utils.load_configuration(conf_file)
def load_pipeline_configuration(self):
config = utils.load_configuration(PIPELINE_CONF_FILE)
self.logger.debug("Pipeline configuration: loading '%s' section" \
" from '%s' file" % (self.bot_id, PIPELINE_CONF_FILE))
self.source_queues = None
self.destination_queues = None
if self.bot_id in config.keys():
if 'source-queue' in config[self.bot_id].keys():
self.source_queues = config[self.bot_id]['source-queue']
self.logger.debug("Pipeline configuration: parameter " \
"'source-queue' loaded with the value '%s'" % self.source_queues)
if 'destination-queues' in config[self.bot_id].keys():
self.destination_queues = config[self.bot_id]['destination-queues']
self.logger.debug("Pipeline configuration: parameter" \
"'destination-queues' loaded with the value" \
" '%s'" % ", ".join(self.destination_queues))
else:
self.logger.error("Pipeline configuration: failed to load configuration")
self.stop()
def load_system_configuration(self):
setattr(self.parameters, 'logging_path' , DEFAULT_LOGGING_PATH)
setattr(self.parameters, 'logging_level' , DEFAULT_LOGGING_LEVEL)
config = utils.load_configuration(SYSTEM_CONF_FILE)
for option, value in config.iteritems():
setattr(self.parameters, option, value)
def __init__(self, message=(), auto=False, harmonization=None):
try:
classname = message['__type'].lower()
del message['__type']
except (KeyError, TypeError):
classname = self.__class__.__name__.lower()
if harmonization is None:
harmonization = utils.load_configuration(HARMONIZATION_CONF_FILE)
try:
self.harmonization_config = harmonization[classname]
except KeyError:
raise exceptions.InvalidArgument('__type',
got=classname,
expected=VALID_MESSSAGE_TYPES,
docs=HARMONIZATION_CONF_FILE)
super(Message, self).__init__()
if isinstance(message, dict):
iterable = message.items()
elif isinstance(message, tuple):
iterable = message
for key, value in iterable:
if not self.add(key, value, sanitize=False, raise_failure=False):
self.add(key, value, sanitize=True)
def set_bot(cls):
cls.bot_reference = SquelcherExpertBot
cls.default_input_message = INPUT1
try:
cls.sysconfig = (utils.load_configuration(RUNTIME_CONF_FILE)
['Expert']['Squelcher'])
except:
cls.sysconfig = {
"configuration_path": "/opt/intelmq/etc/"
"squelcher.conf",
"host": "localhost",
"port": 5432,
"database": "intelmq",
"user": "******",
"password": "******",
"sslmode": "require",
"table": "tests",
}
cls.con = psycopg2.connect(
database=cls.sysconfig['database'],
user=cls.sysconfig['user'],
password=cls.sysconfig['password'],
host=cls.sysconfig['host'],
port=cls.sysconfig['port'],
sslmode=cls.sysconfig['sslmode'],
)
cls.con.autocommit = True
cls.cur = cls.con.cursor()
cls.cur.execute("TRUNCATE TABLE {}".format(cls.sysconfig['table']))
global INSERT_QUERY
INSERT_QUERY = INSERT_QUERY.format(table=cls.sysconfig['table'])
def init(self):
self.config = load_configuration(self.parameters.configuration_path)
self.logger.debug("Connecting to PostgreSQL.")
try:
if hasattr(self.parameters, 'connect_timeout'):
connect_timeout = self.parameters.connect_timeout
else:
connect_timeout = 5
self.con = psycopg2.connect(
database=self.parameters.database,
user=self.parameters.user,
password=self.parameters.password,
host=self.parameters.host,
port=self.parameters.port,
sslmode=self.parameters.sslmode,
connect_timeout=connect_timeout,
)
self.cur = self.con.cursor()
global SELECT_QUERY
SELECT_QUERY = SELECT_QUERY.format(table=self.parameters.table)
except:
self.logger.exception('Failed to connect to database.')
self.stop()
self.logger.info("Connected to PostgreSQL.")
def load_system_configuration(self):
if os.path.exists(SYSTEM_CONF_FILE):
try:
config = utils.load_configuration(SYSTEM_CONF_FILE)
except ValueError as exc: # pragma: no cover
self.abort('Error loading %r: %s' % (SYSTEM_CONF_FILE, exc))
for option, value in config.items():
setattr(self.parameters, option, value)
def load_system_configuration(self):
if os.path.exists(SYSTEM_CONF_FILE):
try:
config = utils.load_configuration(SYSTEM_CONF_FILE)
except ValueError as exc:
exit('Invalid syntax in %r: %s' % (SYSTEM_CONF_FILE, exc))
for option, value in config.items():
setattr(self.parameters, option, value)
def load_defaults_configuration(self):
# Load defaults configuration
try:
config = utils.load_configuration(DEFAULTS_CONF_FILE)
except ValueError as exc: # pragma: no cover
self.abort('Error loading %r: %s' % (DEFAULTS_CONF_FILE, exc))
for option, value in config.items():
setattr(self.parameters, option, value)
def new_get_runtime() -> dict:
runtime_conf = utils.load_configuration(pkg_resources.resource_filename('intelmq', 'etc/runtime.yaml'))
if 'global' not in runtime_conf:
runtime_conf['global'] = {}
runtime_conf['global']['http_proxy'] = 'http://localhost:8080'
runtime_conf['global']['https_proxy'] = 'http://localhost:8080'
runtime_conf['cymru-whois-expert']['parameters']['http_proxy'] = 'http://localhost:8081'
return runtime_conf
def load_defaults_configuration(self):
# Load defaults configuration
try:
config = utils.load_configuration(DEFAULTS_CONF_FILE)
except ValueError as exc: # pragma: no cover
self.abort('Error loading %r: %s' % (DEFAULTS_CONF_FILE, exc))
for option, value in config.items():
setattr(self.parameters, option, value)
def load_defaults_configuration(self):
# Load defaults configuration section
try:
config = utils.load_configuration(DEFAULTS_CONF_FILE)
except ValueError as exc:
exit('Invalid syntax in %r: %s' % (DEFAULTS_CONF_FILE, exc))
for option, value in config.items():
setattr(self.parameters, option, value)
def load_system_configuration(self):
if os.path.exists(SYSTEM_CONF_FILE):
try:
config = utils.load_configuration(SYSTEM_CONF_FILE)
except ValueError as exc:
exit('Invalid syntax in %r: %s' % (SYSTEM_CONF_FILE, exc))
for option, value in config.items():
setattr(self.parameters, option, value)
def load_defaults_configuration(self):
# Load defaults configuration section
try:
config = utils.load_configuration(DEFAULTS_CONF_FILE)
except ValueError as exc:
exit('Invalid syntax in %r: %s' % (DEFAULTS_CONF_FILE, exc))
for option, value in config.items():
setattr(self.parameters, option, value)
def __load_runtime_configuration(self):
self.logger.debug("Loading runtime configuration.")
config = utils.load_configuration(RUNTIME_CONF_FILE)
if self.__bot_id in list(config.keys()):
for option, value in config[self.__bot_id].items():
setattr(self.parameters, option, value)
self.logger.debug("Runtime configuration: parameter {!r} "
"loaded with value {!r}.".format(option, value))
def __init__(self, bot_id):
super(OtherCsvParserBot, self).__init__(bot_id=bot_id)
self._config = utils.load_configuration(CSV_PARSER_CONF_FILE)
self._config = self._config[self.parameters.othername]
self.csv_fieldnames = self._config['sequence']
self.fixed_field = self._config['fixed_field']
if self._config.get('ignore_lines_starting'):
self.ignore_lines_starting = self._config['ignore_lines_starting']
def check(parameters):
try:
harmonization_config = utils.load_configuration(HARMONIZATION_CONF_FILE)
SieveExpertBot.harmonization = harmonization_config['event']
metamodel = SieveExpertBot.init_metamodel()
SieveExpertBot.read_sieve_file(parameters['file'], metamodel)
except Exception:
return [['error', 'Validation of Sieve file failed with the following traceback: %r' % traceback.format_exc()]]
def init(self):
self.config = load_configuration(self.parameters.configuration_path)
if type(self.config) is dict:
self.config = convert_config(self.config)
if getattr(self.parameters, 'case_sensitive', True):
self.re_kwargs = {}
else:
self.re_kwargs = {'flags': re.IGNORECASE}
def init(self):
self.config = load_configuration(self.parameters.configuration_path)
if type(self.config) is dict:
self.config = convert_config(self.config)
if getattr(self.parameters, 'case_sensitive', True):
self.re_kwargs = {}
else:
self.re_kwargs = {'flags': re.IGNORECASE}
def __load_runtime_configuration(self):
self.logger.debug("Loading runtime configuration.")
config = utils.load_configuration(RUNTIME_CONF_FILE)
if self.__bot_id in list(config.keys()):
for option, value in config[self.__bot_id].items():
setattr(self.parameters, option, value)
self.logger.debug("Runtime configuration: parameter {!r} "
"loaded with value {!r}.".format(option, value))
def load_runtime_configuration(self):
self.logger.debug("Loading runtime configuration")
config = utils.load_configuration(RUNTIME_CONF_FILE)
if self.bot_id in list(config.keys()):
for option, value in config[self.bot_id].items():
setattr(self.parameters, option, value)
self.logger.debug("Runtime configuration: parameter '%s' "
"loaded with value '%s'." % (option, value))
def check(parameters):
try:
harmonization_config = utils.load_configuration(HARMONIZATION_CONF_FILE)
SieveExpertBot.harmonization = harmonization_config['event']
metamodel = SieveExpertBot.init_metamodel()
SieveExpertBot.read_sieve_file(parameters['file'], metamodel)
except Exception as e:
return [['error', 'Validation of Sieve file failed with the following traceback: %r' % traceback.format_exc()]]
def load_runtime_configuration(self):
self.logger.debug("Loading runtime configuration")
config = utils.load_configuration(RUNTIME_CONF_FILE)
if self.bot_id in list(config.keys()):
for option, value in config[self.bot_id].items():
setattr(self.parameters, option, value)
self.logger.debug("Runtime configuration: parameter '%s' "
"loaded with value '%s'." % (option, value))
def __load_harmonization_configuration(self):
self.logger.debug("Loading Harmonization configuration.")
config = utils.load_configuration(HARMONIZATION_CONF_FILE)
for message_types in config.keys():
for key in config[message_types].keys():
for _key in config.keys():
if _key.startswith("%s." % key):
raise exceptions.ConfigurationError("harmonization", "Key %s is not valid." % _key)
def __load_system_configuration(self):
self.__log_buffer.append(("debug", "Loading system configuration"))
config = utils.load_configuration(SYSTEM_CONF_FILE)
for option, value in config.items():
setattr(self.parameters, option, value)
self.__log_buffer.append(
("debug", "System configuration: parameter {!r} " "loaded with value {!r}.".format(option, value))
)
def load_system_configuration(self):
self.log_buffer.append(('debug', "Loading system configuration"))
config = utils.load_configuration(SYSTEM_CONF_FILE)
for option, value in config.items():
setattr(self.parameters, option, value)
self.log_buffer.append(
('debug', "System configuration: parameter '{}' "
"loaded with value '{}'.".format(option, value)))
def load_harmonization_configuration(self):
self.logger.debug("Loading Harmonization configuration.")
config = utils.load_configuration(HARMONIZATION_CONF_FILE)
for message_types in config.keys():
for key in config[message_types].keys():
for _key in config.keys():
if _key.startswith("%s." % key):
raise exceptions.ConfigurationError(
'harmonization', "Key %s is not valid." % _key)
def __load_defaults_configuration(self):
self.__log_buffer.append(('debug', "Loading defaults configuration from %r."
"" % DEFAULTS_CONF_FILE))
config = utils.load_configuration(DEFAULTS_CONF_FILE)
setattr(self.parameters, 'logging_path', DEFAULT_LOGGING_PATH)
for option, value in config.items():
setattr(self.parameters, option, value)
self.__log_configuration_parameter("defaults", option, value)
def __load_defaults_configuration(self):
self.__log_buffer.append(('debug', "Loading defaults configuration from %r."
"" % DEFAULTS_CONF_FILE))
config = utils.load_configuration(DEFAULTS_CONF_FILE)
setattr(self.parameters, 'logging_path', DEFAULT_LOGGING_PATH)
for option, value in config.items():
setattr(self.parameters, option, value)
self.__log_configuration_parameter("defaults", option, value)
def load_harmonization_configuration(self):
harmonization_config = utils.load_configuration(HARMONIZATION_CONF_FILE)
self.logger.debug("Harmonization configuration: loading all '%s' file" %
HARMONIZATION_CONF_FILE )
for message_types in harmonization_config.keys():
for key in harmonization_config[message_types].keys():
for _key in harmonization_config.keys():
if _key.startswith("%s." % key): # FIXME: write in devguide the rules for the keys names
raise exceptions.ConfigurationError(
HARMONIZATION_CONF_FILE, "key %s is not valid" % _key)
def load_defaults_configuration(self):
# Load defaults configuration
try:
config = utils.load_configuration(DEFAULTS_CONF_FILE)
except ValueError as exc: # pragma: no cover
msg = 'Error loading %r: %s' % (DEFAULTS_CONF_FILE, exc)
if self.interactive:
exit(msg)
else:
raise ValueError(msg)
for option, value in config.items():
setattr(self.parameters, option, value)
def load_system_configuration(self):
if os.path.exists(SYSTEM_CONF_FILE):
try:
config = utils.load_configuration(SYSTEM_CONF_FILE)
except ValueError as exc: # pragma: no cover
msg = 'Error loading %r: %s' % (SYSTEM_CONF_FILE, exc)
if self.interactive:
exit(msg)
else:
raise ValueError(msg)
for option, value in config.items():
setattr(self.parameters, option, value)
def __load_system_configuration(self):
if os.path.exists(SYSTEM_CONF_FILE):
self.__log_buffer.append(('warning', "system.conf is deprecated "
"and will be removed in 1.0. "
"Use defaults.conf instead!"))
self.__log_buffer.append(('debug', "Loading system configuration from %r."
"" % SYSTEM_CONF_FILE))
config = utils.load_configuration(SYSTEM_CONF_FILE)
for option, value in config.items():
setattr(self.parameters, option, value)
self.__log_configuration_parameter("system", option, value)
def __load_system_configuration(self):
if os.path.exists(SYSTEM_CONF_FILE):
self.__log_buffer.append(('warning', "system.conf is deprecated "
"and will be removed in 1.0. "
"Use defaults.conf instead!"))
self.__log_buffer.append(('debug', "Loading system configuration from %r."
"" % SYSTEM_CONF_FILE))
config = utils.load_configuration(SYSTEM_CONF_FILE)
for option, value in config.items():
setattr(self.parameters, option, value)
self.__log_configuration_parameter("system", option, value)
def load_system_configuration(self):
if os.path.exists(SYSTEM_CONF_FILE):
try:
config = utils.load_configuration(SYSTEM_CONF_FILE)
except ValueError as exc: # pragma: no cover
msg = 'Error loading %r: %s' % (SYSTEM_CONF_FILE, exc)
if self.interactive:
exit(msg)
else:
raise ValueError(msg)
for option, value in config.items():
setattr(self.parameters, option, value)
def __load_system_configuration(self):
if os.path.exists(SYSTEM_CONF_FILE):
self.__log_buffer.append(('warning', "system.conf is deprecated and will be"
"removed in 1.0. Use defaults.conf instead!"))
self.__log_buffer.append(('debug', "Loading system configuration."))
config = utils.load_configuration(SYSTEM_CONF_FILE)
for option, value in config.items():
setattr(self.parameters, option, value)
self.__log_buffer.append(('debug',
"System configuration: parameter {!r} "
"loaded with value {!r}.".format(option, value)))
def load_defaults_configuration(self):
# Load defaults configuration
try:
config = utils.load_configuration(DEFAULTS_CONF_FILE)
except ValueError as exc: # pragma: no cover
msg = 'Error loading %r: %s' % (DEFAULTS_CONF_FILE, exc)
if self.interactive:
exit(msg)
else:
raise ValueError(msg)
for option, value in config.items():
setattr(self.parameters, option, value)
def __load_runtime_configuration(self):
self.logger.debug("Loading runtime configuration from %r." % RUNTIME_CONF_FILE)
config = utils.load_configuration(RUNTIME_CONF_FILE)
if self.__bot_id in list(config.keys()):
params = config[self.__bot_id]
if 'parameters' in params:
params = params['parameters']
else:
self.logger.warning('Old runtime configuration format found.')
for option, value in params.items():
setattr(self.parameters, option, value)
self.__log_configuration_parameter("runtime", option, value)
def load_defaults_configuration(self):
self.log_buffer.append(('debug', "Loading defaults configuration."))
config = utils.load_configuration(DEFAULTS_CONF_FILE)
setattr(self.parameters, 'logging_path', DEFAULT_LOGGING_PATH)
setattr(self.parameters, 'logging_level', DEFAULT_LOGGING_LEVEL)
for option, value in config.items():
setattr(self.parameters, option, value)
self.log_buffer.append(('debug',
"Defaults configuration: parameter '{}' "
"loaded with value '{}'.".format(option,
value)))
def load_defaults_configuration(self):
# Load defaults configuration section
config = utils.load_configuration(DEFAULTS_CONF_FILE)
self.logger.debug("Defaults configuration" \
" from '%s' file" % DEFAULTS_CONF_FILE)
for option, value in config.iteritems():
setattr(self.parameters, option, value)
self.logger.debug("Defaults configuration: parameter '%s' " \
"loaded with value '%s'" % (option, value))
def __load_defaults_configuration(self):
self.__log_buffer.append(("debug", "Loading defaults configuration."))
config = utils.load_configuration(DEFAULTS_CONF_FILE)
setattr(self.parameters, "testing", False)
setattr(self.parameters, "logging_path", DEFAULT_LOGGING_PATH)
setattr(self.parameters, "logging_level", DEFAULT_LOGGING_LEVEL)
for option, value in config.items():
setattr(self.parameters, option, value)
self.__log_buffer.append(
("debug", "Defaults configuration: parameter {!r} " "loaded with value {!r}.".format(option, value))
)
def load_runtime_configuration(self):
# Load bot runtime configuration section
config = utils.load_configuration(RUNTIME_CONF_FILE)
self.logger.debug("Runtime configuration: loading '%s' section from" \
" '%s' file" % (self.bot_id, RUNTIME_CONF_FILE))
if self.bot_id in config.keys():
for option, value in config[self.bot_id].iteritems():
setattr(self.parameters, option, value)
self.logger.debug("Runtime configuration: parameter '%s' " \
"loaded with value '%s'" % (option, value))
def mocked(conf_file):
if conf_file == PIPELINE_CONF_FILE:
return {bot_id: {"source-queue": src_name,
"destination-queues": dst_names},
}
elif conf_file == RUNTIME_CONF_FILE:
conf = BOT_CONFIG.copy()
conf.update(sysconfig)
return {bot_id: {'parameters': conf}}
elif conf_file.startswith(CONFIG_DIR):
confname = os.path.join('etc/', os.path.split(conf_file)[-1])
fname = pkg_resources.resource_filename('intelmq',
confname)
with open(fname, 'rt') as fpconfig:
return json.load(fpconfig)
else:
return utils.load_configuration(conf_file)
def __load_runtime_configuration(self):
self.logger.debug("Loading runtime configuration from %r.", RUNTIME_CONF_FILE)
config = utils.load_configuration(RUNTIME_CONF_FILE)
reinitialize_logging = False
if self.__bot_id in list(config.keys()):
params = config[self.__bot_id]
self.run_mode = params.get('run_mode', 'stream')
for option, value in params['parameters'].items():
setattr(self.parameters, option, value)
self.__log_configuration_parameter("runtime", option, value)
if option.startswith('logging_'):
reinitialize_logging = True
if reinitialize_logging:
self.logger.handlers = [] # remove all existing handlers
self.__init_logger()
