示例#1
0
 def test_conversion(self):
     """ Test if the conversion from old dict-based config to new list based is correct. """
     old_path = resource_filename(
         'intelmq', 'tests/bots/experts/modify/old_format.conf')
     old_config = load_configuration(old_path)
     new_path = resource_filename(
         'intelmq', 'tests/bots/experts/modify/new_format.conf')
     new_config = load_configuration(new_path)
     self.assertDictEqual(convert_config(old_config)[0], new_config[0])
示例#2
0
 def test_conversion(self):
     """ Test if the conversion from old dict-based config to new list based is correct. """
     old_path = resource_filename('intelmq',
                                  'tests/bots/experts/modify/old_format.conf')
     old_config = load_configuration(old_path)
     new_path = resource_filename('intelmq',
                                  'tests/bots/experts/modify/new_format.conf')
     new_config = load_configuration(new_path)
     self.assertDictEqual(convert_config(old_config)[0],
                          new_config[0])
示例#3
0
    def __init__(self, message=(), auto=False, harmonization=None):
        try:
            classname = message['__type'].lower()
            del message['__type']
        except (KeyError, TypeError):
            classname = self.__class__.__name__.lower()

        if harmonization is None:
            harmonization = utils.load_configuration(HARMONIZATION_CONF_FILE)
        try:
            self.harmonization_config = harmonization[classname]
        except KeyError:
            raise exceptions.InvalidArgument('__type',
                                             got=classname,
                                             expected=VALID_MESSSAGE_TYPES,
                                             docs=HARMONIZATION_CONF_FILE)

        if classname == 'event' and self.harmonization_config['extra']['type'] == 'JSON':
            warnings.warn("Assuming harmonization type 'JSONDict' for harmonization field 'extra'. "
                          "This assumption will be removed in version 2.0.", DeprecationWarning)
            self.harmonization_config['extra']['type'] = 'JSONDict'

        super(Message, self).__init__()
        if isinstance(message, dict):
            iterable = message.items()
        elif isinstance(message, tuple):
            iterable = message
        for key, value in iterable:
            if not self.add(key, value, sanitize=False, raise_failure=False):
                self.add(key, value, sanitize=True)
示例#4
0
def harmonization(defaults, runtime, harmonization, dry_run):
    """
    Checks if all harmonization fields and types are correct
    """
    changed = None
    original = load_configuration(resource_filename('intelmq',
                                                    'etc/harmonization.conf'))
    for msg_type, msg in original.items():
        if msg_type not in harmonization:
            harmonization[msg_type] = msg
            changed = True
            continue
        for fieldname, field in msg.items():
            if fieldname not in harmonization[msg_type]:
                harmonization[msg_type][fieldname] = field
                changed = True
                continue
            if harmonization[msg_type][fieldname]['type'] != original[msg_type][fieldname]['type']:
                harmonization[msg_type][fieldname]['type'] = original[msg_type][fieldname]['type']
                changed = True
            installed_regex = harmonization[msg_type][fieldname].get('regex')
            original_regex = original[msg_type][fieldname].get('regex')
            if original_regex and original_regex != installed_regex:
                harmonization[msg_type][fieldname]['regex'] = original[msg_type][fieldname]['regex']
                changed = True
            installed_regex = harmonization[msg_type][fieldname].get('iregex')
            original_regex = original[msg_type][fieldname].get('iregex')
            if original_regex and original_regex != installed_regex:
                harmonization[msg_type][fieldname]['iregex'] = original[msg_type][fieldname]['iregex']
                changed = True
    return changed, defaults, runtime, harmonization
示例#5
0
文件: expert.py 项目: CZ-NIC/intelmq
    def init(self):
        if not SieveExpertBot.harmonization:
            harmonization_config = utils.load_configuration(HARMONIZATION_CONF_FILE)
            SieveExpertBot.harmonization = harmonization_config['event']

        self.metamodel = SieveExpertBot.init_metamodel()
        self.sieve = SieveExpertBot.read_sieve_file(self.parameters.file, self.metamodel)
示例#6
0
def v300_pipeline_file_removal(configuration, harmonization, dry_run, **kwargs):
    """
    Remove the pipeline.conf file
    """
    changed = None
    messages = []
    pipeline_file = Path(CONFIG_DIR) / "pipeline.conf"
    if pipeline_file.exists():
        pipelines = load_configuration(pipeline_file)
        for bot in configuration:
            if bot == 'global':
                continue
            if bot in pipelines:
                if 'destination-queues' in pipelines[bot]:
                    destination_queues = pipelines[bot]['destination-queues']
                    if isinstance(destination_queues, dict):
                        configuration[bot]['parameters']['destination_queues'] = destination_queues
                    if isinstance(destination_queues, list):
                        configuration[bot]['parameters']['destination_queues'] = {'_default': destination_queues}
                    if isinstance(destination_queues, str):
                        configuration[bot]['parameters']['destination_queues'] = {'_default': [destination_queues]}
                if 'source-queue' in pipelines[bot]:
                    if pipelines[bot]['source-queue'] != f"{bot}-queue":
                        configuration[bot]['parameters']['source_queue'] = pipelines[bot]['source-queue']
        if dry_run:
            print(f'Would now remove file {pipeline_file!r}.')
        else:
            pipeline_file.unlink()
        changed = True
    messages = ' '.join(messages)
    return messages if messages else changed, configuration, harmonization
示例#7
0
    def init(self):
        config = load_configuration(self.parameters.configuration_path)
        if type(config) is dict:
            self.logger.warning('Support for dict-based configuration will be '
                                'removed in version 3.0. Have a look at the '
                                'NEWS file section 1.0.0.dev7.')
            config = modify_expert_convert_config(config)

        if getattr(self.parameters, 'case_sensitive', True):
            self.re_kwargs = {}
        else:
            self.re_kwargs = {'flags': re.IGNORECASE}

        if not hasattr(self.parameters, 'overwrite'):
            self.logger.warning("Parameter 'overwrite' is not given, assuming 'True'. "
                                "Please set it explicitly, default will change to "
                                "'False' in version 3.0.0'.")
        self.overwrite = getattr(self.parameters, 'overwrite', True)

        # regex compilation
        self.config = []
        for rule in config:
            self.config.append(rule)
            for field, expression in rule["if"].items():
                if isinstance(expression, str) and expression != '':
                    self.config[-1]["if"][field] = re.compile(expression, **self.re_kwargs)
示例#8
0
文件: message.py 项目: razuz/intelmq
    def __init__(self, message=(), auto=False, harmonization=None):
        try:
            classname = message['__type'].lower()
            del message['__type']
        except (KeyError, TypeError):
            classname = self.__class__.__name__.lower()

        if harmonization is None:
            harmonization = utils.load_configuration(HARMONIZATION_CONF_FILE)
        try:
            self.harmonization_config = harmonization[classname]
        except KeyError:
            raise exceptions.InvalidArgument('__type',
                                             got=classname,
                                             expected=VALID_MESSSAGE_TYPES,
                                             docs=HARMONIZATION_CONF_FILE)

        super(Message, self).__init__()
        if isinstance(message, dict):
            iterable = message.items()
        elif isinstance(message, tuple):
            iterable = message
        for key, value in iterable:
            if not self.add(key, value, sanitize=False, raise_failure=False):
                self.add(key, value, sanitize=True)
示例#9
0
    def __load_pipeline_configuration(self):
        self.logger.debug("Loading pipeline configuration from %r." %
                          PIPELINE_CONF_FILE)
        config = utils.load_configuration(PIPELINE_CONF_FILE)

        self.__source_queues = None
        self.__destination_queues = None

        if self.__bot_id in list(config.keys()):

            if 'source-queue' in config[self.__bot_id].keys():
                self.__source_queues = config[self.__bot_id]['source-queue']
                self.logger.debug("Pipeline configuration: parameter "
                                  "'source-queue' loaded with the value {!r}."
                                  "".format(self.__source_queues))

            if 'destination-queues' in config[self.__bot_id].keys():

                self.__destination_queues = config[
                    self.__bot_id]['destination-queues']
                self.logger.debug("Pipeline configuration: parameter"
                                  "'destination-queues' loaded with the value "
                                  "{!r}.".format(", ".join(
                                      self.__destination_queues)))

        else:
            self.logger.error("Pipeline configuration: no key "
                              "{!r}.".format(self.__bot_id))
            self.stop()
示例#10
0
文件: expert.py 项目: motok/intelmq
    def check(parameters):
        try:
            harmonization_config = utils.load_configuration(HARMONIZATION_CONF_FILE)
            SieveExpertBot._harmonization = harmonization_config['event']

            grammarfile = os.path.join(os.path.dirname(__file__), 'sieve.tx')
            if not os.path.exists(grammarfile):
                raise FileExistsError(f'Sieve grammar file not found: {grammarfile!r}.')

            metamodel = None

            try:
                metamodel = metamodel_from_file(grammarfile)
            except TextXError as e:
                raise ValueError(f'Could not process sieve grammar file. Error in ({e.line}, {e.col}).')

            if not os.path.exists(parameters['file']):
                raise ValueError(f'File does not exist: {parameters["file"]!r}')

            try:
                metamodel.model_from_file(parameters['file'])
            except TextXError as e:
                raise ValueError(f'Could not process sieve file {parameters["file"]!r}. Error in ({e.line}, {e.col}).')
        except Exception:
            return [['error', f'Validation of Sieve file failed with the following traceback: {traceback.format_exc()!r}']]
示例#11
0
文件: expert.py 项目: certat/intelmq
    def init(self):
        self.config = load_configuration(self.parameters.configuration_path)

        self.logger.debug("Connecting to PostgreSQL.")
        try:
            if hasattr(self.parameters, 'connect_timeout'):
                connect_timeout = self.parameters.connect_timeout
            else:
                connect_timeout = 5

            self.con = psycopg2.connect(database=self.parameters.database,
                                        user=self.parameters.user,
                                        password=self.parameters.password,
                                        host=self.parameters.host,
                                        port=self.parameters.port,
                                        sslmode=self.parameters.sslmode,
                                        connect_timeout=connect_timeout,
                                        )
            self.cur = self.con.cursor()
            global SELECT_QUERY
            SELECT_QUERY = SELECT_QUERY.format(table=self.parameters.table)
        except:
            self.logger.exception('Failed to connect to database.')
            self.stop()
        self.logger.info("Connected to PostgreSQL.")
示例#12
0
    def __load_pipeline_configuration(self):
        self.logger.debug("Loading pipeline configuration")
        config = utils.load_configuration(PIPELINE_CONF_FILE)

        self.__source_queues = None
        self.__destination_queues = None

        if self.__bot_id in list(config.keys()):

            if "source-queue" in config[self.__bot_id].keys():
                self.__source_queues = config[self.__bot_id]["source-queue"]
                self.logger.debug(
                    "Pipeline configuration: parameter "
                    "'source-queue' loaded with the value {!r}."
                    "".format(self.__source_queues)
                )

            if "destination-queues" in config[self.__bot_id].keys():

                self.__destination_queues = config[self.__bot_id]["destination-queues"]
                self.logger.debug(
                    "Pipeline configuration: parameter"
                    "'destination-queues' loaded with the value "
                    "{!r}.".format(", ".join(self.__destination_queues))
                )

        else:
            self.logger.error("Pipeline configuration: no key " "{!r}.".format(self.__bot_id))
            self.stop()
示例#13
0
 def set_bot(cls):
     cls.bot_reference = SquelcherExpertBot
     cls.default_input_message = INPUT1
     try:
         cls.sysconfig = (utils.load_configuration(RUNTIME_CONF_FILE)
                          ['Expert']['Squelcher'])
     except:
         cls.sysconfig = {"configuration_path": "/opt/intelmq/etc/"
                                                "squelcher.conf",
                          "host": "localhost",
                          "port": 5432,
                          "database": "intelmq",
                          "user": "******",
                          "password": "******",
                          "sslmode": "require",
                          "table": "tests",
                          }
     cls.con = psycopg2.connect(database=cls.sysconfig['database'],
                                user=cls.sysconfig['user'],
                                password=cls.sysconfig['password'],
                                host=cls.sysconfig['host'],
                                port=cls.sysconfig['port'],
                                sslmode=cls.sysconfig['sslmode'],
                                )
     cls.con.autocommit = True
     cls.cur = cls.con.cursor()
     cls.cur.execute("TRUNCATE TABLE {}".format(cls.sysconfig['table']))
     global INSERT_QUERY
     INSERT_QUERY = INSERT_QUERY.format(table=cls.sysconfig['table'])
示例#14
0
文件: bot.py 项目: cvlli/intelmq
    def load_pipeline_configuration(self):
        self.logger.debug("Loading pipeline configuration")
        config = utils.load_configuration(PIPELINE_CONF_FILE)

        self.source_queues = None
        self.destination_queues = None

        if self.bot_id in list(config.keys()):

            if 'source-queue' in config[self.bot_id].keys():
                self.source_queues = config[self.bot_id]['source-queue']
                self.logger.debug("Pipeline configuration: parameter "
                                  "'source-queue' loaded with the value '%s'."
                                  % self.source_queues)

            if 'destination-queues' in config[self.bot_id].keys():

                self.destination_queues = config[
                    self.bot_id]['destination-queues']
                self.logger.debug("Pipeline configuration: parameter"
                                  "'destination-queues' loaded with the value "
                                  "'%s'." % ", ".join(self.destination_queues))

        else:
            self.logger.error("Pipeline configuration: no key "
                              "'{}'.".format(self.bot_id))
            self.stop()
示例#15
0
def v100_dev7_modify_syntax(defaults, runtime, harmonization, dry_run):
    """
    Migrate modify bot configuration format
    """
    changed = None
    for bot_id, bot in runtime.items():
        if bot["module"] == "intelmq.bots.experts.modify.expert":
            if "configuration_path" in bot["parameters"]:
                config = load_configuration(bot["parameters"]["configuration_path"])
                if type(config) is dict:
                    new_config = modify_expert_convert_config(config)
                    if len(config) != len(new_config):
                        return 'Error converting modify expert syntax. Different size of configurations. Please report this.'
                    changed = True
                    if dry_run:
                        print('Would now convert file %r syntax.',
                              bot["parameters"]["configuration_path"])
                        continue
                    try:
                        write_configuration(bot["parameters"]["configuration_path"],
                                            new_config)
                    except PermissionError:
                        return ('Can\'t update %s\'s configuration: Permission denied.' % bot_id,
                                defaults, runtime, harmonization)

    return changed, defaults, runtime, harmonization
示例#16
0
    def init(self):
        if not SieveExpertBot.harmonization:
            harmonization_config = utils.load_configuration(HARMONIZATION_CONF_FILE)
            SieveExpertBot.harmonization = harmonization_config['event']

        self.metamodel = SieveExpertBot.init_metamodel()
        self.sieve = SieveExpertBot.read_sieve_file(self.parameters.file, self.metamodel)
示例#17
0
 def mocked(conf_file):
     if conf_file == PIPELINE_CONF_FILE:
         return {
             bot_id: {
                 "source-queue": src_name,
                 "destination-queues": dst_names
             },
         }
     elif conf_file == RUNTIME_CONF_FILE:
         conf = BOT_CONFIG.copy()
         conf.update(sysconfig)
         return {
             bot_id: {
                 'description':
                 'Instance of a bot for automated unit tests.',
                 'group': group,
                 'module': module,
                 'name': 'Test Bot',
                 'parameters': conf,
             }
         }
     elif conf_file.startswith(CONFIG_DIR):
         confname = os.path.join('etc/', os.path.split(conf_file)[-1])
         fname = pkg_resources.resource_filename('intelmq', confname)
         with open(fname, 'rt') as fpconfig:
             return json.load(fpconfig)
     else:
         return utils.load_configuration(conf_file)
示例#18
0
    def load_pipeline_configuration(self):
        config = utils.load_configuration(PIPELINE_CONF_FILE)
            

        self.logger.debug("Pipeline configuration: loading '%s' section" \
                          " from '%s' file" % (self.bot_id, PIPELINE_CONF_FILE))

        self.source_queues = None
        self.destination_queues = None
        
        if self.bot_id in config.keys():
        
            if 'source-queue' in config[self.bot_id].keys():
                self.source_queues = config[self.bot_id]['source-queue']
                self.logger.debug("Pipeline configuration: parameter " \
                      "'source-queue' loaded with the value '%s'" % self.source_queues)
            
            if 'destination-queues' in config[self.bot_id].keys():

                self.destination_queues = config[self.bot_id]['destination-queues']
                self.logger.debug("Pipeline configuration: parameter" \
                                  "'destination-queues' loaded with the value" \
                                  " '%s'" % ", ".join(self.destination_queues)) 

        else:
            self.logger.error("Pipeline configuration: failed to load configuration")
            self.stop()
示例#19
0
    def load_system_configuration(self):
        setattr(self.parameters, 'logging_path' , DEFAULT_LOGGING_PATH)
        setattr(self.parameters, 'logging_level' , DEFAULT_LOGGING_LEVEL)

        config = utils.load_configuration(SYSTEM_CONF_FILE) 
        for option, value in config.iteritems():
            setattr(self.parameters, option, value)
示例#20
0
文件: message.py 项目: stdp95/intelmq
    def __init__(self, message=(), auto=False, harmonization=None):
        try:
            classname = message['__type'].lower()
            del message['__type']
        except (KeyError, TypeError):
            classname = self.__class__.__name__.lower()

        if harmonization is None:
            harmonization = utils.load_configuration(HARMONIZATION_CONF_FILE)
        try:
            self.harmonization_config = harmonization[classname]
        except KeyError:
            raise exceptions.InvalidArgument('__type',
                                             got=classname,
                                             expected=VALID_MESSSAGE_TYPES,
                                             docs=HARMONIZATION_CONF_FILE)

        super(Message, self).__init__()
        if isinstance(message, dict):
            iterable = message.items()
        elif isinstance(message, tuple):
            iterable = message
        for key, value in iterable:
            if not self.add(key, value, sanitize=False, raise_failure=False):
                self.add(key, value, sanitize=True)
示例#21
0
 def set_bot(cls):
     cls.bot_reference = SquelcherExpertBot
     cls.default_input_message = INPUT1
     try:
         cls.sysconfig = (utils.load_configuration(RUNTIME_CONF_FILE)
                          ['Expert']['Squelcher'])
     except:
         cls.sysconfig = {
             "configuration_path": "/opt/intelmq/etc/"
             "squelcher.conf",
             "host": "localhost",
             "port": 5432,
             "database": "intelmq",
             "user": "******",
             "password": "******",
             "sslmode": "require",
             "table": "tests",
         }
     cls.con = psycopg2.connect(
         database=cls.sysconfig['database'],
         user=cls.sysconfig['user'],
         password=cls.sysconfig['password'],
         host=cls.sysconfig['host'],
         port=cls.sysconfig['port'],
         sslmode=cls.sysconfig['sslmode'],
     )
     cls.con.autocommit = True
     cls.cur = cls.con.cursor()
     cls.cur.execute("TRUNCATE TABLE {}".format(cls.sysconfig['table']))
     global INSERT_QUERY
     INSERT_QUERY = INSERT_QUERY.format(table=cls.sysconfig['table'])
示例#22
0
文件: expert.py 项目: certbe/intelmq
    def init(self):
        self.config = load_configuration(self.parameters.configuration_path)

        self.logger.debug("Connecting to PostgreSQL.")
        try:
            if hasattr(self.parameters, 'connect_timeout'):
                connect_timeout = self.parameters.connect_timeout
            else:
                connect_timeout = 5

            self.con = psycopg2.connect(
                database=self.parameters.database,
                user=self.parameters.user,
                password=self.parameters.password,
                host=self.parameters.host,
                port=self.parameters.port,
                sslmode=self.parameters.sslmode,
                connect_timeout=connect_timeout,
            )
            self.cur = self.con.cursor()
            global SELECT_QUERY
            SELECT_QUERY = SELECT_QUERY.format(table=self.parameters.table)
        except:
            self.logger.exception('Failed to connect to database.')
            self.stop()
        self.logger.info("Connected to PostgreSQL.")
示例#23
0
 def load_system_configuration(self):
     if os.path.exists(SYSTEM_CONF_FILE):
         try:
             config = utils.load_configuration(SYSTEM_CONF_FILE)
         except ValueError as exc:  # pragma: no cover
             self.abort('Error loading %r: %s' % (SYSTEM_CONF_FILE, exc))
         for option, value in config.items():
             setattr(self.parameters, option, value)
示例#24
0
 def load_system_configuration(self):
     if os.path.exists(SYSTEM_CONF_FILE):
         try:
             config = utils.load_configuration(SYSTEM_CONF_FILE)
         except ValueError as exc:
             exit('Invalid syntax in %r: %s' % (SYSTEM_CONF_FILE, exc))
         for option, value in config.items():
             setattr(self.parameters, option, value)
示例#25
0
 def load_defaults_configuration(self):
     # Load defaults configuration
     try:
         config = utils.load_configuration(DEFAULTS_CONF_FILE)
     except ValueError as exc:  # pragma: no cover
         self.abort('Error loading %r: %s' % (DEFAULTS_CONF_FILE, exc))
     for option, value in config.items():
         setattr(self.parameters, option, value)
示例#26
0
def new_get_runtime() -> dict:
    runtime_conf = utils.load_configuration(pkg_resources.resource_filename('intelmq', 'etc/runtime.yaml'))
    if 'global' not in runtime_conf:
        runtime_conf['global'] = {}
    runtime_conf['global']['http_proxy'] = 'http://localhost:8080'
    runtime_conf['global']['https_proxy'] = 'http://localhost:8080'
    runtime_conf['cymru-whois-expert']['parameters']['http_proxy'] = 'http://localhost:8081'
    return runtime_conf
示例#27
0
 def load_defaults_configuration(self):
     # Load defaults configuration
     try:
         config = utils.load_configuration(DEFAULTS_CONF_FILE)
     except ValueError as exc:  # pragma: no cover
         self.abort('Error loading %r: %s' % (DEFAULTS_CONF_FILE, exc))
     for option, value in config.items():
         setattr(self.parameters, option, value)
示例#28
0
 def load_defaults_configuration(self):
     # Load defaults configuration section
     try:
         config = utils.load_configuration(DEFAULTS_CONF_FILE)
     except ValueError as exc:
         exit('Invalid syntax in %r: %s' % (DEFAULTS_CONF_FILE, exc))
     for option, value in config.items():
         setattr(self.parameters, option, value)
示例#29
0
 def load_system_configuration(self):
     if os.path.exists(SYSTEM_CONF_FILE):
         try:
             config = utils.load_configuration(SYSTEM_CONF_FILE)
         except ValueError as exc:
             exit('Invalid syntax in %r: %s' % (SYSTEM_CONF_FILE, exc))
         for option, value in config.items():
             setattr(self.parameters, option, value)
示例#30
0
 def load_defaults_configuration(self):
     # Load defaults configuration section
     try:
         config = utils.load_configuration(DEFAULTS_CONF_FILE)
     except ValueError as exc:
         exit('Invalid syntax in %r: %s' % (DEFAULTS_CONF_FILE, exc))
     for option, value in config.items():
         setattr(self.parameters, option, value)
示例#31
0
文件: bot.py 项目: PoeBlu/intelmq
    def __load_runtime_configuration(self):
        self.logger.debug("Loading runtime configuration.")
        config = utils.load_configuration(RUNTIME_CONF_FILE)

        if self.__bot_id in list(config.keys()):
            for option, value in config[self.__bot_id].items():
                setattr(self.parameters, option, value)
                self.logger.debug("Runtime configuration: parameter {!r} "
                                  "loaded with value {!r}.".format(option, value))
示例#32
0
    def __init__(self, bot_id):
        super(OtherCsvParserBot, self).__init__(bot_id=bot_id)
        self._config = utils.load_configuration(CSV_PARSER_CONF_FILE)
        self._config = self._config[self.parameters.othername]
        self.csv_fieldnames = self._config['sequence']
        self.fixed_field = self._config['fixed_field']

        if self._config.get('ignore_lines_starting'):
            self.ignore_lines_starting = self._config['ignore_lines_starting']
示例#33
0
    def check(parameters):
        try:
            harmonization_config = utils.load_configuration(HARMONIZATION_CONF_FILE)
            SieveExpertBot.harmonization = harmonization_config['event']

            metamodel = SieveExpertBot.init_metamodel()
            SieveExpertBot.read_sieve_file(parameters['file'], metamodel)
        except Exception:
            return [['error', 'Validation of Sieve file failed with the following traceback: %r' % traceback.format_exc()]]
示例#34
0
文件: expert.py 项目: CZ-NIC/intelmq
    def init(self):
        self.config = load_configuration(self.parameters.configuration_path)
        if type(self.config) is dict:
            self.config = convert_config(self.config)

        if getattr(self.parameters, 'case_sensitive', True):
            self.re_kwargs = {}
        else:
            self.re_kwargs = {'flags': re.IGNORECASE}
示例#35
0
    def init(self):
        self.config = load_configuration(self.parameters.configuration_path)
        if type(self.config) is dict:
            self.config = convert_config(self.config)

        if getattr(self.parameters, 'case_sensitive', True):
            self.re_kwargs = {}
        else:
            self.re_kwargs = {'flags': re.IGNORECASE}
示例#36
0
文件: bot.py 项目: kralca/intelmq
    def __load_runtime_configuration(self):
        self.logger.debug("Loading runtime configuration.")
        config = utils.load_configuration(RUNTIME_CONF_FILE)

        if self.__bot_id in list(config.keys()):
            for option, value in config[self.__bot_id].items():
                setattr(self.parameters, option, value)
                self.logger.debug("Runtime configuration: parameter {!r} "
                                  "loaded with value {!r}.".format(option, value))
示例#37
0
文件: bot.py 项目: cvlli/intelmq
    def load_runtime_configuration(self):
        self.logger.debug("Loading runtime configuration")
        config = utils.load_configuration(RUNTIME_CONF_FILE)

        if self.bot_id in list(config.keys()):
            for option, value in config[self.bot_id].items():
                setattr(self.parameters, option, value)
                self.logger.debug("Runtime configuration: parameter '%s' "
                                  "loaded with value '%s'." % (option, value))
示例#38
0
    def check(parameters):
        try:
            harmonization_config = utils.load_configuration(HARMONIZATION_CONF_FILE)
            SieveExpertBot.harmonization = harmonization_config['event']

            metamodel = SieveExpertBot.init_metamodel()
            SieveExpertBot.read_sieve_file(parameters['file'], metamodel)
        except Exception as e:
            return [['error', 'Validation of Sieve file failed with the following traceback: %r' % traceback.format_exc()]]
示例#39
0
文件: bot.py 项目: slashtolau/intelmq
    def load_runtime_configuration(self):
        self.logger.debug("Loading runtime configuration")
        config = utils.load_configuration(RUNTIME_CONF_FILE)

        if self.bot_id in list(config.keys()):
            for option, value in config[self.bot_id].items():
                setattr(self.parameters, option, value)
                self.logger.debug("Runtime configuration: parameter '%s' "
                                  "loaded with value '%s'." % (option, value))
示例#40
0
    def __load_harmonization_configuration(self):
        self.logger.debug("Loading Harmonization configuration.")
        config = utils.load_configuration(HARMONIZATION_CONF_FILE)

        for message_types in config.keys():
            for key in config[message_types].keys():
                for _key in config.keys():
                    if _key.startswith("%s." % key):
                        raise exceptions.ConfigurationError("harmonization", "Key %s is not valid." % _key)
示例#41
0
    def __load_system_configuration(self):
        self.__log_buffer.append(("debug", "Loading system configuration"))
        config = utils.load_configuration(SYSTEM_CONF_FILE)

        for option, value in config.items():
            setattr(self.parameters, option, value)
            self.__log_buffer.append(
                ("debug", "System configuration: parameter {!r} " "loaded  with value {!r}.".format(option, value))
            )
示例#42
0
文件: bot.py 项目: slashtolau/intelmq
    def load_system_configuration(self):
        self.log_buffer.append(('debug', "Loading system configuration"))
        config = utils.load_configuration(SYSTEM_CONF_FILE)

        for option, value in config.items():
            setattr(self.parameters, option, value)
            self.log_buffer.append(
                ('debug', "System configuration: parameter '{}' "
                 "loaded  with value '{}'.".format(option, value)))
示例#43
0
文件: bot.py 项目: slashtolau/intelmq
    def load_harmonization_configuration(self):
        self.logger.debug("Loading Harmonization configuration.")
        config = utils.load_configuration(HARMONIZATION_CONF_FILE)

        for message_types in config.keys():
            for key in config[message_types].keys():
                for _key in config.keys():
                    if _key.startswith("%s." % key):
                        raise exceptions.ConfigurationError(
                            'harmonization', "Key %s is not valid." % _key)
示例#44
0
文件: bot.py 项目: haam3r/intelmq
    def __load_defaults_configuration(self):
        self.__log_buffer.append(('debug', "Loading defaults configuration from %r."
                                  "" % DEFAULTS_CONF_FILE))
        config = utils.load_configuration(DEFAULTS_CONF_FILE)

        setattr(self.parameters, 'logging_path', DEFAULT_LOGGING_PATH)

        for option, value in config.items():
            setattr(self.parameters, option, value)
            self.__log_configuration_parameter("defaults", option, value)
示例#45
0
文件: bot.py 项目: TW-NCERT/intelmq
    def __load_defaults_configuration(self):
        self.__log_buffer.append(('debug', "Loading defaults configuration from %r."
                                  "" % DEFAULTS_CONF_FILE))
        config = utils.load_configuration(DEFAULTS_CONF_FILE)

        setattr(self.parameters, 'logging_path', DEFAULT_LOGGING_PATH)

        for option, value in config.items():
            setattr(self.parameters, option, value)
            self.__log_configuration_parameter("defaults", option, value)
示例#46
0
    def load_harmonization_configuration(self):
        harmonization_config = utils.load_configuration(HARMONIZATION_CONF_FILE)
        self.logger.debug("Harmonization configuration: loading all '%s' file" %
                          HARMONIZATION_CONF_FILE )

        for message_types in harmonization_config.keys():
            for key in harmonization_config[message_types].keys():
                for _key in harmonization_config.keys():
                    if _key.startswith("%s." % key):    # FIXME: write in devguide the rules for the keys names
                        raise exceptions.ConfigurationError(
                            HARMONIZATION_CONF_FILE, "key %s is not valid" % _key)
示例#47
0
 def load_defaults_configuration(self):
     # Load defaults configuration
     try:
         config = utils.load_configuration(DEFAULTS_CONF_FILE)
     except ValueError as exc:  # pragma: no cover
         msg = 'Error loading %r: %s' % (DEFAULTS_CONF_FILE, exc)
         if self.interactive:
             exit(msg)
         else:
             raise ValueError(msg)
     for option, value in config.items():
         setattr(self.parameters, option, value)
示例#48
0
 def load_system_configuration(self):
     if os.path.exists(SYSTEM_CONF_FILE):
         try:
             config = utils.load_configuration(SYSTEM_CONF_FILE)
         except ValueError as exc:  # pragma: no cover
             msg = 'Error loading %r: %s' % (SYSTEM_CONF_FILE, exc)
             if self.interactive:
                 exit(msg)
             else:
                 raise ValueError(msg)
         for option, value in config.items():
             setattr(self.parameters, option, value)
示例#49
0
文件: bot.py 项目: TW-NCERT/intelmq
    def __load_system_configuration(self):
        if os.path.exists(SYSTEM_CONF_FILE):
            self.__log_buffer.append(('warning', "system.conf is deprecated "
                                      "and will be removed in 1.0. "
                                      "Use defaults.conf instead!"))
            self.__log_buffer.append(('debug', "Loading system configuration from %r."
                                      "" % SYSTEM_CONF_FILE))
            config = utils.load_configuration(SYSTEM_CONF_FILE)

            for option, value in config.items():
                setattr(self.parameters, option, value)
                self.__log_configuration_parameter("system", option, value)
示例#50
0
文件: bot.py 项目: haam3r/intelmq
    def __load_system_configuration(self):
        if os.path.exists(SYSTEM_CONF_FILE):
            self.__log_buffer.append(('warning', "system.conf is deprecated "
                                      "and will be removed in 1.0. "
                                      "Use defaults.conf instead!"))
            self.__log_buffer.append(('debug', "Loading system configuration from %r."
                                      "" % SYSTEM_CONF_FILE))
            config = utils.load_configuration(SYSTEM_CONF_FILE)

            for option, value in config.items():
                setattr(self.parameters, option, value)
                self.__log_configuration_parameter("system", option, value)
示例#51
0
 def load_system_configuration(self):
     if os.path.exists(SYSTEM_CONF_FILE):
         try:
             config = utils.load_configuration(SYSTEM_CONF_FILE)
         except ValueError as exc:  # pragma: no cover
             msg = 'Error loading %r: %s' % (SYSTEM_CONF_FILE, exc)
             if self.interactive:
                 exit(msg)
             else:
                 raise ValueError(msg)
         for option, value in config.items():
             setattr(self.parameters, option, value)
示例#52
0
文件: bot.py 项目: PoeBlu/intelmq
    def __load_system_configuration(self):
        if os.path.exists(SYSTEM_CONF_FILE):
            self.__log_buffer.append(('warning', "system.conf is deprecated and will be"
                                      "removed in 1.0. Use defaults.conf instead!"))
            self.__log_buffer.append(('debug', "Loading system configuration."))
            config = utils.load_configuration(SYSTEM_CONF_FILE)

            for option, value in config.items():
                setattr(self.parameters, option, value)
                self.__log_buffer.append(('debug',
                                          "System configuration: parameter {!r} "
                                          "loaded  with value {!r}.".format(option, value)))
示例#53
0
 def load_defaults_configuration(self):
     # Load defaults configuration
     try:
         config = utils.load_configuration(DEFAULTS_CONF_FILE)
     except ValueError as exc:  # pragma: no cover
         msg = 'Error loading %r: %s' % (DEFAULTS_CONF_FILE, exc)
         if self.interactive:
             exit(msg)
         else:
             raise ValueError(msg)
     for option, value in config.items():
         setattr(self.parameters, option, value)
示例#54
0
文件: bot.py 项目: TW-NCERT/intelmq
    def __load_runtime_configuration(self):
        self.logger.debug("Loading runtime configuration from %r." % RUNTIME_CONF_FILE)
        config = utils.load_configuration(RUNTIME_CONF_FILE)

        if self.__bot_id in list(config.keys()):
            params = config[self.__bot_id]
            if 'parameters' in params:
                params = params['parameters']
            else:
                self.logger.warning('Old runtime configuration format found.')
            for option, value in params.items():
                setattr(self.parameters, option, value)
                self.__log_configuration_parameter("runtime", option, value)
示例#55
0
文件: bot.py 项目: sch3m4/intelmq
    def load_defaults_configuration(self):
        self.log_buffer.append(('debug', "Loading defaults configuration."))
        config = utils.load_configuration(DEFAULTS_CONF_FILE)

        setattr(self.parameters, 'logging_path', DEFAULT_LOGGING_PATH)
        setattr(self.parameters, 'logging_level', DEFAULT_LOGGING_LEVEL)

        for option, value in config.items():
            setattr(self.parameters, option, value)
            self.log_buffer.append(('debug',
                                    "Defaults configuration: parameter '{}' "
                                    "loaded  with value '{}'.".format(option,
                                                                      value)))
示例#56
0
    def load_defaults_configuration(self):

        # Load defaults configuration section

        config = utils.load_configuration(DEFAULTS_CONF_FILE)

        self.logger.debug("Defaults configuration" \
                          " from '%s' file" % DEFAULTS_CONF_FILE)

        for option, value in config.iteritems():
            setattr(self.parameters, option, value)
            self.logger.debug("Defaults configuration: parameter '%s' " \
                              "loaded with value '%s'" % (option, value))
示例#57
0
    def __load_defaults_configuration(self):
        self.__log_buffer.append(("debug", "Loading defaults configuration."))
        config = utils.load_configuration(DEFAULTS_CONF_FILE)

        setattr(self.parameters, "testing", False)
        setattr(self.parameters, "logging_path", DEFAULT_LOGGING_PATH)
        setattr(self.parameters, "logging_level", DEFAULT_LOGGING_LEVEL)

        for option, value in config.items():
            setattr(self.parameters, option, value)
            self.__log_buffer.append(
                ("debug", "Defaults configuration: parameter {!r} " "loaded  with value {!r}.".format(option, value))
            )
示例#58
0
    def load_runtime_configuration(self):

        # Load bot runtime configuration section

        config = utils.load_configuration(RUNTIME_CONF_FILE)
        
        self.logger.debug("Runtime configuration: loading '%s' section from" \
                          " '%s' file" % (self.bot_id, RUNTIME_CONF_FILE))

        if self.bot_id in config.keys():
            for option, value in config[self.bot_id].iteritems():
                setattr(self.parameters, option, value)
                self.logger.debug("Runtime configuration: parameter '%s' " \
                                  "loaded with value '%s'" % (option, value)) 
示例#59
0
文件: test.py 项目: Dognaedis/intelmq
 def mocked(conf_file):
     if conf_file == PIPELINE_CONF_FILE:
         return {bot_id: {"source-queue": src_name,
                          "destination-queues": dst_names},
                 }
     elif conf_file == RUNTIME_CONF_FILE:
         conf = BOT_CONFIG.copy()
         conf.update(sysconfig)
         return {bot_id: {'parameters': conf}}
     elif conf_file.startswith(CONFIG_DIR):
         confname = os.path.join('etc/', os.path.split(conf_file)[-1])
         fname = pkg_resources.resource_filename('intelmq',
                                                 confname)
         with open(fname, 'rt') as fpconfig:
             return json.load(fpconfig)
     else:
         return utils.load_configuration(conf_file)
示例#60
0
文件: bot.py 项目: 0xffca/intelmq
    def __load_runtime_configuration(self):
        self.logger.debug("Loading runtime configuration from %r.", RUNTIME_CONF_FILE)
        config = utils.load_configuration(RUNTIME_CONF_FILE)
        reinitialize_logging = False

        if self.__bot_id in list(config.keys()):
            params = config[self.__bot_id]
            self.run_mode = params.get('run_mode', 'stream')
            for option, value in params['parameters'].items():
                setattr(self.parameters, option, value)
                self.__log_configuration_parameter("runtime", option, value)
                if option.startswith('logging_'):
                    reinitialize_logging = True

        if reinitialize_logging:
            self.logger.handlers = []  # remove all existing handlers
            self.__init_logger()