def parse_config(self, *, ignore_env, config_paths=None):
"""Helper method to parse the config file from disk."""
if config_paths is None:
env_config_file = os.environ.get('DIRBS_CONFIG_FILE', None)
if env_config_file is not None:
config_paths = [env_config_file]
else:
config_paths = _DEFAULT_SEARCH_PATHS # pragma: no cover
for p in config_paths:
_logger.debug('Looking for DIRBS config file in {0}...'.format(p))
try:
cfg = yaml.safe_load(open(p))
if cfg is None:
_logger.error(
'Invalid DIRBS Config file found at {0}!'.format(p))
raise ConfigParseException(
'Invalid DIRBS Config file found at {0}'.format(p))
_logger.debug('Successfully parsed {0} as YAML...'.format(p))
return AppConfig(ignore_env=ignore_env, **cfg)
except yaml.YAMLError as ex:
_logger.error(
'Invalid DIRBS Config file found at {0}!'.format(p))
msg = str(ex)
_logger.error(str(ex))
raise ConfigParseException(msg)
except IOError:
_logger.debug('{0} did not exist, skipping...'.format(p))
continue
msg = 'Missing config file - please create a config file for DIRBS'
_logger.error(msg)
raise ConfigParseException(msg)
def __init__(self, *, ignore_env, **cond_config):
"""Constructor which parses the condition config."""
super(ConditionConfig, self).__init__(ignore_env=ignore_env, **cond_config)
self.label = self._parse_string('label', max_len=64)
# Check that condition name contains only letters, underscores and digits(0-9)
bad_symbol_error_message = 'Condition label {0} must contain only letters, underscores or digits(0-9)!'
parse_alphanum(self.label.lower(), bad_symbol_error_message)
self.grace_period = self._parse_positive_int('grace_period_days')
self.blocking = self._parse_bool('blocking')
self.sticky = self._parse_bool('sticky')
self.reason = self._parse_string('reason')
self.max_allowed_matching_ratio = self._parse_float_ratio('max_allowed_matching_ratio')
self.amnesty_eligible = self._parse_bool('amnesty_eligible')
if self.reason.find('|') != -1:
msg = 'Illegal pipe character in reason string for condition: {0}'.format(self.label)
_logger.error(msg)
raise ConfigParseException(msg)
dimensions = self.raw_config['dimensions']
if not isinstance(dimensions, list):
msg = 'Dimensions not a list type!'
_logger.error('{0}: {1}'.format(self.section_name, msg))
raise ConfigParseException(msg)
self.dimensions = [DimensionConfig(ignore_env=ignore_env, **d) for d in dimensions]
if self.amnesty_eligible and not self.blocking:
msg = 'Informational conditions cannot have amnesty_eligible flag set to True.'
_logger.error('{0}: {1}'.format(self.section_name, msg))
raise ConfigParseException(msg)
def __init__(self, **dim_config):
"""Constructor which parses the dimension config."""
if 'module' not in dim_config:
msg = 'No module specified!'
_logger.error('DimensionConfig: {0}'.format(msg))
raise ConfigParseException(msg)
self.module = dim_config['module']
super(DimensionConfig, self).__init__(**dim_config)
try:
module = self.raw_config['module']
mod = importlib.import_module('dirbs.dimensions.' + module)
except ImportError as ex:
_logger.error(str(ex))
msg = '{0}: module {1} can not be imported'.format(self.section_name, module)
_logger.error('{0}'.format(msg))
raise ConfigParseException(msg)
dim_constructor = mod.__dict__.get('dimension')
try:
params = self.raw_config['parameters']
dim_constructor(**params)
self.params = params
except Exception as e:
msg_error = "Could not create dimension \'{0}\' with supplied parameters".format(self.module)
msg = '{0}: {1}. Cause: {2}'.format(self.section_name, msg_error, str(e))
_logger.error(msg)
raise ConfigParseException(msg)
self.invert = self._parse_bool('invert')
def __init__(self, *, ignore_env, **region_config):
"""Constructor which parses the region config."""
super(RegionConfig, self).__init__(ignore_env=ignore_env, **region_config)
self.name = self._parse_string('name')
self.import_msisdn_data = self._parse_bool('import_msisdn_data')
self.import_rat_data = self._parse_bool('import_rat_data')
# Check that country codes are strings that can be converted to ints
try:
[int(x) for x in self.raw_config['country_codes']]
except ValueError:
msg = '{0}: non-numeric value for country code!'.format(self.section_name)
_logger.error(msg)
raise ConfigParseException(msg)
# Make sure we store country codes as strings
self.country_codes = [str(x) for x in self.raw_config['country_codes']]
if self.country_codes is None or len(self.country_codes) <= 0:
msg = 'Country Code must be provided for "region" section in config'
_logger.error(msg)
raise ConfigParseException(msg)
# Populate operators array
self.operators = [OperatorConfig(ignore_env=ignore_env, **o) for o in region_config.get('operators', [])]
# Check that operator_ids are unique and case-insensitive
dupl_op_id_found_error_message = 'Duplicate operator_ids {0} found in config. ' \
'Operator_ids are case insensitive!'
operator_id_list = [o.id for o in self.operators]
check_for_duplicates(operator_id_list, dupl_op_id_found_error_message)
# Parse exempted device types if present
self.exempted_device_types = [str(x) for x in self.raw_config.get('exempted_device_types', [])]
# Check the mcc_mnc pairs are unique and that no mcc-mnc can begin with another mcc-mnc
dupl_mcc_mnc_found_error_message = 'Duplicate MCC-MNC pairs {0} found in config. ' \
'MCC-MNC pairs must be unique across all operators!'
all_mncs = [p['mcc'] + p['mnc'] for o in self.operators for p in o.mcc_mnc_pairs]
check_for_duplicates(all_mncs, dupl_mcc_mnc_found_error_message)
all_mncs_set = set(all_mncs)
substring_mcc_mnc_error_message = 'MCC-MNC pair {0} found which starts with another configured MCC-MNC pair ' \
'{1}. MCC-MNC pairs must be disjoint from each other (not be prefixed by ' \
'another MCC-MNC)!'
for mcc_mnc in all_mncs_set:
mcc_mncs_to_check = all_mncs_set.copy()
mcc_mncs_to_check.remove(mcc_mnc)
for other_mcc_mnc in mcc_mncs_to_check:
if mcc_mnc.startswith(other_mcc_mnc):
err_msg = substring_mcc_mnc_error_message.format(mcc_mnc, other_mcc_mnc)
_logger.error(err_msg)
raise ConfigParseException(err_msg)
def __init__(self, **operator_config):
"""Constructor which parses the operator config."""
super(OperatorConfig, self).__init__(**operator_config)
self.id = self._parse_string('id', max_len=16)
if self.id != self.id.lower():
_logger.warning('operator_id: {0} has been changed to '
'lower case: {1}'.format(self.id, self.id.lower()))
self.id = self.id.lower()
# Check that operator_ids contains only letters, underscores and digits(0-9)
bad_symbol_error_message = 'Operator_id {0} must contain only letters, underscores or digits(0-9)!'
parse_alphanum(self.id, bad_symbol_error_message)
self.name = self._parse_string('name')
if self.id == self.COUNTRY_OPERATOR_NAME:
msg = "Invalid use of reserved operator name \'__all__\' in config!"
_logger.error(msg)
raise ConfigParseException(msg)
# Make sure mcc_mnc key is there and is a list
if 'mcc_mnc_pairs' not in operator_config or type(
operator_config['mcc_mnc_pairs']) is not list:
msg = 'Missing (or non-list) {0} in config for operator ID {1}!'.format(
'mcc_mnc_pairs', self.id)
_logger.error(msg)
raise ConfigParseException(msg)
# Validate each MCC/MNC pair
for mcc_mnc in self.raw_config['mcc_mnc_pairs']:
for key in ['mcc', 'mnc']:
try:
int(mcc_mnc[key])
except (ValueError, KeyError):
msg = 'Non-existent or non integer {0} in config for operator ID {1}!'.format(
key, self.id)
_logger.error(msg)
raise ConfigParseException(msg)
# Make sure we stringify mcc and mnc values in case they were interpreted as ints by YAML parser
self.mcc_mnc_pairs = \
[{'mcc': str(x['mcc']), 'mnc': str(x['mnc'])} for x in self.raw_config['mcc_mnc_pairs']]
if self.mcc_mnc_pairs is None or len(self.mcc_mnc_pairs) <= 0:
msg = 'At least one valid MCC-MNC pair must be provided for operator ID {0}.'.format(
self.id)
_logger.error(msg)
raise ConfigParseException(msg)
def __init__(self, **kafka_config):
"""Constructor which parses the kafka config."""
super(KafkaConfig, self).__init__(**kafka_config)
self.hostname = self._parse_string('hostname')
self.port = self._parse_positive_int('port')
self.topic = self._parse_string('topic')
# protocol and checks, plain and ssl only
self.security_protocol = self._parse_string('security_protocol').upper()
if self.security_protocol not in ['SSL', 'PLAINTEXT']:
msg = 'Invalid security protocol specified, use one on [PLAIN, SSL] only'
_logger.error(msg)
raise ConfigParseException(msg)
# if security protocol is set to SSL then verify the required options
if self.security_protocol == 'SSL':
self.client_certificate = self._parse_file_path('client_certificate', ext='.pem')
self.client_key = self._parse_file_path('client_key', ext='.pem')
self.caroot_certificate = self._parse_file_path('caroot_certificate', ext='.pem')
self.skip_tls_verifications = self._parse_bool('skip_tls_verifications')
if self.skip_tls_verifications is True:
_logger.warning('TLS verifications should only be turned off in DEV Env, '
'not recommended for production environment')
# if security protocol is set to PLAIN show warning
else:
_logger.warning('Security protocol in broker config is set to PLAIN, which is not recommended '
'in production environment')
def max_db_connections(self, value):
"""Property setter for max_db_connections."""
if value < 1 or value > 32:
msg = 'max_db_connections must be at least 1 and can not be set higher than 32!'
_logger.error(msg)
raise ConfigParseException(msg)
self._max_db_connections = value
def validate_exempted_device_types(conn, config):
"""
Method to validate exempted device types specified in config.
Arguments:
conn: dirbs db connection object
config: dirbs parsed configuration object
Raises:
ConfigParseException: if device types are not valid
"""
with conn.cursor() as cursor:
logger = logging.getLogger('dirbs.config')
exempted_device_types = config.region_config.exempted_device_types
if len(exempted_device_types) > 0:
cursor.execute('SELECT DISTINCT device_type FROM gsma_data')
all_device_types = [x.device_type for x in cursor]
if len(all_device_types) == 0:
logger.warning(
'RegionConfig: Ignoring setting exempted_device_types={0} as GSMA TAC database '
'not imported or no device types found.'.format(
exempted_device_types))
else:
invalid_device_types = set(exempted_device_types) - set(
all_device_types)
if len(invalid_device_types) > 0:
msg = "RegionConfig: exempted_device_types \'{0}\' is/are not valid device type(s). " \
"The valid GSMA device types are: \'{1}\'".format(invalid_device_types, all_device_types)
logger.error(msg)
raise ConfigParseException(msg)
def max_local_cpus(self, value):
"""Property setter for max_local_cpus."""
max_cpus = max(multiprocessing.cpu_count() - 1, 1)
if value < 1 or value > max_cpus:
msg = 'max_local_cpus must be at least 1 and can not be set higher than CPUs present in the ' \
'system minus one!'
_logger.error(msg)
raise ConfigParseException(msg)
self._max_local_cpus = value
def __init__(self, **amnesty_config):
"""Constructor which parses the amnesty config."""
super(AmnestyConfig, self).__init__(**amnesty_config)
self.amnesty_enabled = self._parse_bool('amnesty_enabled')
self.evaluation_period_end_date = self._parse_date(
'evaluation_period_end_date', '%Y%m%d', 'YYYYMMDD')
self.amnesty_period_end_date = self._parse_date(
'amnesty_period_end_date', '%Y%m%d', 'YYYYMMDD')
if self.amnesty_period_end_date <= self.evaluation_period_end_date:
msg = "The \'amnesty_period_end_date\' must be greater than the \'evaluation_period_end_date\'!"
_logger.error(msg)
raise ConfigParseException(msg)
def __init__(self, **catalog_config):
"""Constructor which parses the catalog config."""
super(CatalogConfig, self).__init__(**catalog_config)
self.perform_prevalidation = self._parse_bool('perform_prevalidation')
self.prospectors = [{
'file_type': str(x['file_type']),
'paths': list(x['paths']),
'schema': x['schema_filename']
} for x in self.raw_config['prospectors']]
path_list = [path for x in self.prospectors for path in x['paths']]
if len(path_list) != len(set(path_list)):
msg = 'The paths specified in the catalog config are not globally unique!'
_logger.error(msg)
raise ConfigParseException(msg)
def __init__(self, **operator_config):
"""Constructor which parses the operator config."""
super(BrokerOperatorConfig, self).__init__(**operator_config)
self.id = self._parse_string('id', max_len=16)
if self.id != self.id.lower():
_logger.warning('operator_id: {0} has been changed to '
'lower case: {1}'.format(self.id, self.id.lower()))
self.id = self.id.lower()
# Check that operator_ids contains only letters, underscores and digits(0-9)
bad_symbol_error_message = 'Operator_id {0} must contain only letters, underscores or digits(0-9)!'
parse_alphanum(self.id, bad_symbol_error_message)
self.name = self._parse_string('name')
if self.id == self.COUNTRY_OPERATOR_NAME:
msg = "Invalid use of reserved operator name \'__all__\' in config!"
_logger.error(msg)
raise ConfigParseException(msg)
self.topic = self._parse_string('topic')
def _parse_int_or_neg_one(self, propname):
"""Helper function to parse an integer value or special value neg one and bound-check it."""
try:
self._check_for_missing_propname(propname)
parsed_val = int(self.raw_config[propname])
# -1 is a special value for _import_size_variation_absolute variable.
# If _import_size_variation_absolute is a positive integer (zero allowed), it will
# check that specified absolute rows are bigger than the existing row count.
# By setting this variable to neg one, this check will be disabled.
if parsed_val == -1:
return parsed_val
else:
# _parse_positive_int allows zero values for propname by default
return self._parse_positive_int(propname)
except ValueError:
msg = '{0}: {1} value must be a positive integer or special ' \
'value -1'.format(self.section_name, propname)
_logger.error(msg)
raise ConfigParseException(msg)