def describe_environment(self):
"""
:rtype : Environment
"""
environment = self.manager.environment_create(self.env_name())
networks = []
for name in INTERFACE_ORDER:
ip_networks = [IPNetwork(x) for x in POOLS.get(name)[0].split(',')]
new_prefix = int(POOLS.get(name)[1])
pool = self.manager.create_network_pool(
networks=ip_networks, prefix=int(new_prefix))
networks.append(self.manager.network_create(
name=name, environment=environment, pool=pool,
forward=FORWARDING.get(name), has_dhcp_server=DHCP.get(name)))
for name in self.node_roles().admin_names:
self.describe_admin_node(name, networks)
for name in self.node_roles().other_names:
self.describe_empty_node(name, networks, memory=1024)
return environment
def describe_environment(self):
"""
:rtype : Environment
"""
environment = self.manager.environment_create(self.env_name())
networks = []
for name in INTERFACE_ORDER:
ip_networks = [IPNetwork(x) for x in POOLS.get(name)[0].split(',')]
new_prefix = int(POOLS.get(name)[1])
pool = self.manager.create_network_pool(
networks=ip_networks, prefix=int(new_prefix))
networks.append(self.manager.network_create(
name=name, environment=environment, pool=pool,
forward=FORWARDING.get(name), has_dhcp_server=DHCP.get(name)))
for name in self.node_roles().admin_names:
self.describe_admin_node(name, networks)
for name in self.node_roles().other_names:
self.describe_empty_node(name, networks, memory=1024)
return environment
def check_stats_private_info(collector_remote, postgres_actions, master_uuid,
_settings):
def _contain_secret_data(data):
_has_private_data = False
# Check that stats doesn't contain private data (e.g.
# specific passwords, settings, emails)
for _private in private_data.keys():
_regex = r'(?P<key>"\S+"): (?P<value>[^:]*"{0}"[^:]*)'.format(
private_data[_private])
for _match in re.finditer(_regex, data):
logger.warning('Found private info in usage statistics using '
'pattern: {0}'.format(_regex))
logger.debug(
'Usage statistics with private data:\n {0}'.format(data))
logger.error("Usage statistics contains private info: '{type}:"
" {value}'. Part of the stats: {match}".format(
type=_private,
value=private_data[_private],
match=_match.group('key', 'value')))
_has_private_data = True
# Check that stats doesn't contain private types of data (e.g. any kind
# of passwords)
for _data_type in secret_data_types.keys():
_regex = (r'(?P<secret>"[^"]*{0}[^"]*": (\{{[^\}}]+\}}|\[[^\]+]\]|'
r'"[^"]+"))').format(secret_data_types[_data_type])
for _match in re.finditer(_regex, data, re.IGNORECASE):
logger.warning('Found private info in usage statistics using '
'pattern: {0}'.format(_regex))
logger.debug(
'Usage statistics with private data:\n {0}'.format(data))
logger.error("Usage statistics contains private info: '{type}:"
" {value}'. Part of the stats: {match}".format(
type=_data_type,
value=secret_data_types[_data_type],
match=_match.group('secret')))
_has_private_data = True
return _has_private_data
def _contain_public_ip(data, _used_networks):
_has_public_ip = False
_ip_regex = (r'\b((\d|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5])\.){3}'
r'(\d|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5])\b')
_not_public_regex = [
r'\b10(\.\d{1,3}){3}', r'\b127(\.\d{1,3}){3}',
r'\b169\.254(\.\d{1,3}){2}',
r'172\.(1[6-9]|2[0-9]|3[0-1])(\.\d{1,3}){2}',
r'192\.168(\.\d{1,3}){2}', r'2(2[4-9]|[3-5][0-9])(\.\d{1,3}){3}'
]
for _match in re.finditer(_ip_regex, data):
# If IP address isn't public and doesn't belong to defined for
# deployment pools (e.g. admin, public, storage), then skip it
if any(re.search(_r, _match.group()) for _r in _not_public_regex) \
and not any(IPAddress(_match.group()) in IPNetwork(net) for
net in _used_networks):
continue
logger.debug(
'Usage statistics with public IP(s):\n {0}'.format(data))
logger.error('Found public IP in usage statistics: "{0}"'.format(
_match.group()))
_has_public_ip = True
return _has_public_ip
private_data = {
'hostname':
_settings['HOSTNAME'],
'dns_domain':
_settings['DNS_DOMAIN'],
'dns_search':
_settings['DNS_SEARCH'],
'dns_upstream':
_settings['DNS_UPSTREAM'],
'fuel_password':
_settings['FUEL_ACCESS']['password']
if _settings['FUEL_ACCESS']['password'] != 'admin' else
'DefaultPasswordIsNotAcceptableForSearch',
'nailgun_password':
_settings['postgres']['nailgun_password'],
'keystone_password':
_settings['postgres']['keystone_password'],
'ostf_password':
_settings['postgres']['ostf_password'],
'cobbler_password':
_settings['cobbler']['password'],
'astute_password':
_settings['astute']['password'],
'mcollective_password':
_settings['mcollective']['password'],
'keystone_admin_token':
_settings['keystone']['admin_token'],
'keystone_nailgun_password':
_settings['keystone']['nailgun_password'],
'kesytone_ostf_password':
_settings['keystone']['ostf_password'],
}
secret_data_types = {
'some_password': '******',
'some_login': '******',
'some_tenant': 'tenant',
'some_token': 'token',
'some_ip': '\bip\b',
'some_netmask': 'netmask',
'some_network': 'network\b',
}
action_logs = [
l.strip() for l in postgres_actions.run_query(
'nailgun', 'select id from action_logs;').split('\n')
]
sent_stats = str(collector_remote.get_installation_info_data(master_uuid))
logger.debug('installation structure is {0}'.format(sent_stats))
used_networks = [POOLS[net_name][0] for net_name in POOLS.keys()]
has_no_private_data = True
logger.debug("Looking for private data in the installation structure, "
"that was sent to collector")
if _contain_secret_data(sent_stats) or _contain_public_ip(
sent_stats, used_networks):
has_no_private_data = False
for log_id in action_logs:
log_data = postgres_actions.run_query(
'nailgun',
"select additional_info from action_logs where id = '{0}';".format(
log_id))
logger.debug(
"Looking for private data in action log with ID={0}".format(
log_id))
if _contain_secret_data(log_data) or _contain_public_ip(
log_data, used_networks):
has_no_private_data = False
assert_true(
has_no_private_data, 'Found private data in stats, check test '
'output and logs for details.')
logger.info('Found no private data in logs')
def check_stats_private_info(collector_remote, postgres_actions,
master_uuid, _settings):
def _contain_secret_data(data):
_has_private_data = False
# Check that stats doesn't contain private data (e.g.
# specific passwords, settings, emails)
for _private in private_data.keys():
_regex = r'(?P<key>"\S+"): (?P<value>[^:]*"{0}"[^:]*)'.format(
private_data[_private])
for _match in re.finditer(_regex, data):
logger.warning('Found private info in usage statistics using '
'pattern: {0}'. format(_regex))
logger.debug('Usage statistics with private data:\n {0}'.
format(data))
logger.error("Usage statistics contains private info: '{type}:"
" {value}'. Part of the stats: {match}".format(
type=_private,
value=private_data[_private],
match=_match.group('key', 'value')))
_has_private_data = True
# Check that stats doesn't contain private types of data (e.g. any kind
# of passwords)
for _data_type in secret_data_types.keys():
_regex = (r'(?P<secret>"[^"]*{0}[^"]*": (\{{[^\}}]+\}}|\[[^\]+]\]|'
r'"[^"]+"))').format(secret_data_types[_data_type])
for _match in re.finditer(_regex, data, re.IGNORECASE):
logger.warning('Found private info in usage statistics using '
'pattern: {0}'. format(_regex))
logger.debug('Usage statistics with private data:\n {0}'.
format(data))
logger.error("Usage statistics contains private info: '{type}:"
" {value}'. Part of the stats: {match}".format(
type=_data_type,
value=secret_data_types[_data_type],
match=_match.group('secret')))
_has_private_data = True
return _has_private_data
def _contain_public_ip(data, _used_networks):
_has_puplic_ip = False
_ip_regex = (r'\b((\d|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5])\.){3}'
r'(\d|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5])\b')
_not_public_regex = [
r'\b10(\.\d{1,3}){3}',
r'\b127(\.\d{1,3}){3}',
r'\b169\.254(\.\d{1,3}){2}',
r'172\.(1[6-9]|2[0-9]|3[0-1])(\.\d{1,3}){2}',
r'192\.168(\.\d{1,3}){2}',
r'2(2[4-9]|[3-5][0-9])(\.\d{1,3}){3}'
]
for _match in re.finditer(_ip_regex, data):
# If IP address isn't public and doesn't belong to defined for
# deployment pools (e.g. admin, public, storage), then skip it
if any(re.search(_r, _match.group()) for _r in _not_public_regex) \
and not any(IPAddress(_match.group()) in IPNetwork(net) for
net in _used_networks):
continue
logger.debug('Usage statistics with piblic IP(s):\n {0}'.
format(data))
logger.error('Found public IP in usage statistics: "{0}"'.format(
_match.group()))
_has_puplic_ip = True
return _has_puplic_ip
private_data = {
'hostname': _settings['HOSTNAME'],
'dns_domain': _settings['DNS_DOMAIN'],
'dns_search': _settings['DNS_SEARCH'],
'dns_upstream': _settings['DNS_UPSTREAM'],
'fuel_password': _settings['FUEL_ACCESS']['password'] if
_settings['FUEL_ACCESS']['password'] != 'admin'
else 'DefaultPasswordIsNotAcceptableForSearch',
'nailgun_password': _settings['postgres']['nailgun_password'],
'keystone_password': _settings['postgres']['keystone_password'],
'ostf_password': _settings['postgres']['ostf_password'],
'cobbler_password': _settings['cobbler']['password'],
'astute_password': _settings['astute']['password'],
'mcollective_password': _settings['mcollective']['password'],
'keystone_admin_token': _settings['keystone']['admin_token'],
'keystone_nailgun_password': _settings['keystone']['nailgun_password'],
'kesytone_ostf_password': _settings['keystone']['ostf_password'],
}
secret_data_types = {
'some_password': '******',
'some_login': '******',
'some_tenant': 'tenant',
'some_token': 'token',
'some_ip': '\bip\b',
'some_netmask': 'netmask',
'some_network': 'network(?!_assignment)',
}
action_logs = [l.strip() for l in postgres_actions.run_query(
'nailgun', 'select id from action_logs;').split('\n')]
sent_stats = execute_query_on_collector(
collector_remote, master_uuid,
query="SELECT structure from installation_structures"
)
used_networks = [POOLS[net_name][0] for net_name in POOLS.keys()]
has_no_private_data = True
logger.debug("Looking for private data in the installation structure, "
"that was sent to collector")
if _contain_secret_data(sent_stats) or _contain_public_ip(sent_stats,
used_networks):
has_no_private_data = False
for log_id in action_logs:
log_data = postgres_actions.run_query(
'nailgun',
"select additional_info from action_logs where id = '{0}';".format(
log_id
))
logger.debug("Looking for private data in action log with ID={0}".
format(log_id))
if _contain_secret_data(log_data) or _contain_public_ip(log_data,
used_networks):
has_no_private_data = False
assert_true(has_no_private_data, 'Found private data in stats, check test '
'output and logs for details.')
logger.info('Found no private data in logs')