def test_issue_7754(self):
old_cwd = os.getcwd()
config_dir = os.path.join(integration.TMP, 'issue-7754')
if not os.path.isdir(config_dir):
os.makedirs(config_dir)
os.chdir(config_dir)
config_file_name = 'master'
pid_path = os.path.join(config_dir, '{0}.pid'.format(config_file_name))
shutil.copyfile(
self.get_config_file_path('minion'),
os.path.join(config_dir, 'minion')
)
config = salt.config.syndic_config(
master_config_path=self.get_config_file_path('syndic'),
minion_config_path=self.get_config_file_path('minion')
)
config.pop('include')
config['root_dir'] = config_dir
config['log_file'] = 'file:///dev/log/LOG_LOCAL3'
config['ret_port'] = int(config['ret_port']) + 10
config['publish_port'] = int(config['publish_port']) + 10
open(os.path.join(config_dir, config_file_name), 'w').write(
yaml.dump(config, default_flow_style=False)
)
self.run_script(
self._call_binary_,
'--config-dir {0} --pid-file {1} -l debug'.format(
config_dir,
pid_path
),
timeout=5,
catch_stderr=True
)
# Now kill it if still running
if os.path.exists(pid_path):
try:
os.kill(int(open(pid_path).read()), signal.SIGKILL)
except OSError:
pass
try:
self.assertFalse(os.path.isdir(os.path.join(config_dir, 'file:')))
finally:
os.chdir(old_cwd)
if os.path.isdir(config_dir):
shutil.rmtree(config_dir)
def test_issue_7754(self):
old_cwd = os.getcwd()
config_dir = os.path.join(integration.TMP, 'issue-7754')
if not os.path.isdir(config_dir):
os.makedirs(config_dir)
os.chdir(config_dir)
config_file_name = 'master'
pid_path = os.path.join(config_dir, '{0}.pid'.format(config_file_name))
shutil.copyfile(self.get_config_file_path('minion'),
os.path.join(config_dir, 'minion'))
config = salt.config.syndic_config(
master_config_path=self.get_config_file_path('syndic'),
minion_config_path=self.get_config_file_path('minion'))
config.pop('include')
config['root_dir'] = config_dir
config['log_file'] = 'file:///dev/log/LOG_LOCAL3'
config['ret_port'] = int(config['ret_port']) + 10
config['publish_port'] = int(config['publish_port']) + 10
open(os.path.join(config_dir, config_file_name),
'w').write(yaml.dump(config, default_flow_style=False))
self.run_script(self._call_binary_,
'--config-dir {0} --pid-file {1} -l debug'.format(
config_dir, pid_path),
timeout=5,
catch_stderr=True)
# Now kill it if still running
if os.path.exists(pid_path):
try:
os.kill(int(open(pid_path).read()), signal.SIGKILL)
except OSError:
pass
try:
self.assertFalse(os.path.isdir(os.path.join(config_dir, 'file:')))
finally:
os.chdir(old_cwd)
if os.path.isdir(config_dir):
shutil.rmtree(config_dir)
def write_config(cls, config):
cls.verify_config(config)
config_file = config.pop("conf_file")
log.debug(
"Writing to configuration file %s. Configuration:\n%s",
config_file,
pprint.pformat(config),
)
# Write down the computed configuration into the config file
with salt.utils.files.fopen(config_file, "w") as wfh:
salt.utils.yaml.safe_dump(config, wfh, default_flow_style=False)
return salt.config.cloud_config(config_file)
def apply_cloud_providers_config(overrides, defaults=None):
'''
Apply the loaded cloud providers configuration.
'''
if defaults is None:
defaults = PROVIDER_CONFIG_DEFAULTS
config = defaults.copy()
if overrides:
config.update(overrides)
# Is the user still using the old format in the new configuration file?!
for name, settings in config.copy().items():
if '.' in name:
log.warn(
'Please switch to the new providers configuration syntax'
)
# Let's help out and migrate the data
config = old_to_new(config)
# old_to_new will migrate the old data into the 'providers' key of
# the config dictionary. Let's map it correctly
for prov_name, prov_settings in config.pop('providers').items():
config[prov_name] = prov_settings
break
providers = {}
for key, val in config.items():
if key in ('conf_file', 'include', 'default_include'):
continue
if not isinstance(val, (list, tuple)):
val = [val]
else:
# Need to check for duplicate cloud provider entries per "alias" or
# we won't be able to properly reference it.
handled_providers = set()
for details in val:
if 'provider' not in details:
if 'extends' not in details:
log.error(
'Please check your cloud providers configuration. '
'There\'s no \'provider\' nor \'extends\' '
'definition. So it\'s pretty useless.'
)
continue
if details['provider'] in handled_providers:
log.error(
'You can only have one entry per cloud provider. For '
'example, if you have a cloud provider configuration '
'section named, \'production\', you can only have a '
'single entry for EC2, Joyent, Openstack, and so '
'forth.'
)
raise salt.cloud.exceptions.SaltCloudConfigError(
'The cloud provider alias {0!r} has multiple entries '
'for the {1[provider]!r} driver.'.format(key, details)
)
handled_providers.add(details['provider'])
for entry in val:
if 'provider' not in entry:
entry['provider'] = '-only-extendable-'
if key not in providers:
providers[key] = {}
provider = entry['provider']
if provider in providers[key] and provider == '-only-extendable-':
raise salt.cloud.exceptions.SaltCloudConfigError(
'There\'s multiple entries under {0!r} which do not set '
'a provider setting. This is most likely just a holder '
'for data to be extended from, however, there can be '
'only one entry which does not define it\'s \'provider\' '
'setting.'.format(key)
)
elif provider not in providers[key]:
providers[key][provider] = entry
# Is any provider extending data!?
while True:
keep_looping = False
for provider_alias, entries in providers.copy().items():
for driver, details in entries.iteritems():
# Set a holder for the defined profiles
providers[provider_alias][driver]['profiles'] = {}
if 'extends' not in details:
continue
extends = details.pop('extends')
if ':' in extends:
alias, provider = extends.split(':')
if alias not in providers:
raise salt.cloud.exceptions.SaltCloudConfigError(
'The {0!r} cloud provider entry in {1!r} is '
'trying to extend data from {2!r} though {2!r} '
'is not defined in the salt cloud providers '
'loaded data.'.format(
details['provider'],
provider_alias,
alias
)
)
if provider not in providers.get(alias):
raise salt.cloud.exceptions.SaltCloudConfigError(
'The {0!r} cloud provider entry in {1!r} is '
'trying to extend data from \'{2}:{3}\' though '
'{3!r} is not defined in {1!r}'.format(
details['provider'],
provider_alias,
alias,
provider
)
)
details['extends'] = '{0}:{1}'.format(alias, provider)
elif providers.get(extends) and len(providers[extends]) > 1:
raise salt.cloud.exceptions.SaltCloudConfigError(
'The {0!r} cloud provider entry in {1!r} is trying '
'to extend from {2!r} which has multiple entries '
'and no provider is being specified. Not '
'extending!'.format(
details['provider'], provider_alias, extends
)
)
elif extends not in providers:
raise salt.cloud.exceptions.SaltCloudConfigError(
'The {0!r} cloud provider entry in {1!r} is trying '
'to extend data from {2!r} though {2!r} is not '
'defined in the salt cloud providers loaded '
'data.'.format(
details['provider'], provider_alias, extends
)
)
else:
provider = providers.get(extends)
if driver in providers.get(extends):
details['extends'] = '{0}:{1}'.format(extends, driver)
elif '-only-extendable-' in providers.get(extends):
details['extends'] = '{0}:{1}'.format(
extends, '-only-extendable-'
)
else:
# We're still not aware of what we're trying to extend
# from. Let's try on next iteration
details['extends'] = extends
keep_looping = True
if not keep_looping:
break
while True:
# Merge provided extends
keep_looping = False
for alias, entries in providers.copy().items():
for driver, details in entries.iteritems():
if 'extends' not in details:
# Extends resolved or non existing, continue!
continue
if 'extends' in details['extends']:
# Since there's a nested extends, resolve this one in the
# next iteration
keep_looping = True
continue
# Let's get a reference to what we're supposed to extend
extends = details.pop('extends')
# Split the setting in (alias, driver)
ext_alias, ext_driver = extends.split(':')
# Grab a copy of what should be extended
extended = providers.get(ext_alias).get(ext_driver).copy()
# Merge the data to extend with the details
extended.update(details)
# Update the providers dictionary with the merged data
providers[alias][driver] = extended
if not keep_looping:
break
# Now clean up any providers entry that was just used to be a data tree to
# extend from
for provider_alias, entries in providers.copy().items():
for driver, details in entries.copy().iteritems():
if driver != '-only-extendable-':
continue
log.info(
'There\'s at least one cloud driver details under the {0!r} '
'cloud provider alias which does not have the required '
'\'provider\' setting. Was probably just used as a holder '
'for additional data. Removing it from the available '
'providers listing'.format(
provider_alias
)
)
providers[provider_alias].pop(driver)
if not providers[provider_alias]:
providers.pop(provider_alias)
return providers
def apply_cloud_providers_config(overrides, defaults=None):
'''
Apply the loaded cloud providers configuration.
'''
if defaults is None:
defaults = PROVIDER_CONFIG_DEFAULTS
config = defaults.copy()
if overrides:
config.update(overrides)
# Is the user still using the old format in the new configuration file?!
for name, settings in config.copy().items():
if '.' in name:
log.warn('Please switch to the new providers configuration syntax')
# Let's help out and migrate the data
config = old_to_new(config)
# old_to_new will migrate the old data into the 'providers' key of
# the config dictionary. Let's map it correctly
for prov_name, prov_settings in config.pop('providers').items():
config[prov_name] = prov_settings
break
providers = {}
for key, val in config.items():
if key in ('conf_file', 'include', 'default_include'):
continue
if not isinstance(val, (list, tuple)):
val = [val]
else:
# Need to check for duplicate cloud provider entries per "alias" or
# we won't be able to properly reference it.
handled_providers = set()
for details in val:
if 'provider' not in details:
if 'extends' not in details:
log.error(
'Please check your cloud providers configuration. '
'There\'s no \'provider\' nor \'extends\' '
'definition. So it\'s pretty useless.')
continue
if details['provider'] in handled_providers:
log.error(
'You can only have one entry per cloud provider. For '
'example, if you have a cloud provider configuration '
'section named, \'production\', you can only have a '
'single entry for EC2, Joyent, Openstack, and so '
'forth.')
raise salt.cloud.exceptions.SaltCloudConfigError(
'The cloud provider alias {0!r} has multiple entries '
'for the {1[provider]!r} driver.'.format(key, details))
handled_providers.add(details['provider'])
for entry in val:
if 'provider' not in entry:
entry['provider'] = '-only-extendable-'
if key not in providers:
providers[key] = {}
provider = entry['provider']
if provider in providers[key] and provider == '-only-extendable-':
raise salt.cloud.exceptions.SaltCloudConfigError(
'There\'s multiple entries under {0!r} which do not set '
'a provider setting. This is most likely just a holder '
'for data to be extended from, however, there can be '
'only one entry which does not define it\'s \'provider\' '
'setting.'.format(key))
elif provider not in providers[key]:
providers[key][provider] = entry
# Is any provider extending data!?
while True:
keep_looping = False
for provider_alias, entries in providers.copy().items():
for driver, details in entries.iteritems():
# Set a holder for the defined profiles
providers[provider_alias][driver]['profiles'] = {}
if 'extends' not in details:
continue
extends = details.pop('extends')
if ':' in extends:
alias, provider = extends.split(':')
if alias not in providers:
raise salt.cloud.exceptions.SaltCloudConfigError(
'The {0!r} cloud provider entry in {1!r} is '
'trying to extend data from {2!r} though {2!r} '
'is not defined in the salt cloud providers '
'loaded data.'.format(details['provider'],
provider_alias, alias))
if provider not in providers.get(alias):
raise salt.cloud.exceptions.SaltCloudConfigError(
'The {0!r} cloud provider entry in {1!r} is '
'trying to extend data from \'{2}:{3}\' though '
'{3!r} is not defined in {1!r}'.format(
details['provider'], provider_alias, alias,
provider))
details['extends'] = '{0}:{1}'.format(alias, provider)
elif providers.get(extends) and len(providers[extends]) > 1:
raise salt.cloud.exceptions.SaltCloudConfigError(
'The {0!r} cloud provider entry in {1!r} is trying '
'to extend from {2!r} which has multiple entries '
'and no provider is being specified. Not '
'extending!'.format(details['provider'],
provider_alias, extends))
elif extends not in providers:
raise salt.cloud.exceptions.SaltCloudConfigError(
'The {0!r} cloud provider entry in {1!r} is trying '
'to extend data from {2!r} though {2!r} is not '
'defined in the salt cloud providers loaded '
'data.'.format(details['provider'], provider_alias,
extends))
else:
provider = providers.get(extends)
if driver in providers.get(extends):
details['extends'] = '{0}:{1}'.format(extends, driver)
elif '-only-extendable-' in providers.get(extends):
details['extends'] = '{0}:{1}'.format(
extends, '-only-extendable-')
else:
# We're still not aware of what we're trying to extend
# from. Let's try on next iteration
details['extends'] = extends
keep_looping = True
if not keep_looping:
break
while True:
# Merge provided extends
keep_looping = False
for alias, entries in providers.copy().items():
for driver, details in entries.iteritems():
if 'extends' not in details:
# Extends resolved or non existing, continue!
continue
if 'extends' in details['extends']:
# Since there's a nested extends, resolve this one in the
# next iteration
keep_looping = True
continue
# Let's get a reference to what we're supposed to extend
extends = details.pop('extends')
# Split the setting in (alias, driver)
ext_alias, ext_driver = extends.split(':')
# Grab a copy of what should be extended
extended = providers.get(ext_alias).get(ext_driver).copy()
# Merge the data to extend with the details
extended.update(details)
# Update the providers dictionary with the merged data
providers[alias][driver] = extended
if not keep_looping:
break
# Now clean up any providers entry that was just used to be a data tree to
# extend from
for provider_alias, entries in providers.copy().items():
for driver, details in entries.copy().iteritems():
if driver != '-only-extendable-':
continue
log.info(
'There\'s at least one cloud driver details under the {0!r} '
'cloud provider alias which does not have the required '
'\'provider\' setting. Was probably just used as a holder '
'for additional data. Removing it from the available '
'providers listing'.format(provider_alias))
providers[provider_alias].pop(driver)
if not providers[provider_alias]:
providers.pop(provider_alias)
return providers
