def test_cache_file_with_alternate_cachedir_and_relative_path(self):
'''
Ensure file is cached to correct location when an alternate cachedir is
specified and that cachedir is a relative path
'''
patched_opts = dict((x, y) for x, y in six.iteritems(self.minion_opts))
patched_opts.update(MOCKED_OPTS)
alt_cachedir = 'foo'
with patch.dict(fileclient.__opts__, patched_opts):
client = fileclient.get_file_client(fileclient.__opts__, pillar=False)
for saltenv in SALTENVS:
self.assertTrue(
client.cache_file('salt://foo.txt',
saltenv,
cachedir=alt_cachedir)
)
cache_loc = os.path.join(fileclient.__opts__['cachedir'],
alt_cachedir,
'files',
saltenv,
'foo.txt')
# Double check that the content of the cached file identifies
# it as being from the correct saltenv. The setUp function
# creates the file with the name of the saltenv mentioned in
# the file, so a simple 'in' check is sufficient here. If
# opening the file raises an exception, this is a problem, so
# we are not catching the exception and letting it be raised so
# that the test fails.
with salt.utils.fopen(cache_loc) as fp_:
content = fp_.read()
log.debug('cache_loc = %s', cache_loc)
log.debug('content = %s', content)
self.assertTrue(saltenv in content)
def test_cache_dir_with_alternate_cachedir_and_absolute_path(self):
'''
Ensure entire directory is cached to correct location when an alternate
cachedir is specified and that cachedir is an absolute path
'''
patched_opts = dict((x, y) for x, y in six.iteritems(self.minion_opts))
patched_opts.update(MOCKED_OPTS)
alt_cachedir = os.path.join(integration.TMP, 'abs_cachedir')
with patch.dict(fileclient.__opts__, patched_opts):
client = fileclient.get_file_client(fileclient.__opts__, pillar=False)
for saltenv in SALTENVS:
self.assertTrue(
client.cache_dir(
'salt://{0}'.format(SUBDIR),
saltenv,
cachedir=alt_cachedir
)
)
for subdir_file in SUBDIR_FILES:
cache_loc = os.path.join(alt_cachedir,
'files',
saltenv,
SUBDIR,
subdir_file)
# Double check that the content of the cached file
# identifies it as being from the correct saltenv. The
# setUp function creates the file with the name of the
# saltenv mentioned in the file, so a simple 'in' check is
# sufficient here. If opening the file raises an exception,
# this is a problem, so we are not catching the exception
# and letting it be raised so that the test fails.
with salt.utils.fopen(cache_loc) as fp_:
content = fp_.read()
log.debug('cache_loc = %s', cache_loc)
log.debug('content = %s', content)
self.assertTrue(subdir_file in content)
self.assertTrue(SUBDIR in content)
self.assertTrue(saltenv in content)
def render(template, saltenv='base', sls='', salt_data=True, **kwargs):
if 'pyobjects_states' not in __context__:
load_states()
# these hold the scope that our sls file will be executed with
_locals = {}
_globals = {}
# create our StateFactory objects
mod_globals = {'StateFactory': StateFactory}
for mod in __context__['pyobjects_states']:
mod_locals = {}
mod_camel = ''.join([
part.capitalize()
for part in mod.split('_')
])
valid_funcs = "','".join(
__context__['pyobjects_states'][mod].keys()
)
mod_cmd = "{0} = StateFactory('{1!s}', valid_funcs=['{2}'])".format(
mod_camel,
mod,
valid_funcs
)
if sys.version_info[0] > 2:
# in py3+ exec is a function
exec(mod_cmd, mod_globals, mod_locals)
else:
# prior to that it is a statement
exec mod_cmd in mod_globals, mod_locals
_globals[mod_camel] = mod_locals[mod_camel]
# add our include and extend functions
_globals['include'] = Registry.include
_globals['extend'] = Registry.make_extend
# add our map class
Map.__salt__ = __salt__
_globals['Map'] = Map
# add some convenience methods to the global scope as well as the "dunder"
# format of all of the salt objects
try:
_globals.update({
# salt, pillar & grains all provide shortcuts or object interfaces
'salt': SaltObject(__salt__),
'pillar': __salt__['pillar.get'],
'grains': __salt__['grains.get'],
'mine': __salt__['mine.get'],
'config': __salt__['config.get'],
# the "dunder" formats are still available for direct use
'__salt__': __salt__,
'__pillar__': __pillar__,
'__grains__': __grains__
})
except NameError:
pass
# if salt_data is not True then we just return the global scope we've
# built instead of returning salt data from the registry
if not salt_data:
return _globals
# this will be used to fetch any import files
client = get_file_client(__opts__)
# process our sls imports
#
# we allow pyobjects users to use a special form of the import statement
# so that they may bring in objects from other files. while we do this we
# disable the registry since all we're looking for here is python objects,
# not salt state data
template_data = []
Registry.enabled = False
for line in template.readlines():
line = line.rstrip('\r\n')
matched = False
for RE in (IMPORT_RE, FROM_RE):
matches = re.match(RE, line)
if not matches:
continue
import_file = matches.group(1).strip()
try:
imports = matches.group(2).split(',')
except IndexError:
# if we don't have a third group in the matches object it means
# that we're importing everything
imports = None
state_file = client.cache_file(import_file, saltenv)
if not state_file:
raise ImportError("Could not find the file {0!r}".format(import_file))
with open(state_file) as f:
state_contents = f.read()
state_locals = {}
if sys.version_info[0] > 2:
# in py3+ exec is a function
exec(state_contents, _globals, state_locals)
else:
# prior to that it is a statement
exec state_contents in _globals, state_locals
if imports is None:
imports = state_locals.keys()
for name in imports:
name = name.strip()
if name not in state_locals:
raise ImportError("{0!r} was not found in {1!r}".format(
name,
import_file
))
_globals[name] = state_locals[name]
matched = True
break
if not matched:
template_data.append(line)
final_template = "\n".join(template_data)
# re-enable the registry
Registry.enabled = True
# now exec our template using our created scopes
if sys.version_info[0] > 2:
# in py3+ exec is a function
exec(final_template, _globals, _locals)
else:
# prior to that it is a statement
exec final_template in _globals, _locals
return Registry.salt_data()
def render(template, saltenv='base', sls='', **kwargs):
if 'pyobjects_states' not in __context__:
load_states()
# these hold the scope that our sls file will be executed with
_globals = {}
_locals = {}
# this will be used to fetch any import files
client = get_file_client(__opts__)
# create our StateFactory objects
mod_globals = {'StateFactory': StateFactory}
for mod in __context__['pyobjects_states']:
mod_locals = {}
mod_camel = ''.join([part.capitalize() for part in mod.split('_')])
valid_funcs = "','".join(__context__['pyobjects_states'][mod].keys())
mod_cmd = "{0} = StateFactory('{1!s}', valid_funcs=['{2}'])".format(
mod_camel, mod, valid_funcs)
if sys.version_info[0] > 2:
# in py3+ exec is a function
exec(mod_cmd, mod_globals, mod_locals)
else:
# prior to that it is a statement
exec mod_cmd in mod_globals, mod_locals
_globals[mod_camel] = mod_locals[mod_camel]
# add our include and extend functions
_globals['include'] = Registry.include
_globals['extend'] = Registry.make_extend
# add our map class
Map.__salt__ = __salt__
_globals['Map'] = Map
# add some convenience methods to the global scope as well as the "dunder"
# format of all of the salt objects
try:
_globals.update({
# salt, pillar & grains all provide shortcuts or object interfaces
'salt': SaltObject(__salt__),
'pillar': __salt__['pillar.get'],
'grains': __salt__['grains.get'],
'mine': __salt__['mine.get'],
# the "dunder" formats are still available for direct use
'__salt__': __salt__,
'__pillar__': __pillar__,
'__grains__': __grains__
})
except NameError:
pass
# process our sls imports
#
# we allow pyobjects users to use a special form of the import statement
# so that they may bring in objects from other files. while we do this we
# disable the registry since all we're looking for here is python objects,
# not salt state data
#
# once we have our template we scan through it and look for any "from X
# import Y" statements and if X ends in .sls then we process it by passing
# the file in X through the pyobjects renderer and putting the requested
# variables into our template
template_data = []
Registry.enabled = False
for line in template.readlines():
line = line.rstrip('\r\n')
matched = False
for RE in (IMPORT_RE, FROM_RE):
matches = re.match(RE, line)
if not matches:
continue
import_file = matches.group(1).strip()
try:
imports = matches.group(2).split(',')
except IndexError:
# if we don't have a third group in the matches object it means
# that we're importing everything
imports = None
state_file = client.cache_file(import_file, saltenv)
if not state_file:
raise ImportError(
"Could not find the file {0!r}".format(import_file))
with open(state_file) as f:
state_contents = f.read()
state_locals = {}
if sys.version_info[0] > 2:
# in py3+ exec is a function
exec(state_contents, _globals, state_locals)
else:
# prior to that it is a statement
exec state_contents in _globals, state_locals
if imports is None:
imports = state_locals.keys()
for name in imports:
name = name.strip()
if name not in state_locals:
raise ImportError("{0!r} was not found in {1!r}".format(
name, import_file))
_globals[name] = state_locals[name]
matched = True
break
if not matched:
template_data.append(line)
final_template = "\n".join(template_data)
# re-enable the registry
Registry.enabled = True
# now exec our template using our created scopes
if sys.version_info[0] > 2:
# in py3+ exec is a function
exec(final_template, _globals, _locals)
else:
# prior to that it is a statement
exec final_template in _globals, _locals
return Registry.salt_data()
def render(template, saltenv='base', sls='', salt_data=True, **kwargs):
if 'pyobjects_states' not in __context__:
load_states()
# these hold the scope that our sls file will be executed with
_globals = {}
# create our StateFactory objects
mod_globals = {'StateFactory': StateFactory}
for mod in __context__['pyobjects_states']:
mod_locals = {}
mod_camel = ''.join([
part.capitalize()
for part in mod.split('_')
])
valid_funcs = "','".join(
__context__['pyobjects_states'][mod]
)
mod_cmd = "{0} = StateFactory('{1!s}', valid_funcs=['{2}'])".format(
mod_camel,
mod,
valid_funcs
)
exec_(mod_cmd, mod_globals, mod_locals)
_globals[mod_camel] = mod_locals[mod_camel]
# add our include and extend functions
_globals['include'] = Registry.include
_globals['extend'] = Registry.make_extend
# add our map class
Map.__salt__ = __salt__
_globals['Map'] = Map
# add some convenience methods to the global scope as well as the "dunder"
# format of all of the salt objects
try:
_globals.update({
# salt, pillar & grains all provide shortcuts or object interfaces
'salt': SaltObject(__salt__),
'pillar': __salt__['pillar.get'],
'grains': __salt__['grains.get'],
'mine': __salt__['mine.get'],
'config': __salt__['config.get'],
# the "dunder" formats are still available for direct use
'__salt__': __salt__,
'__pillar__': __pillar__,
'__grains__': __grains__
})
except NameError:
pass
# if salt_data is not True then we just return the global scope we've
# built instead of returning salt data from the registry
if not salt_data:
return _globals
# this will be used to fetch any import files
client = get_file_client(__opts__)
# process our sls imports
#
# we allow pyobjects users to use a special form of the import statement
# so that they may bring in objects from other files. while we do this we
# disable the registry since all we're looking for here is python objects,
# not salt state data
template_data = []
Registry.enabled = False
for line in template.readlines():
line = line.rstrip('\r\n')
matched = False
for RE in (IMPORT_RE, FROM_RE):
matches = RE.match(line)
if not matches:
continue
import_file = matches.group(1).strip()
try:
imports = matches.group(2).split(',')
except IndexError:
# if we don't have a third group in the matches object it means
# that we're importing everything
imports = None
state_file = client.cache_file(import_file, saltenv)
if not state_file:
raise ImportError("Could not find the file {0!r}".format(import_file))
with salt.utils.fopen(state_file) as f:
state_contents = f.read()
state_locals = {}
exec_(state_contents, _globals, state_locals)
if imports is None:
imports = list(state_locals)
for name in imports:
name = alias = name.strip()
matches = FROM_AS_RE.match(name)
if matches is not None:
name = matches.group(1).strip()
alias = matches.group(2).strip()
if name not in state_locals:
raise ImportError("{0!r} was not found in {1!r}".format(
name,
import_file
))
_globals[alias] = state_locals[name]
matched = True
break
if not matched:
template_data.append(line)
final_template = "\n".join(template_data)
# re-enable the registry
Registry.enabled = True
# now exec our template using our created scopes
exec_(final_template, _globals)
return Registry.salt_data()
def render(template, saltenv='base', sls='', salt_data=True, **kwargs):
if 'pyobjects_states' not in __context__:
load_states()
# these hold the scope that our sls file will be executed with
_globals = {}
# create our StateFactory objects
mod_globals = {'StateFactory': StateFactory}
for mod in __context__['pyobjects_states']:
mod_locals = {}
mod_camel = ''.join([
part.capitalize()
for part in mod.split('_')
])
valid_funcs = "','".join(
__context__['pyobjects_states'][mod]
)
mod_cmd = "{0} = StateFactory('{1!s}', valid_funcs=['{2}'])".format(
mod_camel,
mod,
valid_funcs
)
exec_(mod_cmd, mod_globals, mod_locals)
_globals[mod_camel] = mod_locals[mod_camel]
# add our include and extend functions
_globals['include'] = Registry.include
_globals['extend'] = Registry.make_extend
# add our map class
Map.__salt__ = __salt__
_globals['Map'] = Map
# add some convenience methods to the global scope as well as the "dunder"
# format of all of the salt objects
try:
_globals.update({
# salt, pillar & grains all provide shortcuts or object interfaces
'salt': SaltObject(__salt__),
'pillar': __salt__['pillar.get'],
'grains': __salt__['grains.get'],
'mine': __salt__['mine.get'],
'config': __salt__['config.get'],
# the "dunder" formats are still available for direct use
'__salt__': __salt__,
'__pillar__': __pillar__,
'__grains__': __grains__
})
except NameError:
pass
# if salt_data is not True then we just return the global scope we've
# built instead of returning salt data from the registry
if not salt_data:
return _globals
# this will be used to fetch any import files
client = get_file_client(__opts__)
# process our sls imports
#
# we allow pyobjects users to use a special form of the import statement
# so that they may bring in objects from other files. while we do this we
# disable the registry since all we're looking for here is python objects,
# not salt state data
Registry.enabled = False
def process_template(template, template_globals):
template_data = []
state_globals = {}
for line in template.readlines():
line = line.rstrip('\r\n')
matched = False
for RE in (IMPORT_RE, FROM_RE):
matches = RE.match(line)
if not matches:
continue
import_file = matches.group(1).strip()
try:
imports = matches.group(2).split(',')
except IndexError:
# if we don't have a third group in the matches object it means
# that we're importing everything
imports = None
state_file = client.cache_file(import_file, saltenv)
if not state_file:
raise ImportError(
'Could not find the file \'{0}\''.format(import_file)
)
state_locals = {}
with salt.utils.fopen(state_file) as state_fh:
state_contents, state_locals = process_template(state_fh, template_globals)
exec_(state_contents, template_globals, state_locals)
# if no imports have been specified then we are being imported as: import salt://foo.sls
# so we want to stick all of the locals from our state file into the template globals
# under the name of the module -> i.e. foo.MapClass
if imports is None:
import_name = os.path.splitext(os.path.basename(state_file))[0]
state_globals[import_name] = PyobjectsModule(import_name, state_locals)
else:
for name in imports:
name = alias = name.strip()
matches = FROM_AS_RE.match(name)
if matches is not None:
name = matches.group(1).strip()
alias = matches.group(2).strip()
if name not in state_locals:
raise ImportError(
'\'{0}\' was not found in \'{1}\''.format(
name,
import_file
)
)
state_globals[alias] = state_locals[name]
matched = True
break
if not matched:
template_data.append(line)
return "\n".join(template_data), state_globals
# process the template that triggered the render
final_template, final_locals = process_template(template, _globals)
_globals.update(final_locals)
# re-enable the registry
Registry.enabled = True
# now exec our template using our created scopes
exec_(final_template, _globals)
return Registry.salt_data()
def test_get_file_client(self):
with patch.dict(self.minion_opts, {'file_client': 'remote'}):
with patch('salt.fileclient.RemoteClient', MagicMock(return_value='remote_client')):
ret = fileclient.get_file_client(self.minion_opts)
self.assertEqual('remote_client', ret)
def render(template, saltenv="base", sls="", salt_data=True, **kwargs):
if "pyobjects_states" not in __context__:
load_states()
# these hold the scope that our sls file will be executed with
_globals = {}
# create our StateFactory objects
mod_globals = {"StateFactory": StateFactory}
for mod in __context__["pyobjects_states"]:
mod_locals = {}
mod_camel = "".join([part.capitalize() for part in mod.split("_")])
valid_funcs = "','".join(__context__["pyobjects_states"][mod])
mod_cmd = "{} = StateFactory('{!s}', valid_funcs=['{}'])".format(
mod_camel, mod, valid_funcs)
exec(mod_cmd, mod_globals, mod_locals)
_globals[mod_camel] = mod_locals[mod_camel]
# add our include and extend functions
_globals["include"] = Registry.include
_globals["extend"] = Registry.make_extend
# add our map class
Map.__salt__ = __salt__
_globals["Map"] = Map
# add some convenience methods to the global scope as well as the "dunder"
# format of all of the salt objects
try:
_globals.update({
# salt, pillar & grains all provide shortcuts or object interfaces
"salt": SaltObject(__salt__),
"pillar": __salt__["pillar.get"],
"grains": __salt__["grains.get"],
"mine": __salt__["mine.get"],
"config": __salt__["config.get"],
# the "dunder" formats are still available for direct use
"__salt__": __salt__,
"__pillar__": __pillar__,
"__grains__": __grains__,
"__opts__": __opts__,
"__sls__": sls,
})
except NameError:
pass
# if salt_data is not True then we just return the global scope we've
# built instead of returning salt data from the registry
if not salt_data:
return _globals
# this will be used to fetch any import files
client = get_file_client(__opts__)
# process our sls imports
#
# we allow pyobjects users to use a special form of the import statement
# so that they may bring in objects from other files. while we do this we
# disable the registry since all we're looking for here is python objects,
# not salt state data
Registry.enabled = False
def process_template(template):
template_data = []
# Do not pass our globals to the modules we are including and keep the root _globals untouched
template_globals = dict(_globals)
for line in template.readlines():
line = line.rstrip("\r\n")
matched = False
for RE in (IMPORT_RE, FROM_RE):
matches = RE.match(line)
if not matches:
continue
import_file = matches.group(1).strip()
try:
imports = matches.group(2).split(",")
except IndexError:
# if we don't have a third group in the matches object it means
# that we're importing everything
imports = None
state_file = client.cache_file(import_file, saltenv)
if not state_file:
raise ImportError(
"Could not find the file '{}'".format(import_file))
with salt.utils.files.fopen(state_file) as state_fh:
state_contents, state_globals = process_template(state_fh)
exec(state_contents, state_globals)
# if no imports have been specified then we are being imported as: import salt://foo.sls
# so we want to stick all of the locals from our state file into the template globals
# under the name of the module -> i.e. foo.MapClass
if imports is None:
import_name = os.path.splitext(
os.path.basename(state_file))[0]
template_globals[import_name] = PyobjectsModule(
import_name, state_globals)
else:
for name in imports:
name = alias = name.strip()
matches = FROM_AS_RE.match(name)
if matches is not None:
name = matches.group(1).strip()
alias = matches.group(2).strip()
if name not in state_globals:
raise ImportError(
"'{}' was not found in '{}'".format(
name, import_file))
template_globals[alias] = state_globals[name]
matched = True
break
if not matched:
template_data.append(line)
return "\n".join(template_data), template_globals
# process the template that triggered the render
final_template, final_globals = process_template(template)
_globals.update(final_globals)
# re-enable the registry
Registry.enabled = True
# now exec our template using our created scopes
exec(final_template, _globals)
return Registry.salt_data()
def test_get_file_client(self):
minion_opts = self.get_temp_config('minion')
minion_opts['file_client'] = 'remote'
with patch('salt.fileclient.RemoteClient', MagicMock(return_value='remote_client')):
ret = fileclient.get_file_client(minion_opts)
self.assertEqual('remote_client', ret)
