def _find_image(name):
'''
Tries to find image with given name, returns
- image, 'Found image <name>'
- None, 'No such image found'
- False, 'Found more than one image with given name'
'''
try:
images_dict = __salt__['glance.image_list'](name=name)
except kstone_Unauthorized:
return False, 'keystoneclient: Unauthorized'
except glance_Unauthorized:
return False, 'glanceclient: Unauthorized'
log.debug('Got images_dict: {0}'.format(images_dict))
warn_until(
'Boron', 'Starting with Boron '
'\'glance.image_list\' is not supposed to return '
'the images wrapped in a separate dict anymore.')
if len(images_dict) == 1 and 'images' in images_dict:
images_dict = images_dict['images']
# I /think/ this will still work when glance.image_list
# starts returning a list instead of a dictionary...
if len(images_dict) == 0:
return None, 'No image with name "{0}"'.format(name)
elif len(images_dict) == 1:
return images_dict.values()[0], 'Found image {0}'.format(name)
elif len(images_dict) > 1:
return False, 'Found more than one image with given name'
else:
raise NotImplementedError
def _find_image(name):
'''
Tries to find image with given name, returns
- image, 'Found image <name>'
- None, 'No such image found'
- False, 'Found more than one image with given name'
'''
try:
images_dict = __salt__['glance.image_list'](name=name)
except kstone_Unauthorized:
return False, 'keystoneclient: Unauthorized'
except glance_Unauthorized:
return False, 'glanceclient: Unauthorized'
log.debug('Got images_dict: {0}'.format(images_dict))
warn_until('Boron', 'Starting with Boron '
'\'glance.image_list\' is not supposed to return '
'the images wrapped in a separate dict anymore.')
if len(images_dict) == 1 and 'images' in images_dict:
images_dict = images_dict['images']
# I /think/ this will still work when glance.image_list
# starts returning a list instead of a dictionary...
if len(images_dict) == 0:
return None, 'No image with name "{0}"'.format(name)
elif len(images_dict) == 1:
return images_dict.values()[0], 'Found image {0}'.format(name)
elif len(images_dict) > 1:
return False, 'Found more than one image with given name'
else:
raise NotImplementedError
def image_update(id=None, name=None, profile=None, **kwargs): # pylint: disable=C0103
'''
Update properties of given image.
Known to work for:
- min_ram (in MB)
- protected (bool)
- visibility ('public' or 'private')
CLI Example:
.. code-block:: bash
salt '*' glance.image_update id=c2eb2eb0-53e1-4a80-b990-8ec887eae7df
salt '*' glance.image_update name=f16-jeos
'''
if id:
image = image_show(id=id, profile=profile)
if 'result' in image and not image['result']:
return image
elif len(image) == 1:
image = image.values()[0]
elif name:
img_list = image_list(name=name, profile=profile)
if img_list is dict and 'result' in img_list:
return img_list
elif len(img_list) == 0:
return {
'result': False,
'comment':
'No image with name \'{0}\' '
'found.'.format(name)
}
elif len(img_list) == 1:
try:
image = img_list[0]
except KeyError:
image = img_list[name]
else:
raise SaltInvocationError
log.debug('Found image:\n{0}'.format(image))
to_update = {}
for key, value in kwargs.items():
if key.startswith('_'):
continue
if key not in image or image[key] != value:
log.debug('add <{0}={1}> to to_update'.format(key, value))
to_update[key] = value
g_client = _auth(profile)
updated = g_client.images.update(image['id'], **to_update)
# I may want to use this code on Beryllium
# until we got 2016.3.0 packages for Ubuntu
# so please keep this code until Carbon!
warn_until('Carbon', 'Starting with \'2016.3.0\' image_update() '
'will stop wrapping the returned, updated image in '
'another dictionary.')
if CUR_VER < BORON:
updated = {updated.name: updated}
return updated
def image_update(id=None, name=None, profile=None, **kwargs): # pylint: disable=C0103
'''
Update properties of given image.
Known to work for:
- min_ram (in MB)
- protected (bool)
- visibility ('public' or 'private')
CLI Example:
.. code-block:: bash
salt '*' glance.image_update id=c2eb2eb0-53e1-4a80-b990-8ec887eae7df
salt '*' glance.image_update name=f16-jeos
'''
if id:
image = image_show(id=id, profile=profile)
if 'result' in image and not image['result']:
return image
elif len(image) == 1:
image = image.values()[0]
elif name:
img_list = image_list(name=name, profile=profile)
if img_list is dict and 'result' in img_list:
return img_list
elif len(img_list) == 0:
return {
'result': False,
'comment': 'No image with name \'{0}\' '
'found.'.format(name)
}
elif len(img_list) == 1:
try:
image = img_list[0]
except KeyError:
image = img_list[name]
else:
raise SaltInvocationError
log.debug('Found image:\n{0}'.format(image))
to_update = {}
for key, value in kwargs.items():
if key.startswith('_'):
continue
if key not in image or image[key] != value:
log.debug('add <{0}={1}> to to_update'.format(key, value))
to_update[key] = value
g_client = _auth(profile)
updated = g_client.images.update(image['id'], **to_update)
# I may want to use this code on Beryllium
# until we got Boron packages for Ubuntu
# so please keep this code until Carbon!
warn_until(
'Carbon', 'Starting with \'Boron\' image_update() '
'will stop wrapping the returned, updated image in '
'another dictionary.')
if CUR_VER < BORON:
updated = {updated.name: updated}
return updated
def get_configured_provider():
'''
Return the first configured instance.
'''
warn_until(
'Carbon',
'The vsphere driver is deprecated in favor of the vmware driver and will be removed '
'in Salt Carbon. Please convert your vsphere provider configs to use the vmware driver.'
)
return config.is_provider_configured(__opts__, __active_provider_name__
or 'vsphere', ('user', ))
def image_show(id=None, name=None, profile=None): # pylint: disable=C0103
'''
Return details about a specific image (glance image-show)
CLI Example:
.. code-block:: bash
salt '*' glance.image_show
'''
g_client = _auth(profile)
ret = {}
if name:
for image in g_client.images.list():
if image.name == name:
id = image.id # pylint: disable=C0103
continue
if not id:
return {
'result': False,
'comment':
'Unable to resolve image ID '
'for name \'{0}\''.format(name)
}
try:
image = g_client.images.get(id)
except exc.HTTPNotFound:
return {
'result': False,
'comment': 'No image with ID {0}'.format(id)
}
pformat = pprint.PrettyPrinter(indent=4).pformat
log.debug('Properties of image {0}:\n{1}'.format(
image.name, pformat(image)))
ret_details = {}
# I may want to use this code on Beryllium
# until we got Boron packages for Ubuntu
# so please keep this code until Carbon!
warn_until('Carbon', 'Starting with \'Boron\' image_show() '
'will stop wrapping the returned image in another '
'dictionary.')
if CUR_VER < BORON:
ret[image.name] = ret_details
else:
ret = ret_details
schema = image_schema(profile=profile)
if len(schema.keys()) == 1:
schema = schema['image']
for key in schema.keys():
if key in image:
ret_details[key] = image[key]
return ret
def image_show(id=None, name=None, profile=None): # pylint: disable=C0103
'''
Return details about a specific image (glance image-show)
CLI Example:
.. code-block:: bash
salt '*' glance.image_show
'''
g_client = _auth(profile)
ret = {}
if name:
for image in g_client.images.list():
if image.name == name:
id = image.id # pylint: disable=C0103
continue
if not id:
return {
'result': False,
'comment':
'Unable to resolve image ID '
'for name \'{0}\''.format(name)
}
try:
image = g_client.images.get(id)
except exc.HTTPNotFound:
return {
'result': False,
'comment': 'No image with ID {0}'.format(id)
}
pformat = pprint.PrettyPrinter(indent=4).pformat
log.debug('Properties of image {0}:\n{1}'.format(
image.name, pformat(image)))
ret_details = {}
# I may want to use this code on Beryllium
# until we got Boron packages for Ubuntu
# so please keep this code until Carbon!
warn_until('Carbon', 'Starting with \'Boron\' image_show() '
'will stop wrapping the returned image in another '
'dictionary.')
if CUR_VER < BORON:
ret[image.name] = ret_details
else:
ret = ret_details
schema = image_schema(profile=profile)
if len(schema.keys()) == 1:
schema = schema['image']
for key in schema.keys():
if key in image:
ret_details[key] = image[key]
return ret
def get_configured_provider():
'''
Return the first configured instance.
'''
warn_until(
'Beryllium',
'The digital_ocean driver is deprecated and will be removed in Salt Beryllium. '
'Please convert your digital ocean provider configs to use the digital_ocean_v2 '
'driver.')
return config.is_provider_configured(
__opts__, __active_provider_name__ or 'digital_ocean',
('personal_access_token', ))
def get_configured_provider():
'''
Return the first configured instance.
'''
warn_until(
'Carbon',
'The vsphere driver is deprecated in favor of the vmware driver and will be removed '
'in Salt Carbon. Please convert your vsphere provider configs to use the vmware driver.'
)
return config.is_provider_configured(
__opts__,
__active_provider_name__ or 'vsphere',
('user',)
)
def include(self, *sls_names, **kws):
if 'env' in kws:
warn_until(
'Oxygen',
'Parameter \'env\' has been detected in the argument list. This '
'parameter is no longer used and has been replaced by \'saltenv\' '
'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.'
)
kws.pop('env')
saltenv = kws.get('saltenv', self.saltenv)
if kws.get('delayed', False):
for incl in sls_names:
self.includes.append((saltenv, incl))
return
HIGHSTATE = HighState.get_active()
global SLS_MATCHES
if SLS_MATCHES is None:
SLS_MATCHES = HIGHSTATE.top_matches(HIGHSTATE.get_top())
highstate = self.included_highstate
slsmods = [] # a list of pydsl sls modules rendered.
for sls in sls_names:
r_env = '{0}:{1}'.format(saltenv, sls)
if r_env not in self.rendered_sls:
self.rendered_sls.add(
sls
) # needed in case the starting sls uses the pydsl renderer.
histates, errors = HIGHSTATE.render_state(
sls, saltenv, self.rendered_sls, SLS_MATCHES)
HIGHSTATE.merge_included_states(highstate, histates, errors)
if errors:
raise PyDslError('\n'.join(errors))
HIGHSTATE.clean_duplicate_extends(highstate)
state_id = '_slsmod_{0}'.format(sls)
if state_id not in highstate:
slsmods.append(None)
else:
for arg in highstate[state_id]['stateconf']:
if isinstance(arg, dict) and next(iter(arg)) == 'slsmod':
slsmods.append(arg['slsmod'])
break
if not slsmods:
return None
return slsmods[0] if len(slsmods) == 1 else slsmods
def include(self, *sls_names, **kws):
if kws.get('env', None) is not None:
warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt Boron.'
)
# Backwards compatibility
kws['saltenv'] = kws.pop('env')
saltenv = kws.get('saltenv', self.saltenv)
if kws.get('delayed', False):
for incl in sls_names:
self.includes.append((saltenv, incl))
return
HIGHSTATE = HighState.get_active()
global SLS_MATCHES
if SLS_MATCHES is None:
SLS_MATCHES = HIGHSTATE.top_matches(HIGHSTATE.get_top())
highstate = self.included_highstate
slsmods = [] # a list of pydsl sls core rendered.
for sls in sls_names:
r_env = '{0}:{1}'.format(saltenv, sls)
if r_env not in self.rendered_sls:
self.rendered_sls.add(
sls
) # needed in case the starting sls uses the pydsl renderer.
histates, errors = HIGHSTATE.render_state(
sls, saltenv, self.rendered_sls, SLS_MATCHES)
HIGHSTATE.merge_included_states(highstate, histates, errors)
if errors:
raise PyDslError('\n'.join(errors))
HIGHSTATE.clean_duplicate_extends(highstate)
state_id = '_slsmod_{0}'.format(sls)
if state_id not in highstate:
slsmods.append(None)
else:
for arg in highstate[state_id]['stateconf']:
if isinstance(arg, dict) and next(iter(arg)) == 'slsmod':
slsmods.append(arg['slsmod'])
break
if not slsmods:
return None
return slsmods[0] if len(slsmods) == 1 else slsmods
def include(self, *sls_names, **kws):
if kws.get('env', None) is not None:
warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt Boron.'
)
# Backwards compatibility
kws['saltenv'] = kws.pop('env')
saltenv = kws.get('saltenv', self.saltenv)
if kws.get('delayed', False):
for incl in sls_names:
self.includes.append((saltenv, incl))
return
HIGHSTATE = HighState.get_active()
global SLS_MATCHES
if SLS_MATCHES is None:
SLS_MATCHES = HIGHSTATE.top_matches(HIGHSTATE.get_top())
highstate = self.included_highstate
slsmods = [] # a list of pydsl sls modules rendered.
for sls in sls_names:
r_env = '{0}:{1}'.format(saltenv, sls)
if r_env not in self.rendered_sls:
self.rendered_sls.add(sls) # needed in case the starting sls
# uses the pydsl renderer.
histates, errors = HIGHSTATE.render_state(
sls, saltenv, self.rendered_sls, SLS_MATCHES
)
HIGHSTATE.merge_included_states(highstate, histates, errors)
if errors:
raise PyDslError('\n'.join(errors))
HIGHSTATE.clean_duplicate_extends(highstate)
state_id = '_slsmod_{0}'.format(sls)
if state_id not in highstate:
slsmods.append(None)
else:
for arg in highstate[state_id]['stateconf']:
if isinstance(arg, dict) and iter(arg).next() == 'slsmod':
slsmods.append(arg['slsmod'])
break
if not slsmods:
return None
return slsmods[0] if len(slsmods) == 1 else slsmods
def include(self, *sls_names, **kws):
if 'env' in kws:
warn_until(
'Oxygen',
'Parameter \'env\' has been detected in the argument list. This '
'parameter is no longer used and has been replaced by \'saltenv\' '
'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.'
)
kws.pop('env')
saltenv = kws.get('saltenv', self.saltenv)
if kws.get('delayed', False):
for incl in sls_names:
self.includes.append((saltenv, incl))
return
HIGHSTATE = HighState.get_active()
global SLS_MATCHES
if SLS_MATCHES is None:
SLS_MATCHES = HIGHSTATE.top_matches(HIGHSTATE.get_top())
highstate = self.included_highstate
slsmods = [] # a list of pydsl sls modules rendered.
for sls in sls_names:
r_env = '{0}:{1}'.format(saltenv, sls)
if r_env not in self.rendered_sls:
self.rendered_sls.add(sls) # needed in case the starting sls uses the pydsl renderer.
histates, errors = HIGHSTATE.render_state(
sls, saltenv, self.rendered_sls, SLS_MATCHES
)
HIGHSTATE.merge_included_states(highstate, histates, errors)
if errors:
raise PyDslError('\n'.join(errors))
HIGHSTATE.clean_duplicate_extends(highstate)
state_id = '_slsmod_{0}'.format(sls)
if state_id not in highstate:
slsmods.append(None)
else:
for arg in highstate[state_id]['stateconf']:
if isinstance(arg, dict) and next(iter(arg)) == 'slsmod':
slsmods.append(arg['slsmod'])
break
if not slsmods:
return None
return slsmods[0] if len(slsmods) == 1 else slsmods
def test_warn_until_warning_raised(self, salt_version_mock):
# We *always* want *all* warnings thrown on this module
warnings.filterwarnings('always', '', DeprecationWarning, __name__)
# Define a salt version info
salt_version_mock.__version_info__ = (0, 16)
def raise_warning():
warn_until(
(0, 17), 'Deprecation Message!'
)
# raise_warning should show warning until version info is >= (0, 17)
with warnings.catch_warnings(record=True) as recorded_warnings:
raise_warning()
self.assertEqual(
'Deprecation Message!', str(recorded_warnings[0].message)
)
# the deprecation warning is not issued because we passed
# _dont_call_warning
with warnings.catch_warnings(record=True) as recorded_warnings:
warn_until(
(0, 17), 'Foo', _dont_call_warnings=True
)
self.assertEqual(0, len(recorded_warnings))
# Let's set version info to (0, 17), a RuntimeError should be raised
salt_version_mock.__version_info__ = (0, 17)
with self.assertRaisesRegexp(
RuntimeError,
r'The warning triggered on filename \'(.*)warnings_test.py\', '
r'line number ([\d]+), is supposed to be shown until version '
r'\'0.17\' is released. Current version is now \'0.17\'. Please '
r'remove the warning.'):
raise_warning()
# Even though we're calling warn_until, we pass _dont_call_warnings
# because we're only after the RuntimeError
with self.assertRaisesRegexp(
RuntimeError,
r'The warning triggered on filename \'(.*)warnings_test.py\', '
r'line number ([\d]+), is supposed to be shown until version '
r'\'0.17\' is released. Current version is now \'0.17\'. Please '
r'remove the warning.'):
warn_until(
(0, 17), 'Foo', _dont_call_warnings=True
)
def get_configured_provider():
'''
Return the first configured instance.
'''
warn_until(
'Beryllium',
'The digital_ocean driver is deprecated and will be removed in Salt Beryllium. '
'Please convert your digital ocean provider configs to use the digital_ocean_v2 '
'driver.'
)
return config.is_provider_configured(
__opts__,
__active_provider_name__ or 'digital_ocean',
('personal_access_token',)
)
def low(self, fun, low):
'''
Check for deprecated usage and allow until Salt Oxygen.
'''
msg = []
if 'args' in low:
msg.append('call with arg instead')
low['arg'] = low.pop('args')
if 'kwargs' in low:
msg.append('call with kwarg instead')
low['kwarg'] = low.pop('kwargs')
if msg:
warn_until('Oxygen', ' '.join(msg))
return self._low(fun, low)
def get_private_ip(vm_):
'''
Return True if a private ip address is requested
'''
if 'private_ip' in vm_:
warn_until(
'Carbon',
'The \'private_ip\' option is being deprecated in favor of the '
'\'assign_private_ip\' option. Please convert your Linode configuration '
'files to use \'assign_private_ip\'.'
)
vm_['assign_private_ip'] = vm_['private_ip']
vm_.pop('private_ip')
return config.get_cloud_config_value(
'assign_private_ip', vm_, __opts__, default=False
)
def image_list(id=None, profile=None, name=None): # pylint: disable=C0103
'''
Return a list of available images (glance image-list)
CLI Example:
.. code-block:: bash
salt '*' glance.image_list
'''
#try:
g_client = _auth(profile)
#except kstone_exc.Unauthorized:
# return False
#
# I may want to use this code on Beryllium
# until we got Boron packages for Ubuntu
# so please keep this code until Carbon!
warn_until(
'Carbon', 'Starting in \'Boron\' image_list() '
'will return a list of images instead of a dictionary '
'keyed with the images\' names.')
if CUR_VER < BORON:
ret = {}
else:
ret = []
for image in g_client.images.list():
if id is None and name is None:
_add_image(ret, image)
else:
if id is not None and id == image.id:
_add_image(ret, image)
return ret
if name == image.name:
if name in ret and CUR_VER < BORON:
# Not really worth an exception
return {
'result':
False,
'comment':
'More than one image with '
'name "{0}"'.format(name)
}
_add_image(ret, image)
log.debug('Returning images: {0}'.format(ret))
return ret
def get_private_ip(vm_):
'''
Return True if a private ip address is requested
'''
if 'private_ip' in vm_:
warn_until(
'Carbon',
'The \'private_ip\' option is being deprecated in favor of the '
'\'assign_private_ip\' option. Please convert your Linode configuration '
'files to use \'assign_private_ip\'.')
vm_['assign_private_ip'] = vm_['private_ip']
vm_.pop('private_ip')
return config.get_cloud_config_value('assign_private_ip',
vm_,
__opts__,
default=False)
def __virtual__():
'''
Check for vSphere configurations.
'''
if get_configured_provider() is False:
return False
if get_dependencies() is False:
return False
warn_until(
'Carbon',
'The vsphere driver is deprecated in favor of the vmware driver and will be removed '
'in Salt Carbon. Please convert your vsphere provider configs to use the vmware driver.'
)
return __virtualname__
def __virtual__():
'''
Check for vSphere configurations.
'''
if get_configured_provider() is False:
return False
if get_dependencies() is False:
return False
warn_until(
'Carbon',
'The vsphere driver is deprecated in favor of the vmware driver and will be removed '
'in Salt Carbon. Please convert your vsphere provider configs to use the vmware driver.'
)
return __virtualname__
def image_list(id=None, profile=None, name=None): # pylint: disable=C0103
'''
Return a list of available images (glance image-list)
CLI Example:
.. code-block:: bash
salt '*' glance.image_list
'''
#try:
g_client = _auth(profile)
#except kstone_exc.Unauthorized:
# return False
#
# I may want to use this code on Beryllium
# until we got Boron packages for Ubuntu
# so please keep this code until Carbon!
warn_until('Carbon', 'Starting in \'Boron\' image_list() '
'will return a list of images instead of a dictionary '
'keyed with the images\' names.')
if CUR_VER < BORON:
ret = {}
else:
ret = []
for image in g_client.images.list():
if id is None and name is None:
_add_image(ret, image)
else:
if id is not None and id == image.id:
_add_image(ret, image)
return ret
if name == image.name:
if name in ret and CUR_VER < BORON:
# Not really worth an exception
return {
'result': False,
'comment':
'More than one image with '
'name "{0}"'.format(name)
}
_add_image(ret, image)
log.debug('Returning images: {0}'.format(ret))
return ret
def test_warn_until_warning_raised(self, salt_version_mock):
# We *always* want *all* warnings thrown on this module
warnings.filterwarnings('always', '', DeprecationWarning, __name__)
# Define a salt version info
salt_version_mock.__version_info__ = (0, 16)
def raise_warning():
warn_until((0, 17), 'Deprecation Message!')
# raise_warning should show warning until version info is >= (0, 17)
with warnings.catch_warnings(record=True) as recorded_warnings:
raise_warning()
self.assertEqual('Deprecation Message!',
str(recorded_warnings[0].message))
# the deprecation warning is not issued because we passed
# _dont_call_warning
with warnings.catch_warnings(record=True) as recorded_warnings:
warn_until((0, 17), 'Foo', _dont_call_warnings=True)
self.assertEqual(0, len(recorded_warnings))
# Let's set version info to (0, 17), a RuntimeError should be raised
salt_version_mock.__version_info__ = (0, 17)
with self.assertRaisesRegexp(
RuntimeError,
r'The warning triggered on filename \'(.*)warnings_test.py\', '
r'line number ([\d]+), is supposed to be shown until version '
r'\'0.17\' is released. Current version is now \'0.17\'. Please '
r'remove the warning.'):
raise_warning()
# Even though we're calling warn_until, we pass _dont_call_warnings
# because we're only after the RuntimeError
with self.assertRaisesRegexp(
RuntimeError,
r'The warning triggered on filename \'(.*)warnings_test.py\', '
r'line number ([\d]+), is supposed to be shown until version '
r'\'0.17\' is released. Current version is now \'0.17\'. Please '
r'remove the warning.'):
warn_until((0, 17), 'Foo', _dont_call_warnings=True)
def raise_warning():
warn_until(
(0, 17), 'Deprecation Message!'
)
def image_create(name, location=None, profile=None, visibility=None,
container_format='bare', disk_format='raw', protected=None,
copy_from=None, is_public=None):
'''
Create an image (glance image-create)
CLI Example, old format:
.. code-block:: bash
salt '*' glance.image_create name=f16-jeos is_public=true \\
disk_format=qcow2 container_format=ovf \\
copy_from=http://berrange.fedorapeople.org/\
images/2012-02-29/f16-x86_64-openstack-sda.qcow2
CLI Example, new format resembling Glance API v2:
.. code-block:: bash
salt '*' glance.image_create name=f16-jeos visibility=public \\
disk_format=qcow2 container_format=ovf \\
copy_from=http://berrange.fedorapeople.org/\
images/2012-02-29/f16-x86_64-openstack-sda.qcow2
The parameter 'visibility' defaults to 'public' if neither
'visibility' nor 'is_public' is specified.
'''
kwargs = {}
# valid options for "visibility":
v_list = ['public', 'private']
# valid options for "container_format":
cf_list = ['ami', 'ari', 'aki', 'bare', 'ovf']
# valid options for "disk_format":
df_list = ['ami', 'ari', 'aki', 'vhd', 'vmdk',
'raw', 'qcow2', 'vdi', 'iso']
# 'location' and 'visibility' are the parameters used in
# Glance API v2. For now we have to use v1 for now (see below)
# but this modules interface will change in Carbon.
if copy_from is not None or is_public is not None:
warn_until('Carbon', 'The parameters \'copy_from\' and '
'\'is_public\' are deprecated and will be removed. '
'Use \'location\' and \'visibility\' instead.')
if is_public is not None and visibility is not None:
raise SaltInvocationError('Must only specify one of '
'\'is_public\' and \'visibility\'')
if copy_from is not None and location is not None:
raise SaltInvocationError('Must only specify one of '
'\'copy_from\' and \'location\'')
if copy_from is not None:
kwargs['copy_from'] = copy_from
else:
kwargs['copy_from'] = location
if is_public is not None:
kwargs['is_public'] = is_public
elif visibility is not None:
if visibility not in v_list:
raise SaltInvocationError('"visibility" needs to be one ' +
'of the following: {0}'.format(', '.join(v_list)))
elif visibility == 'public':
kwargs['is_public'] = True
else:
kwargs['is_public'] = False
else:
kwargs['is_public'] = True
if container_format not in cf_list:
raise SaltInvocationError('"container_format" needs to be ' +
'one of the following: {0}'.format(', '.join(cf_list)))
else:
kwargs['container_format'] = container_format
if disk_format not in df_list:
raise SaltInvocationError('"disk_format" needs to be one ' +
'of the following: {0}'.format(', '.join(df_list)))
else:
kwargs['disk_format'] = disk_format
if protected is not None:
kwargs['protected'] = protected
# Icehouse's glanceclient doesn't have add_location() and
# glanceclient.v2 doesn't implement Client.images.create()
# in a usable fashion. Thus we have to use v1 for now.
g_client = _auth(profile, api_version=1)
image = g_client.images.create(name=name, **kwargs)
return image_show(image.id, profile=profile)
def image_present(name,
visibility='public',
protected=None,
checksum=None,
location=None,
wait_for=None,
timeout=30):
'''
Checks if given image is present with properties
set as specified.
An image should got through the stages 'queued', 'saving'
before becoming 'active'. The attribute 'checksum' can
only be checked once the image is active.
If you don't specify 'wait_for' but 'checksum' the function
will wait for the image to become active before comparing
checksums. If you don't specify checksum either the function
will return when the image reached 'saving'.
The default timeout for both is 30 seconds.
Supported properties:
- visibility ('public' or 'private')
- protected (bool)
- checksum (string, md5sum)
- location (URL, to copy from)
'''
ret = {
'name': name,
'changes': {},
'result': True,
'comment': '',
}
acceptable = ['queued', 'saving', 'active']
if wait_for is None and checksum is None:
wait_for = 'saving'
elif wait_for is None and checksum is not None:
wait_for = 'active'
# Just pop states until we reach the
# first acceptable one:
while len(acceptable) > 1:
if acceptable[0] == wait_for:
break
else:
acceptable.pop(0)
image, msg = _find_image(name)
if image is False:
if __opts__['test']:
ret['result'] = None
else:
ret['result'] = False
ret['comment'] = msg
return ret
log.debug(msg)
# No image yet and we know where to get one
if image is None and location is not None:
if __opts__['test']:
ret['result'] = None
ret['comment'] = 'glance.image_present would ' \
'create an image from {0}'.format(location)
return ret
image = __salt__['glance.image_create'](name=name,
protected=protected,
visibility=visibility,
location=location)
# See Salt issue #24568
warn_until(
'Boron', 'Starting with Boron '
'\'glance.image_create\' is not supposed to return '
'the image wrapped in a dict anymore.')
if len(image.keys()) == 1:
image = image.values()[0]
log.debug('Created new image:\n{0}'.format(image))
ret['changes'] = {name: {'new': {'id': image['id']}, 'old': None}}
timer = timeout
# Kinda busy-loopy but I don't think the Glance
# API has events we can listen for
while timer > 0:
if 'status' in image and \
image['status'] in acceptable:
log.debug('Image {0} has reached status {1}'.format(
image['name'], image['status']))
break
else:
timer -= 5
time.sleep(5)
image, msg = _find_image(name)
if not image:
ret['result'] = False
ret['comment'] += 'Created image {0} '.format(
name) + ' vanished:\n' + msg
return ret
elif len(image.keys()) == 1:
# See Salt issue #24568
warn_until(
'Boron', 'Starting with Boron '
'\'_find_image()\' is not supposed to return '
'the image wrapped in a dict anymore.')
image = image.values()[0]
if timer <= 0 and image['status'] not in acceptable:
ret['result'] = False
ret['comment'] += 'Image didn\'t reach an acceptable '+\
'state ({0}) before timeout:\n'.format(acceptable)+\
'\tLast status was "{0}".\n'.format(image['status'])
# See Salt issue #24568
warn_until(
'Boron', 'Starting with Boron '
'\'_find_image()\' is not supposed to return '
'the image wrapped in a dict anymore.')
if len(image.keys()) == 1:
image = image.values()[0]
# ret[comment] +=
# There's no image but where would I get one??
elif location is None:
if __opts__['test']:
ret['result'] = None
ret['comment'] = 'No location to copy image from specified,\n' +\
'glance.image_present would not create one'
else:
ret['result'] = False
ret['comment'] = 'No location to copy image from specified,\n' +\
'not creating a new image.'
return ret
# If we've created a new image also return its last status:
if name in ret['changes']:
ret['changes'][name]['new']['status'] = image['status']
if visibility:
if image['visibility'] != visibility:
old_value = image['visibility']
if not __opts__['test']:
image = __salt__['glance.image_update'](id=image['id'],
visibility=visibility)
# See Salt issue #24568
warn_until(
'Boron', 'Starting with Boron '
'\'glance.image_update\' is not supposed to return '
'the image wrapped in a dict anymore.')
if len(image.keys()) == 1:
image = image.values()[0]
# Check if image_update() worked:
if image['visibility'] != visibility:
if not __opts__['test']:
ret['result'] = False
elif __opts__['test']:
ret['result'] = None
ret['comment'] += '"visibility" is {0}, '\
'should be {1}.\n'.format(image['visibility'],
visibility)
else:
if 'new' in ret['changes']:
ret['changes']['new']['visibility'] = visibility
else:
ret['changes']['new'] = {'visibility': visibility}
if 'old' in ret['changes']:
ret['changes']['old']['visibility'] = old_value
else:
ret['changes']['old'] = {'visibility': old_value}
else:
ret['comment'] += '"visibility" is correct ({0}).\n'.format(
visibility)
if protected is not None:
if not isinstance(protected, bool) or image['protected'] ^ protected:
if not __opts__['test']:
ret['result'] = False
else:
ret['result'] = None
ret['comment'] += '"protected" is {0}, should be {1}.\n'.format(
image['protected'], protected)
else:
ret['comment'] += '"protected" is correct ({0}).\n'.format(
protected)
if 'status' in image and checksum:
if image['status'] == 'active':
if 'checksum' not in image:
# Refresh our info about the image
image = __salt__['glance.image_show'](image['id'])
warn_until(
'Boron', 'Starting with Boron '
'\'glance.image_show\' is not supposed to return '
'the image wrapped in a dict anymore.')
if len(image.keys()) == 1:
image = image.values()[0]
if 'checksum' not in image:
if not __opts__['test']:
ret['result'] = False
else:
ret['result'] = None
ret['comment'] += 'No checksum available for this image:\n' +\
'\tImage has status "{0}".'.format(image['status'])
elif image['checksum'] != checksum:
if not __opts__['test']:
ret['result'] = False
else:
ret['result'] = None
ret['comment'] += '"checksum" is {0}, should be {1}.\n'.format(
image['checksum'], checksum)
else:
ret['comment'] += '"checksum" is correct ({0}).\n'.format(
checksum)
elif image['status'] in ['saving', 'queued']:
ret['comment'] += 'Checksum won\'t be verified as image ' +\
'hasn\'t reached\n\t "status=active" yet.\n'
log.debug('glance.image_present will return: {0}'.format(ret))
return ret
def raise_named_version_warning(_version_info_=(0, 16, 0)):
warn_until(
'Hydrogen', 'Deprecation Message!',
_version_info_=_version_info_
)
def image_create(name,
location=None,
profile=None,
visibility=None,
container_format='bare',
disk_format='raw',
protected=None,
copy_from=None,
is_public=None):
'''
Create an image (glance image-create)
CLI Example, old format:
.. code-block:: bash
salt '*' glance.image_create name=f16-jeos is_public=true \\
disk_format=qcow2 container_format=ovf \\
copy_from=http://berrange.fedorapeople.org/\
images/2012-02-29/f16-x86_64-openstack-sda.qcow2
CLI Example, new format resembling Glance API v2:
.. code-block:: bash
salt '*' glance.image_create name=f16-jeos visibility=public \\
disk_format=qcow2 container_format=ovf \\
copy_from=http://berrange.fedorapeople.org/\
images/2012-02-29/f16-x86_64-openstack-sda.qcow2
The parameter 'visibility' defaults to 'public' if neither
'visibility' nor 'is_public' is specified.
'''
kwargs = {}
# valid options for "visibility":
v_list = ['public', 'private']
# valid options for "container_format":
cf_list = ['ami', 'ari', 'aki', 'bare', 'ovf']
# valid options for "disk_format":
df_list = [
'ami', 'ari', 'aki', 'vhd', 'vmdk', 'raw', 'qcow2', 'vdi', 'iso'
]
# 'location' and 'visibility' are the parameters used in
# Glance API v2. For now we have to use v1 for now (see below)
# but this modules interface will change in Carbon.
if copy_from is not None or is_public is not None:
warn_until(
'Carbon', 'The parameters \'copy_from\' and '
'\'is_public\' are deprecated and will be removed. '
'Use \'location\' and \'visibility\' instead.')
if is_public is not None and visibility is not None:
raise SaltInvocationError('Must only specify one of '
'\'is_public\' and \'visibility\'')
if copy_from is not None and location is not None:
raise SaltInvocationError('Must only specify one of '
'\'copy_from\' and \'location\'')
if copy_from is not None:
kwargs['copy_from'] = copy_from
else:
kwargs['copy_from'] = location
if is_public is not None:
kwargs['is_public'] = is_public
elif visibility is not None:
if visibility not in v_list:
raise SaltInvocationError(
'"visibility" needs to be one ' +
'of the following: {0}'.format(', '.join(v_list)))
elif visibility == 'public':
kwargs['is_public'] = True
else:
kwargs['is_public'] = False
else:
kwargs['is_public'] = True
if container_format not in cf_list:
raise SaltInvocationError(
'"container_format" needs to be ' +
'one of the following: {0}'.format(', '.join(cf_list)))
else:
kwargs['container_format'] = container_format
if disk_format not in df_list:
raise SaltInvocationError(
'"disk_format" needs to be one ' +
'of the following: {0}'.format(', '.join(df_list)))
else:
kwargs['disk_format'] = disk_format
if protected is not None:
kwargs['protected'] = protected
# Icehouse's glanceclient doesn't have add_location() and
# glanceclient.v2 doesn't implement Client.images.create()
# in a usable fashion. Thus we have to use v1 for now.
g_client = _auth(profile, api_version=1)
image = g_client.images.create(name=name, **kwargs)
return image_show(image.id)
def image_present(name, visibility='public', protected=None,
checksum=None, location=None, wait_for=None, timeout=30):
'''
Checks if given image is present with properties
set as specified.
An image should got through the stages 'queued', 'saving'
before becoming 'active'. The attribute 'checksum' can
only be checked once the image is active.
If you don't specify 'wait_for' but 'checksum' the function
will wait for the image to become active before comparing
checksums. If you don't specify checksum either the function
will return when the image reached 'saving'.
The default timeout for both is 30 seconds.
Supported properties:
- visibility ('public' or 'private')
- protected (bool)
- checksum (string, md5sum)
- location (URL, to copy from)
'''
ret = {'name': name,
'changes': {},
'result': True,
'comment': '',
}
acceptable = ['queued', 'saving', 'active']
if wait_for is None and checksum is None:
wait_for = 'saving'
elif wait_for is None and checksum is not None:
wait_for = 'active'
# Just pop states until we reach the
# first acceptable one:
while len(acceptable) > 1:
if acceptable[0] == wait_for:
break
else:
acceptable.pop(0)
image, msg = _find_image(name)
if image is False:
if __opts__['test']:
ret['result'] = None
else:
ret['result'] = False
ret['comment'] = msg
return ret
log.debug(msg)
# No image yet and we know where to get one
if image is None and location is not None:
if __opts__['test']:
ret['result'] = None
ret['comment'] = 'glance.image_present would ' \
'create an image from {0}'.format(location)
return ret
image = __salt__['glance.image_create'](name=name,
protected=protected, visibility=visibility,
location=location)
# See Salt issue #24568
warn_until('Boron', 'Starting with Boron '
'\'glance.image_create\' is not supposed to return '
'the image wrapped in a dict anymore.')
if len(image.keys()) == 1:
image = image.values()[0]
log.debug('Created new image:\n{0}'.format(image))
ret['changes'] = {
name:
{
'new':
{
'id': image['id']
},
'old': None
}
}
timer = timeout
# Kinda busy-loopy but I don't think the Glance
# API has events we can listen for
while timer > 0:
if 'status' in image and \
image['status'] in acceptable:
log.debug('Image {0} has reached status {1}'.format(
image['name'], image['status']))
break
else:
timer -= 5
time.sleep(5)
image, msg = _find_image(name)
if not image:
ret['result'] = False
ret['comment'] += 'Created image {0} '.format(
name) + ' vanished:\n' + msg
return ret
elif len(image.keys()) == 1:
# See Salt issue #24568
warn_until('Boron', 'Starting with Boron '
'\'_find_image()\' is not supposed to return '
'the image wrapped in a dict anymore.')
image = image.values()[0]
if timer <= 0 and image['status'] not in acceptable:
ret['result'] = False
ret['comment'] += 'Image didn\'t reach an acceptable '+\
'state ({0}) before timeout:\n'.format(acceptable)+\
'\tLast status was "{0}".\n'.format(image['status'])
# See Salt issue #24568
warn_until('Boron', 'Starting with Boron '
'\'_find_image()\' is not supposed to return '
'the image wrapped in a dict anymore.')
if len(image.keys()) == 1:
image = image.values()[0]
# ret[comment] +=
# There's no image but where would I get one??
elif location is None:
if __opts__['test']:
ret['result'] = None
ret['comment'] = 'No location to copy image from specified,\n' +\
'glance.image_present would not create one'
else:
ret['result'] = False
ret['comment'] = 'No location to copy image from specified,\n' +\
'not creating a new image.'
return ret
# If we've created a new image also return its last status:
if name in ret['changes']:
ret['changes'][name]['new']['status'] = image['status']
if visibility:
if image['visibility'] != visibility:
old_value = image['visibility']
if not __opts__['test']:
image = __salt__['glance.image_update'](
id=image['id'], visibility=visibility)
# See Salt issue #24568
warn_until('Boron', 'Starting with Boron '
'\'glance.image_update\' is not supposed to return '
'the image wrapped in a dict anymore.')
if len(image.keys()) == 1:
image = image.values()[0]
# Check if image_update() worked:
if image['visibility'] != visibility:
if not __opts__['test']:
ret['result'] = False
elif __opts__['test']:
ret['result'] = None
ret['comment'] += '"visibility" is {0}, '\
'should be {1}.\n'.format(image['visibility'],
visibility)
else:
if 'new' in ret['changes']:
ret['changes']['new']['visibility'] = visibility
else:
ret['changes']['new'] = {'visibility': visibility}
if 'old' in ret['changes']:
ret['changes']['old']['visibility'] = old_value
else:
ret['changes']['old'] = {'visibility': old_value}
else:
ret['comment'] += '"visibility" is correct ({0}).\n'.format(
visibility)
if protected is not None:
if not isinstance(protected, bool) or image['protected'] ^ protected:
if not __opts__['test']:
ret['result'] = False
else:
ret['result'] = None
ret['comment'] += '"protected" is {0}, should be {1}.\n'.format(
image['protected'], protected)
else:
ret['comment'] += '"protected" is correct ({0}).\n'.format(
protected)
if 'status' in image and checksum:
if image['status'] == 'active':
if 'checksum' not in image:
# Refresh our info about the image
image = __salt__['glance.image_show'](image['id'])
warn_until('Boron', 'Starting with Boron '
'\'glance.image_show\' is not supposed to return '
'the image wrapped in a dict anymore.')
if len(image.keys()) == 1:
image = image.values()[0]
if 'checksum' not in image:
if not __opts__['test']:
ret['result'] = False
else:
ret['result'] = None
ret['comment'] += 'No checksum available for this image:\n' +\
'\tImage has status "{0}".'.format(image['status'])
elif image['checksum'] != checksum:
if not __opts__['test']:
ret['result'] = False
else:
ret['result'] = None
ret['comment'] += '"checksum" is {0}, should be {1}.\n'.format(
image['checksum'], checksum)
else:
ret['comment'] += '"checksum" is correct ({0}).\n'.format(
checksum)
elif image['status'] in ['saving', 'queued']:
ret['comment'] += 'Checksum won\'t be verified as image ' +\
'hasn\'t reached\n\t "status=active" yet.\n'
log.debug('glance.image_present will return: {0}'.format(ret))
return ret
def test_warn_until_warning_raised(self):
# We *always* want *all* warnings thrown on this module
warnings.filterwarnings('always', '', DeprecationWarning, __name__)
def raise_warning(_version_info_=(0, 16, 0)):
warn_until((0, 17),
'Deprecation Message!',
_version_info_=_version_info_)
def raise_named_version_warning(_version_info_=(0, 16, 0)):
warn_until('Hydrogen',
'Deprecation Message!',
_version_info_=_version_info_)
# raise_warning should show warning until version info is >= (0, 17)
with warnings.catch_warnings(record=True) as recorded_warnings:
raise_warning()
self.assertEqual('Deprecation Message!',
str(recorded_warnings[0].message))
# raise_warning should show warning until version info is >= (0, 17)
with warnings.catch_warnings(record=True) as recorded_warnings:
raise_named_version_warning()
self.assertEqual('Deprecation Message!',
str(recorded_warnings[0].message))
# the deprecation warning is not issued because we passed
# _dont_call_warning
with warnings.catch_warnings(record=True) as recorded_warnings:
warn_until((0, 17),
'Foo',
_dont_call_warnings=True,
_version_info_=(0, 16))
self.assertEqual(0, len(recorded_warnings))
# Let's set version info to (0, 17), a RuntimeError should be raised
with self.assertRaisesRegexp(
RuntimeError,
r'The warning triggered on filename \'(.*)warnings_test.py\', '
r'line number ([\d]+), is supposed to be shown until version '
r'\'0.17.0\' is released. Current version is now \'0.17.0\'. '
r'Please remove the warning.'):
raise_warning(_version_info_=(0, 17, 0))
# Let's set version info to (0, 17), a RuntimeError should be raised
with self.assertRaisesRegexp(
RuntimeError,
r'The warning triggered on filename \'(.*)warnings_test.py\', '
r'line number ([\d]+), is supposed to be shown until version '
r'\'Hydrogen((.*))\' is released. Current version is now '
r'\'([\d.]+)\'. Please remove the warning.'):
raise_named_version_warning(_version_info_=(sys.maxint, 16, 0))
# Even though we're calling warn_until, we pass _dont_call_warnings
# because we're only after the RuntimeError
with self.assertRaisesRegexp(
RuntimeError,
r'The warning triggered on filename \'(.*)warnings_test.py\', '
r'line number ([\d]+), is supposed to be shown until version '
r'\'0.17.0\' is released. Current version is now \'0.17.0\'. '
r'Please remove the warning.'):
warn_until((0, 17), 'Foo', _dont_call_warnings=True)
with self.assertRaisesRegexp(
RuntimeError,
r'The warning triggered on filename \'(.*)warnings_test.py\', '
r'line number ([\d]+), is supposed to be shown until version '
r'\'Hydrogen((.*))\' is released. Current version is now '
r'\'([\d.]+)\'. Please remove the warning.'):
warn_until('Hydrogen',
'Foo',
_dont_call_warnings=True,
_version_info_=(sys.maxint, 16, 0))
# version on the deprecation message gets properly formatted
with warnings.catch_warnings(record=True) as recorded_warnings:
vrs = SaltStackVersion.from_name('Helium')
warn_until('Helium',
'Deprecation Message until {version}!',
_version_info_=(vrs.major - 1, 0))
self.assertEqual(
'Deprecation Message until {0}!'.format(vrs.formatted_version),
str(recorded_warnings[0].message))
def extracted(name,
source,
archive_format,
archive_user=None,
user=None,
group=None,
tar_options=None,
source_hash=None,
if_missing=None,
keep=False,
trim_output=False,
source_hash_update=None):
'''
.. versionadded:: 2014.1.0
State that make sure an archive is extracted in a directory.
The downloaded archive is erased if successfully extracted.
The archive is downloaded only if necessary.
.. note::
If ``if_missing`` is not defined, this state will check for ``name``
instead. If ``name`` exists, it will assume the archive was previously
extracted successfully and will not extract it again.
Example, tar with flag for lmza compression:
.. code-block:: yaml
graylog2-server:
archive.extracted:
- name: /opt/
- source: https://github.com/downloads/Graylog2/graylog2-server/graylog2-server-0.9.6p1.tar.lzma
- source_hash: md5=499ae16dcae71eeb7c3a30c75ea7a1a6
- tar_options: J
- archive_format: tar
- if_missing: /opt/graylog2-server-0.9.6p1/
Example, tar with flag for verbose output:
.. code-block:: yaml
graylog2-server:
archive.extracted:
- name: /opt/
- source: https://github.com/downloads/Graylog2/graylog2-server/graylog2-server-0.9.6p1.tar.gz
- source_hash: md5=499ae16dcae71eeb7c3a30c75ea7a1a6
- archive_format: tar
- tar_options: v
- user: root
- group: root
- if_missing: /opt/graylog2-server-0.9.6p1/
Example, tar with flag for lmza compression and update based if source_hash differs from what was
previously extracted:
.. code-block:: yaml
graylog2-server:
archive.extracted:
- name: /opt/
- source: https://github.com/downloads/Graylog2/graylog2-server/graylog2-server-0.9.6p1.tar.lzma
- source_hash: md5=499ae16dcae71eeb7c3a30c75ea7a1a6
- source_hash_update: true
- tar_options: J
- archive_format: tar
- if_missing: /opt/graylog2-server-0.9.6p1/
name
Directory name where to extract the archive
source
Archive source, same syntax as file.managed source argument.
source_hash
Hash of source file, or file with list of hash-to-file mappings.
It uses the same syntax as the file.managed source_hash argument.
source_hash_update
Set this to true if archive should be extracted if source_hash has
changed. This would extract regardless of the `if_missing`
parameter.
archive_format
tar, zip or rar
archive_user
The user to own each extracted file.
.. deprecated:: Boron
replaced by standardized `user` parameter.
user
The user to own each extracted file.
.. versionadded:: 2015.8.0
group
The group to own each extracted file.
.. versionadded:: 2015.8.0
if_missing
Some archives, such as tar, extract themselves in a subfolder.
This directive can be used to validate if the archive had been
previously extracted.
tar_options
Required if used with ``archive_format: tar``, otherwise optional.
It needs to be the tar argument specific to the archive being extracted,
such as 'J' for LZMA or 'v' to verbosely list files processed.
Using this option means that the tar executable on the target will
be used, which is less platform independent.
Main operators like -x, --extract, --get, -c and -f/--file
**should not be used** here.
If ``archive_format`` is ``zip`` or ``rar`` and this option is not set,
then the Python tarfile module is used. The tarfile module supports gzip
and bz2 in Python 2.
keep
Keep the archive in the minion's cache
trim_output
The number of files we should output on success before the rest are trimmed, if this is
set to True then it will default to 100
'''
ret = {'name': name, 'result': None, 'changes': {}, 'comment': ''}
valid_archives = ('tar', 'rar', 'zip')
if archive_format not in valid_archives:
ret['result'] = False
ret['comment'] = '{0} is not supported, valid formats are: {1}'.format(
archive_format, ','.join(valid_archives))
return ret
# remove this whole block after formal deprecation.
if archive_user is not None:
warn_until(
'Boron',
'Passing \'archive_user\' is deprecated.'
'Pass \'user\' instead.'
)
if user is None:
user = archive_user
if not name.endswith('/'):
name += '/'
if if_missing is None:
if_missing = name
if source_hash and source_hash_update:
hash = source_hash.split("=")
source_file = '{0}.{1}'.format(os.path.basename(source), hash[0])
hash_fname = os.path.join(__opts__['cachedir'],
'files',
__env__,
source_file)
if compareChecksum(hash_fname, name, hash[1]):
ret['result'] = True
ret['comment'] = 'Hash {0} has not changed'.format(hash[1])
return ret
elif (
__salt__['file.directory_exists'](if_missing)
or __salt__['file.file_exists'](if_missing)
):
ret['result'] = True
ret['comment'] = '{0} already exists'.format(if_missing)
return ret
log.debug('Input seem valid so far')
filename = os.path.join(__opts__['cachedir'],
'files',
__env__,
'{0}.{1}'.format(if_missing.replace('/', '_'),
archive_format))
if not os.path.exists(filename):
if __opts__['test']:
ret['result'] = None
ret['comment'] = \
'Archive {0} would have been downloaded in cache'.format(source)
return ret
log.debug('Archive file {0} is not in cache, download it'.format(source))
file_result = __salt__['state.single']('file.managed',
filename,
source=source,
source_hash=source_hash,
makedirs=True,
saltenv=__env__)
log.debug('file.managed: {0}'.format(file_result))
# get value of first key
try:
file_result = file_result[next(six.iterkeys(file_result))]
except AttributeError:
pass
try:
if not file_result['result']:
log.debug('failed to download {0}'.format(source))
return file_result
except TypeError:
if not file_result:
log.debug('failed to download {0}'.format(source))
return file_result
else:
log.debug('Archive file {0} is already in cache'.format(name))
if __opts__['test']:
ret['result'] = None
ret['comment'] = 'Archive {0} would have been extracted in {1}'.format(
source, name)
return ret
__salt__['file.makedirs'](name, user=user, group=group)
log.debug('Extract {0} in {1}'.format(filename, name))
if archive_format == 'zip':
files = __salt__['archive.unzip'](filename, name, trim_output=trim_output)
elif archive_format == 'rar':
files = __salt__['archive.unrar'](filename, name, trim_output=trim_output)
else:
if tar_options is None:
with closing(tarfile.open(filename, 'r')) as tar:
files = tar.getnames()
tar.extractall(name)
else:
tar_opts = tar_options.split(' ')
tar_cmd = ['tar']
tar_shortopts = 'x'
tar_longopts = []
for position, opt in enumerate(tar_opts):
if opt.startswith('-'):
tar_longopts.append(opt)
else:
if position > 0:
tar_longopts.append(opt)
else:
append_opt = opt
append_opt = append_opt.replace('x', '').replace('f', '')
tar_shortopts = tar_shortopts + append_opt
tar_cmd.append(tar_shortopts)
tar_cmd.extend(tar_longopts)
tar_cmd.extend(['-f', filename])
results = __salt__['cmd.run_all'](tar_cmd, cwd=name, python_shell=False)
if results['retcode'] != 0:
ret['result'] = False
ret['changes'] = results
return ret
if 'bsdtar' in __salt__['cmd.run']('tar --version', python_shell=False):
files = results['stderr']
else:
files = results['stdout']
if not files:
files = 'no tar output so far'
# Recursively set user and group ownership of files after extraction.
# Note: We do this here because we might not have access to the cachedir.
if user or group:
dir_result = __salt__['state.single']('file.directory',
name,
user=user,
group=group,
recurse=['user', 'group'])
log.debug('file.directory: {0}'.format(dir_result))
if len(files) > 0:
ret['result'] = True
ret['changes']['directories_created'] = [name]
if if_missing != name:
ret['changes']['directories_created'].append(if_missing)
ret['changes']['extracted_files'] = files
ret['comment'] = '{0} extracted in {1}'.format(source, name)
if not keep:
os.unlink(filename)
if source_hash and source_hash_update:
updateChecksum(hash_fname, name, hash[1])
else:
__salt__['file.remove'](if_missing)
ret['result'] = False
ret['comment'] = 'Can\'t extract content of {0}'.format(source)
return ret
def raise_warning():
warn_until((0, 17), 'Deprecation Message!')
def test_warn_until_warning_raised(self):
# We *always* want *all* warnings thrown on this module
warnings.filterwarnings('always', '', DeprecationWarning, __name__)
def raise_warning(_version_info_=(0, 16, 0)):
warn_until(
(0, 17), 'Deprecation Message!',
_version_info_=_version_info_
)
def raise_named_version_warning(_version_info_=(0, 16, 0)):
warn_until(
'Hydrogen', 'Deprecation Message!',
_version_info_=_version_info_
)
# raise_warning should show warning until version info is >= (0, 17)
with warnings.catch_warnings(record=True) as recorded_warnings:
raise_warning()
self.assertEqual(
'Deprecation Message!', str(recorded_warnings[0].message)
)
# raise_warning should show warning until version info is >= (0, 17)
with warnings.catch_warnings(record=True) as recorded_warnings:
raise_named_version_warning()
self.assertEqual(
'Deprecation Message!', str(recorded_warnings[0].message)
)
# the deprecation warning is not issued because we passed
# _dont_call_warning
with warnings.catch_warnings(record=True) as recorded_warnings:
warn_until(
(0, 17), 'Foo', _dont_call_warnings=True,
_version_info_=(0, 16)
)
self.assertEqual(0, len(recorded_warnings))
# Let's set version info to (0, 17), a RuntimeError should be raised
with self.assertRaisesRegexp(
RuntimeError,
r'The warning triggered on filename \'(.*)warnings_test.py\', '
r'line number ([\d]+), is supposed to be shown until version '
r'\'0.17.0\' is released. Current version is now \'0.17.0\'. '
r'Please remove the warning.'):
raise_warning(_version_info_=(0, 17, 0))
# Let's set version info to (0, 17), a RuntimeError should be raised
with self.assertRaisesRegexp(
RuntimeError,
r'The warning triggered on filename \'(.*)warnings_test.py\', '
r'line number ([\d]+), is supposed to be shown until version '
r'\'Hydrogen((.*))\' is released. Current version is now '
r'\'([\d.]+)\'. Please remove the warning.'):
raise_named_version_warning(_version_info_=(sys.maxint, 16, 0))
# Even though we're calling warn_until, we pass _dont_call_warnings
# because we're only after the RuntimeError
with self.assertRaisesRegexp(
RuntimeError,
r'The warning triggered on filename \'(.*)warnings_test.py\', '
r'line number ([\d]+), is supposed to be shown until version '
r'\'0.17.0\' is released. Current version is now '
r'\'([\d.]+)\'. Please remove the warning.'):
warn_until(
(0, 17), 'Foo', _dont_call_warnings=True
)
with self.assertRaisesRegexp(
RuntimeError,
r'The warning triggered on filename \'(.*)warnings_test.py\', '
r'line number ([\d]+), is supposed to be shown until version '
r'\'Hydrogen((.*))\' is released. Current version is now '
r'\'([\d.]+)\'. Please remove the warning.'):
warn_until(
'Hydrogen', 'Foo', _dont_call_warnings=True,
_version_info_=(sys.maxint, 16, 0)
)
# version on the deprecation message gets properly formatted
with warnings.catch_warnings(record=True) as recorded_warnings:
vrs = SaltStackVersion.from_name('Helium')
warn_until(
'Helium', 'Deprecation Message until {version}!',
_version_info_=(vrs.major - 1, 0)
)
self.assertEqual(
'Deprecation Message until {0}!'.format(vrs.formatted_version),
str(recorded_warnings[0].message)
)
def extracted(name,
source,
archive_format,
archive_user=None,
password=None,
user=None,
group=None,
tar_options=None,
source_hash=None,
if_missing=None,
keep=False,
trim_output=False,
skip_verify=False,
source_hash_update=None):
'''
.. versionadded:: 2014.1.0
State that make sure an archive is extracted in a directory.
The downloaded archive is erased if successfully extracted.
The archive is downloaded only if necessary.
.. note::
If ``if_missing`` is not defined, this state will check for ``name``
instead. If ``name`` exists, it will assume the archive was previously
extracted successfully and will not extract it again.
Example, tar with flag for lmza compression:
.. code-block:: yaml
graylog2-server:
archive.extracted:
- name: /opt/
- source: https://github.com/downloads/Graylog2/graylog2-server/graylog2-server-0.9.6p1.tar.lzma
- source_hash: md5=499ae16dcae71eeb7c3a30c75ea7a1a6
- tar_options: J
- archive_format: tar
- if_missing: /opt/graylog2-server-0.9.6p1/
Example, tar with flag for verbose output:
.. code-block:: yaml
graylog2-server:
archive.extracted:
- name: /opt/
- source: https://github.com/downloads/Graylog2/graylog2-server/graylog2-server-0.9.6p1.tar.gz
- source_hash: md5=499ae16dcae71eeb7c3a30c75ea7a1a6
- archive_format: tar
- tar_options: v
- user: root
- group: root
- if_missing: /opt/graylog2-server-0.9.6p1/
Example, tar with flag for lmza compression and update based if source_hash differs from what was
previously extracted:
.. code-block:: yaml
graylog2-server:
archive.extracted:
- name: /opt/
- source: https://github.com/downloads/Graylog2/graylog2-server/graylog2-server-0.9.6p1.tar.lzma
- source_hash: md5=499ae16dcae71eeb7c3a30c75ea7a1a6
- source_hash_update: true
- tar_options: J
- archive_format: tar
- if_missing: /opt/graylog2-server-0.9.6p1/
name
Location where archive should be extracted
password
Password to use with password protected zip files. Currently only zip
files with passwords are supported.
.. versionadded:: 2016.3.0
source
Archive source, same syntax as file.managed source argument.
source_hash
Hash of source file, or file with list of hash-to-file mappings.
It uses the same syntax as the file.managed source_hash argument.
source_hash_update
Set this to ``True`` if archive should be extracted if source_hash has
changed. This would extract regardless of the ``if_missing`` parameter.
.. versionadded:: 2016.3.0
skip_verify:False
If ``True``, hash verification of remote file sources (``http://``,
``https://``, ``ftp://``) will be skipped, and the ``source_hash``
argument will be ignored.
.. versionadded:: 2016.3.4
archive_format
``tar``, ``zip`` or ``rar``
archive_user
The user to own each extracted file.
.. deprecated:: 2014.7.2
Replaced by ``user`` parameter
user
The user to own each extracted file.
.. versionadded:: 2015.8.0
.. versionchanged:: 2016.3.0
When used in combination with ``if_missing``, ownership will only
be enforced if ``if_missing`` is a directory.
group
The group to own each extracted file.
.. versionadded:: 2015.8.0
.. versionchanged:: 2016.3.0
When used in combination with ``if_missing``, ownership will only
be enforced if ``if_missing`` is a directory.
if_missing
If specified, this path will be checked, and if it exists then the
archive will not be extracted. This can be helpful if the archive
extracts all files into a subfolder. This path can be either a
directory or a file, so this option can also be used to check for a
semaphore file and conditionally skip extraction.
.. versionchanged:: 2016.3.0
When used in combination with either ``user`` or ``group``,
ownership will only be enforced when ``if_missing`` is a directory.
tar_options
If ``archive_format`` is set to ``tar``, this option can be used to
specify a string of additional arguments to pass to the tar command. If
``archive_format`` is set to ``tar`` and this option is *not* used,
then the minion will attempt to use Python's native tarfile_ support to
extract it. Python's native tarfile_ support can only handle gzip and
bzip2 compression, however.
.. versionchanged:: 2015.8.11,2016.3.2
XZ-compressed archives no longer require ``J`` to manually be set
in the ``tar_options``, they are now detected automatically and
Salt will extract them using ``xz-utils``. This is a more
platform-independent solution, as not all tar implementations
support the ``J`` argument for extracting archives.
.. note::
Main operators like -x, --extract, --get, -c and -f/--file **should
not be used** here.
Using this option means that the ``tar`` command will be used,
which is less platform-independent, so keep this in mind when using
this option; the options must be valid options for the ``tar``
implementation on the minion's OS.
.. _tarfile: https://docs.python.org/2/library/tarfile.html
keep
Keep the archive in the minion's cache
trim_output
The number of files we should output on success before the rest are
trimmed, if this is set to True then it will default to 100
.. versionadded:: 2016.3.0
'''
ret = {'name': name, 'result': None, 'changes': {}, 'comment': ''}
valid_archives = ('tar', 'rar', 'zip')
if archive_format not in valid_archives:
ret['result'] = False
ret['comment'] = '{0} is not supported, valid formats are: {1}'.format(
archive_format, ','.join(valid_archives))
return ret
# remove this whole block after formal deprecation.
if archive_user is not None:
warn_until(
'Carbon',
'Passing \'archive_user\' is deprecated.'
'Pass \'user\' instead.'
)
if user is None:
user = archive_user
if not name.endswith('/'):
name += '/'
if __opts__['test']:
source_match = source
else:
try:
source_match = __salt__['file.source_list'](source,
source_hash,
__env__)[0]
except CommandExecutionError as exc:
ret['result'] = False
ret['comment'] = exc.strerror
return ret
urlparsed_source = _urlparse(source_match)
source_hash_name = urlparsed_source.path or urlparsed_source.netloc
source_is_local = urlparsed_source.scheme in ('', 'file')
if source_is_local:
# Get rid of "file://" from start of source_match
source_match = urlparsed_source.path
if not os.path.isfile(source_match):
ret['comment'] = 'Source file \'{0}\' does not exist'.format(source_match)
return ret
if if_missing is None:
if_missing = name
if source_hash and source_hash_update:
if urlparsed_source.scheme != '':
ret['result'] = False
ret['comment'] = (
'\'source_hash_update\' is not yet implemented for a remote '
'source_hash'
)
return ret
else:
try:
hash_type, hsum = source_hash.split('=')
except ValueError:
ret['result'] = False
ret['comment'] = 'Invalid source_hash format'
return ret
source_file = '{0}.{1}'.format(os.path.basename(source), hash_type)
hash_fname = os.path.join(__opts__['cachedir'],
'files',
__env__,
source_file)
if _compare_checksum(hash_fname, name, hsum):
ret['result'] = True
ret['comment'] = 'Hash {0} has not changed'.format(hsum)
return ret
elif (
__salt__['file.directory_exists'](if_missing)
or __salt__['file.file_exists'](if_missing)
):
ret['result'] = True
ret['comment'] = '{0} already exists'.format(if_missing)
return ret
log.debug('Input seem valid so far')
if source_is_local:
filename = source_match
else:
filename = os.path.join(
__opts__['cachedir'],
'files',
__env__,
'{0}.{1}'.format(re.sub('[:/\\\\]', '_', if_missing), archive_format))
if not source_is_local and not os.path.isfile(filename):
if __opts__['test']:
ret['result'] = None
ret['comment'] = \
'{0} {1} would be downloaded to cache'.format(
'One of' if not isinstance(source_match, six.string_types)
else 'Archive',
source_match
)
return ret
log.debug('%s is not in cache, downloading it', source_match)
file_result = __states__['file.managed'](filename,
source=source_match,
source_hash=source_hash,
makedirs=True,
skip_verify=skip_verify,
source_hash_name=source_hash_name)
log.debug('file.managed: {0}'.format(file_result))
# get value of first key
try:
file_result = file_result[next(six.iterkeys(file_result))]
except AttributeError:
pass
try:
if not file_result['result']:
log.debug('failed to download {0}'.format(source))
return file_result
except TypeError:
if not file_result:
log.debug('failed to download {0}'.format(source))
return file_result
else:
log.debug('Archive %s is already in cache', source)
if __opts__['test']:
ret['result'] = None
ret['comment'] = '{0} {1} would be extracted to {2}'.format(
'One of' if not isinstance(source_match, six.string_types)
else 'Archive',
source_match,
name
)
return ret
created_destdir = False
if __salt__['file.file_exists'](name.rstrip('/')):
ret['result'] = False
ret['comment'] = ('{0} exists and is not a directory'
.format(name.rstrip('/')))
return ret
elif not __salt__['file.directory_exists'](name):
__salt__['file.makedirs'](name, user=archive_user)
created_destdir = True
log.debug('Extracting {0} to {1}'.format(filename, name))
if archive_format == 'zip':
if password is None and salt.utils.which('unzip'):
files = __salt__['archive.cmd_unzip'](filename, name, trim_output=trim_output)
else:
# https://bugs.python.org/issue15795
if password is not None:
log.warning('Password supplied: using archive.unzip')
if not salt.utils.which('unzip'):
log.warning('Cannot find unzip command for archive.cmd_unzip:'
' using archive.unzip instead')
files = __salt__['archive.unzip'](filename, name, trim_output=trim_output, password=password)
elif archive_format == 'rar':
files = __salt__['archive.unrar'](filename, name, trim_output=trim_output)
else:
if tar_options is None:
try:
with closing(tarfile.open(filename, 'r')) as tar:
files = tar.getnames()
tar.extractall(name)
except tarfile.ReadError:
if salt.utils.which('xz'):
if __salt__['cmd.retcode'](['xz', '-l', filename],
python_shell=False,
ignore_retcode=True) == 0:
# XZ-compressed data
log.debug(
'Tar file is XZ-compressed, attempting '
'decompression and extraction using xz-utils '
'and the tar command'
)
# Must use python_shell=True here because not all tar
# implementations support the -J flag for decompressing
# XZ-compressed data. We need to dump the decompressed
# data to stdout and pipe it to tar for extraction.
cmd = 'xz --decompress --stdout {0} | tar xvf -'
results = __salt__['cmd.run_all'](
cmd.format(_cmd_quote(filename)),
cwd=name,
python_shell=True)
if results['retcode'] != 0:
if created_destdir:
_cleanup_destdir(name)
ret['result'] = False
ret['changes'] = results
return ret
if _is_bsdtar():
files = results['stderr']
else:
files = results['stdout']
else:
# Failed to open tar archive and it is not
# XZ-compressed, gracefully fail the state
if created_destdir:
_cleanup_destdir(name)
ret['result'] = False
ret['comment'] = (
'Failed to read from tar archive using Python\'s '
'native tar file support. If archive is '
'compressed using something other than gzip or '
'bzip2, the \'tar_options\' parameter may be '
'required to pass the correct options to the tar '
'command in order to extract the archive.'
)
return ret
else:
if created_destdir:
_cleanup_destdir(name)
ret['result'] = False
ret['comment'] = (
'Failed to read from tar archive. If it is '
'XZ-compressed, install xz-utils to attempt '
'extraction.'
)
return ret
else:
try:
tar_opts = tar_options.split(' ')
except AttributeError:
tar_opts = str(tar_options).split(' ')
tar_cmd = ['tar']
tar_shortopts = 'x'
tar_longopts = []
for position, opt in enumerate(tar_opts):
if opt.startswith('-'):
tar_longopts.append(opt)
else:
if position > 0:
tar_longopts.append(opt)
else:
append_opt = opt
append_opt = append_opt.replace('x', '').replace('f', '')
tar_shortopts = tar_shortopts + append_opt
tar_cmd.append(tar_shortopts)
tar_cmd.extend(tar_longopts)
tar_cmd.extend(['-f', filename])
results = __salt__['cmd.run_all'](tar_cmd, cwd=name, python_shell=False)
if results['retcode'] != 0:
ret['result'] = False
ret['changes'] = results
return ret
if _is_bsdtar():
files = results['stderr']
else:
files = results['stdout']
if not files:
files = 'no tar output so far'
# Recursively set user and group ownership of files after extraction.
# Note: We do this here because we might not have access to the cachedir.
if user or group:
if os.path.isdir(if_missing):
recurse = []
if user:
recurse.append('user')
if group:
recurse.append('group')
dir_result = __states__['file.directory'](if_missing,
user=user,
group=group,
recurse=recurse)
log.debug('file.directory: %s', dir_result)
elif os.path.isfile(if_missing):
log.debug('if_missing (%s) is a file, not enforcing user/group '
'permissions', if_missing)
if len(files) > 0:
ret['result'] = True
ret['changes']['directories_created'] = [name]
ret['changes']['extracted_files'] = files
ret['comment'] = '{0} extracted to {1}'.format(source_match, name)
if not source_is_local and not keep:
os.unlink(filename)
if source_hash and source_hash_update:
_update_checksum(hash_fname, name, hash[1])
else:
__salt__['file.remove'](if_missing)
ret['result'] = False
ret['comment'] = 'Can\'t extract content of {0}'.format(source_match)
return ret
def raise_warning(_version_info_=(0, 16, 0)):
warn_until(
(0, 17), 'Deprecation Message!',
_version_info_=_version_info_
)
def raise_named_version_warning(_version_info_=(0, 16, 0)):
warn_until('Hydrogen',
'Deprecation Message!',
_version_info_=_version_info_)
def raise_warning(_version_info_=(0, 16, 0)):
warn_until((0, 17),
'Deprecation Message!',
_version_info_=_version_info_)
