def _have_error(reply, netconf_namespace):
# error check
# https://tools.ietf.org/html/rfc6241#section-4.3
error = None
if 'rpc-error' in reply:
errors = reply['rpc-error']
elif (netconf_namespace + '@rpc-error') in reply:
# default namespace can't be not netconf 1.0
errors = reply[netconf_namespace + '@rpc-error']
else:
return
if not isinstance(errors, list):
# case when we have only one error struct
errors = [errors]
for error in errors:
if not error:
# we have empty rpc-error?
raise cfy_exc.RecoverableError("Empty error struct:" + str(errors))
# server can send warning as error, lets check
error_severity = 'error'
if 'error-severity' in error:
error_severity = error['error-severity']
elif (netconf_namespace + '@error-severity') in error:
error_severity = error[netconf_namespace + '@error-severity']
if error_severity != 'warning':
raise cfy_exc.RecoverableError("We have error in reply" +
str(errors))
def _cleanup_response(self, text, prefix, warning_examples, error_examples,
critical_examples):
if (not error_examples and not warning_examples
and not critical_examples):
return text.strip()
# check command echo
have_correct_prefix = False
prefix_pos = text.find(prefix)
if prefix_pos == -1:
if self.logger:
self.logger.debug("Have not found '%s' in response: '%s'" %
(prefix, repr(text)))
else:
if text[:prefix_pos].strip():
if self.logger:
self.logger.debug(
"Some mess before '%s' in response: '%s'" %
(prefix, repr(text)))
else:
have_correct_prefix = True
if have_correct_prefix:
# looks as we have correct line
response = text[prefix_pos + len(prefix):]
else:
# skip first line(where must be echo from commands input)
if "\n" in text:
response = text[text.find("\n"):]
else:
response = text
# check for warnings started only from new line
if warning_examples:
warnings_with_new_line = [
"\n" + warning for warning in warning_examples
]
if self._find_any_in(response, warnings_with_new_line) != -1:
# close is not needed, we will rerun later
raise RecoverableWarning(
"Looks as we have warning in response: %s" % (text))
# check for errors started only from new line
if error_examples:
errors_with_new_line = ["\n" + error for error in error_examples]
if self._find_any_in(response, errors_with_new_line) != -1:
if not self.is_closed():
self.close()
raise cfy_exc.RecoverableError(
"Looks as we have error in response: %s" % (text))
# check for criticals started only from new line
if critical_examples:
criticals_with_new_line = [
"\n" + critical for critical in critical_examples
]
if self._find_any_in(response, criticals_with_new_line) != -1:
if not self.is_closed():
self.close()
raise cfy_exc.NonRecoverableError(
"Looks as we have critical in response: %s" % (text))
return response.strip()
def link(**kwargs):
vm_id = ctx.source.instance.runtime_properties.get('resource_id')
net_id = ctx.target.instance.runtime_properties.get('resource_id')
ctx.logger.info('Link network: {} to VM: {}.'.format(
repr(net_id), repr(vm_id)))
libvirt_auth = ctx.target.instance.runtime_properties.get('libvirt_auth')
conn = libvirt.open(libvirt_auth)
if conn is None:
raise cfy_exc.NonRecoverableError(
'Failed to open connection to the hypervisor')
# lookup the default network by name
network = conn.networkLookupByName(net_id)
if network is None:
raise cfy_exc.NonRecoverableError('Failed to find the network')
MAX_RETRY = 10
for i in xrange(MAX_RETRY):
ctx.logger.info("{}: Tring to get vm ip: {}/{}".format(
vm_id, i, MAX_RETRY))
for lease in network.DHCPLeases():
vm_params = ctx.source.instance.runtime_properties.get(
'params', {})
for vm_network in vm_params.get("networks", []):
if vm_network.get('mac') == lease.get('mac'):
ctx.source.instance.runtime_properties['ip'] = lease.get(
'ipaddr')
ctx.logger.info("{}:Found: {}".format(
vm_id, lease.get('ipaddr')))
return
# we have never get ip before 60 sec, so wait 60 as minimum
time.sleep(60)
raise cfy_exc.RecoverableError('No ip for now, try later')
def _subscription_login(self, url, username, password, token, service,
org_name):
"""
login to subscription service
"""
logined = False
vdc_logined = False
vca = vcloudair.VCA(
url, username, service_type=SUBSCRIPTION_SERVICE_TYPE,
version='5.6')
# login with token
if token:
for _ in range(self.LOGIN_RETRY_NUM):
logined = vca.login(token=token)
if logined is False:
ctx.logger.info("Login using token failed.")
time.sleep(RELOGIN_TIMEOUT)
continue
else:
ctx.logger.info("Login using token successful.")
break
# outdated token, try login by password
if logined is False and password:
for _ in range(self.LOGIN_RETRY_NUM):
logined = vca.login(password)
if logined is False:
ctx.logger.info("Login using password failed. Retrying...")
time.sleep(RELOGIN_TIMEOUT)
continue
else:
ctx.logger.info("Login using password successful.")
break
# can't login to system at all
if logined is False:
raise cfy_exc.NonRecoverableError("Invalid login credentials")
for _ in range(self.LOGIN_RETRY_NUM):
vdc_logined = vca.login_to_org(service, org_name)
if vdc_logined is False:
ctx.logger.info("Login to VDC failed. Retrying...")
time.sleep(RELOGIN_TIMEOUT)
continue
else:
ctx.logger.info("Login to VDC successful.")
break
# we can login to system,
# but have some troubles with login to organization,
# lets retry later
if vdc_logined is False:
raise cfy_exc.RecoverableError(message="Could not login to VDC",
retry_after=RELOGIN_TIMEOUT)
atexit.register(vca.logout)
return vca
def test_delete_backup(self, cinder_m):
cinder_instance = cinder_m.return_value
volume_ctx, volume_id = self._simple_volume_ctx()
# remove any, nothing
cinder_instance.backups.list = mock.Mock(return_value=[])
cinder_instance.volume_snapshots.list = mock.Mock(return_value=[])
volume._delete_backup(cinder_instance, {'volume_id': volume_id})
cinder_instance.backups.list.assert_has_calls([
mock.call(search_opts={'volume_id': volume_id})])
cinder_instance.volume_snapshots.list.assert_not_called()
# remove any, but we have other backup
backup_mock = mock.Mock()
backup_mock.delete = mock.Mock()
backup_mock.name = 'backup_other'
backup_mock.id = 'backup_id'
backup_mock.status = 'available'
cinder_instance.backups.list = mock.Mock(return_value=[backup_mock])
cinder_instance.volume_snapshots.list = mock.Mock(return_value=[])
volume._delete_backup(cinder_instance, {'volume_id': volume_id,
'name': 'backup_name'})
cinder_instance.backups.list.assert_has_calls([
mock.call(search_opts={'volume_id': volume_id,
'name': 'backup_name'}),
mock.call(search_opts={'volume_id': volume_id,
'name': 'backup_name'})])
cinder_instance.volume_snapshots.list.assert_not_called()
backup_mock.delete.assert_not_called()
# can't delete snapshot
backup_mock = mock.Mock()
backup_mock.delete = mock.Mock()
backup_mock.name = 'backup_name'
backup_mock.id = 'backup_id'
backup_mock.status = 'available'
cinder_instance.backups.list = mock.Mock(return_value=[backup_mock])
cinder_instance.volume_snapshots.list = mock.Mock(return_value=[])
volume_ctx.operation.retry = mock.Mock(
side_effect=cfy_exc.RecoverableError())
with self.assertRaises(cfy_exc.RecoverableError):
volume._delete_backup(cinder_instance, {'volume_id': volume_id,
'name': 'backup_name'})
cinder_instance.backups.list.assert_has_calls([
mock.call(search_opts={'volume_id': volume_id,
'name': 'backup_name'}),
mock.call(search_opts={'volume_id': volume_id,
'name': 'backup_name'})])
cinder_instance.volume_snapshots.list.assert_not_called()
backup_mock.delete.assert_called_once_with()
def _subscription_login(self, url, username, password, token, service,
org_name, session_token=None, org_url=None):
"""
login to subscription service
"""
version = '5.6'
logined = False
vdc_logined = False
vca = vcloudair.VCA(
url, username, service_type=SUBSCRIPTION_SERVICE_TYPE,
version=version)
if session_token:
vca = login_to_vca_with_token(vca, org_url, session_token, version)
if vca:
return vca
else:
raise cfy_exc.NonRecoverableError("Invalid session credentials")
global local_org_url
global local_session_token
if local_session_token:
vca = login_to_vca_with_token(vca, local_org_url,
local_session_token, version)
if vca:
return vca
# login with token
if token:
logined = login_with_retry(vca.login, [None, token],
"Login using token")
# outdated token, try login by password
if logined is False and password:
logined = login_with_retry(vca.login, [password, None],
"Login using token")
# can't login to system at all
if logined is False:
raise cfy_exc.NonRecoverableError("Invalid login credentials")
vdc_logined = login_with_retry(vca.login_to_org, [service, org_name],
"Login to org")
# we can login to system,
# but have some troubles with login to organization,
# lets retry later
if vdc_logined:
local_session_token = vca.vcloud_session.token
local_org_url = vca.vcloud_session.org_url
else:
raise cfy_exc.RecoverableError(message="Could not login to VDC",
retry_after=RELOGIN_TIMEOUT)
atexit.register(vca.logout)
return vca
def _delete_force(dom):
"""remove domain internaly without cleanup for properties"""
if dom.snapshotNum():
ctx.logger.info("Domain has {} snapshots.".format(dom.snapshotNum()))
_cleanup_snapshots(ctx, dom)
state, _ = dom.state()
if state != libvirt.VIR_DOMAIN_SHUTOFF:
if dom.destroy() < 0:
raise cfy_exc.RecoverableError('Can not destroy guest domain.')
try:
if dom.undefineFlags(libvirt.VIR_DOMAIN_UNDEFINE_NVRAM) < 0:
raise cfy_exc.RecoverableError(
'Can not undefine guest domain with NVRAM.')
except AttributeError as e:
ctx.logger.info("Non critical error: {}".format(str(e)))
if dom.undefine() < 0:
raise cfy_exc.RecoverableError('Can not undefine guest domain.')
def rerun(ctx, func, args, kwargs, retry_count=10, retry_sleep=15):
while retry_count > 0:
try:
return func(*args, **kwargs)
except exceptions.RecoverableWarning as e:
ctx.logger.info("Need for rerun: {e}".format(e=repr(e)))
retry_count -= 1
time.sleep(retry_sleep)
except (exceptions.RecoverableError,
exceptions.RecoverableResponseException,
exceptions.RecoverableStatusCodeCodeException,
exceptions.ExpectationException) as e:
raise cfy_exc.RecoverableError(str(e))
except (exceptions.NonRecoverableError,
exceptions.NonRecoverableResponseException) as e:
raise cfy_exc.NonRecoverableError(str(e))
raise cfy_exc.RecoverableError(
"Failed to rerun: {args}:{kwargs}"
.format(args=repr(args), kwargs=repr(kwargs)))
def _have_error(reply):
# error check
# https://tools.ietf.org/html/rfc6241#section-4.3
error = None
errors = []
for k in reply:
# skip attributes
if k[:2] == "_@":
continue
if 'rpc-error' == k or '@rpc-error' == k[-len('@rpc-error'):]:
# we can have list of rpc-errors with different namespaces
if not isinstance(reply[k], list):
# only one error with such namespace
errors.append(reply[k])
else:
# we can combine several errors lists
errors += reply[k]
if not errors:
return
for error in errors:
if not error:
# we have empty rpc-error?
raise cfy_exc.RecoverableError("Empty error struct:" + str(errors))
# server can send warning as error, lets check
error_severities = []
for k in error:
# skip attributes
if k[:2] == "_@":
continue
if ('error-severity' == k
or '@error-severity' == k[-len('@error-severity'):]):
# and error severity that we found
error_severities.append(error[k])
for error_severity in error_severities:
if error_severity != 'warning':
raise cfy_exc.RecoverableError(
"We have error in reply: {}".format(repr(errors)))
def _rerun(ctx, func, args, kwargs, retry_count=10, retry_sleep=15):
retry_count = 10
while retry_count > 0:
try:
return func(*args, **kwargs)
except terminal_connection.RecoverableWarning as e:
ctx.logger.info("Need for rerun: {}".format(repr(e)))
retry_count -= 1
time.sleep(retry_sleep)
raise cfy_exc.RecoverableError(
"Failed to rerun: {}:{}".format(repr(args), repr(kwargs)))
def start(**kwargs):
ctx.logger.info("start")
resource_id = ctx.instance.runtime_properties.get('resource_id')
if not resource_id:
# not uninstall workflow, raise exception
raise cfy_exc.NonRecoverableError("No pool for start")
if ctx.instance.runtime_properties.get('use_external_resource'):
ctx.logger.info("External resource, skip")
return
libvirt_auth, _ = common.get_libvirt_params(**kwargs)
conn = libvirt.open(libvirt_auth)
if conn is None:
raise cfy_exc.NonRecoverableError(
'Failed to open connection to the hypervisor')
try:
try:
pool = conn.storagePoolLookupByName(resource_id)
except libvirt.libvirtError as e:
raise cfy_exc.NonRecoverableError(
'Failed to find the pool: {}'.format(repr(e)))
# pool create
for i in xrange(10):
if pool.isActive():
ctx.logger.info("Looks as active.")
break
ctx.logger.info("Tring to start pool {}/10".format(i))
if pool.create() < 0:
raise cfy_exc.RecoverableError('Can not start pool.')
time.sleep(30)
else:
raise cfy_exc.RecoverableError('Can not start pool.')
finally:
conn.close()
def start(**kwargs):
ctx.logger.info("start")
resource_id = ctx.instance.runtime_properties.get('resource_id')
if not resource_id:
# not uninstall workflow, raise exception
raise cfy_exc.NonRecoverableError("No servers for start")
libvirt_auth, template_params = common.get_libvirt_params(**kwargs)
conn = libvirt.open(libvirt_auth)
if conn is None:
raise cfy_exc.NonRecoverableError(
'Failed to open connection to the hypervisor')
# wait for ip on start
wait_for_ip = template_params.get('wait_for_ip', False)
try:
try:
dom = conn.lookupByName(resource_id)
except libvirt.libvirtError as e:
raise cfy_exc.NonRecoverableError(
'Failed to find the domain: {}'.format(repr(e)))
for i in xrange(10):
state, _ = dom.state()
ctx.logger.info("Tring to start vm {}/10".format(i))
if wait_for_ip:
ctx.logger.info("Waiting for ip.")
if state == libvirt.VIR_DOMAIN_RUNNING:
_update_network_list(dom)
# wait for ip check
if not wait_for_ip or (
wait_for_ip
and ctx.instance.runtime_properties.get('ip')):
ctx.logger.info("Looks as running.")
return
elif dom.create() < 0:
raise cfy_exc.NonRecoverableError(
'Can not start guest domain.')
time.sleep(30)
# still no ip
if wait_for_ip:
raise cfy_exc.RecoverableError('No ip for now, try later')
finally:
conn.close()
def attempt_with_rerun(func, **kwargs):
"""Rerun func several times, useful after dlr/esg delete"""
i = 10
while i >= 0:
try:
func(**kwargs)
return
except cfy_exc.RecoverableError as ex:
ctx.logger.error("%s: %s attempts left: Message: %s " %
(func.__name__, i, str(ex)))
if not i:
raise cfy_exc.RecoverableError(
message="Retry %s little later" % func.__name__)
time.sleep(30)
i -= 1
def stop(**kwargs):
ctx.logger.info("stop")
resource_id = ctx.instance.runtime_properties.get('resource_id')
if not resource_id:
# not raise exception on 'uninstall' workflow
ctx.logger.info("No pools for stop")
return
if ctx.instance.runtime_properties.get('use_external_resource'):
ctx.logger.info("External resource, skip")
return
libvirt_auth, _ = common.get_libvirt_params(**kwargs)
conn = libvirt.open(libvirt_auth)
if conn is None:
raise cfy_exc.NonRecoverableError(
'Failed to open connection to the hypervisor')
try:
try:
pool = conn.storagePoolLookupByName(resource_id)
except libvirt.libvirtError as e:
raise cfy_exc.NonRecoverableError(
'Failed to find the pool: {}'.format(repr(e)))
for i in xrange(10):
if not pool.isActive():
ctx.logger.info("Looks as not active.")
break
ctx.logger.info("Tring to stop pool {}/10".format(i))
if pool.destroy() < 0:
raise cfy_exc.NonRecoverableError('Can not destroy pool.')
time.sleep(30)
state, capacity, allocation, available = pool.info()
ctx.logger.info(
"State: {}, Capacity: {}, Allocation: {}, Available: {}".format(
repr(state), repr(capacity), repr(allocation),
repr(available)))
if state != libvirt.VIR_STORAGE_POOL_INACTIVE:
if pool.delete() < 0:
raise cfy_exc.RecoverableError('Can not delete guest pool.')
finally:
conn.close()
def _cleanup_snapshots(ctx, dom):
snapshots = dom.listAllSnapshots()
snapshots_count = len(snapshots)
for _ in xrange(snapshots_count):
for snapshot in snapshots:
# we can delete only snapshot without child
if not snapshot.numChildren():
ctx.logger.info("Remove {} snapshot.".format(
snapshot.getName()))
snapshot.delete()
snapshots = dom.listAllSnapshots()
if len(snapshots):
subsnapshots = [snap.getName() for snap in snapshots]
raise cfy_exc.RecoverableError(
"Still have several snapshots: {subsnapshots}.".format(
subsnapshots=repr(subsnapshots)))
def _cleanup_response(self, text, prefix, error_examples):
if not error_examples:
return text.strip()
# check command echo
have_correct_prefix = False
prefix_pos = text.find(prefix)
if prefix_pos == -1:
if self.logger:
self.logger.info(
"Have not found '%s' in response: '%s'" % (
prefix, repr(text)
)
)
else:
if text[:prefix_pos].strip():
if self.logger:
self.logger.info(
"Some mess before '%s' in response: '%s'" % (
prefix, repr(text)
)
)
else:
have_correct_prefix = True
if have_correct_prefix:
# looks as we have correct line
response = text[prefix_pos + len(prefix):]
else:
# skip first line(where must be echo from commands input)
if "\n" in text:
response = text[text.find("\n"):]
else:
response = text
# check for errors started only from new line
errors_with_new_line = ["\n" + error for error in error_examples]
if self._find_any_in(response, errors_with_new_line) != -1:
if not self.is_closed():
self.close()
raise cfy_exc.RecoverableError(
"Looks as we have error in response: %s" % (text)
)
return response.strip()
def _amqp_client(self):
# initialize an amqp client only when needed, ie. if the task is
# not local
with_amqp = bool(self.ctx.task_target)
if with_amqp:
try:
amqp_client_utils.init_events_publisher()
except Exception:
_, ex, tb = sys.exc_info()
# This one should never (!) raise an exception.
amqp_client_utils.close_amqp_client()
raise exceptions.RecoverableError(
'Failed initializing AMQP connection',
causes=[utils.exception_to_error_cause(ex, tb)])
try:
yield
finally:
if with_amqp:
amqp_client_utils.close_amqp_client()
def verify_worker_alive(name, target, get_registered):
cache = RemoteWorkflowTask.cache
registered = cache.get(target)
if not registered:
registered = get_registered()
cache[target] = registered
if registered is None:
raise exceptions.RecoverableError(
'Timed out querying worker celery@{0} for its registered '
'tasks. [timeout={1} seconds]'.format(target, INSPECT_TIMEOUT))
if DISPATCH_TASK not in registered:
raise exceptions.NonRecoverableError(
'Missing {0} task in worker {1} \n'
'Registered tasks are: {2}. (This probably means the agent '
'configuration is invalid) [{3}]'.format(DISPATCH_TASK, target,
registered, name))
def delete(**kwargs):
resource_id = ctx.instance.runtime_properties.get('resource_id')
ctx.logger.info("Delete: {}".format(repr(resource_id)))
if not resource_id:
# not raise exception on 'uninstall' workflow
ctx.logger.info("No pool for delete")
return
if ctx.instance.runtime_properties.get('use_external_resource'):
ctx.logger.info("External resource, skip")
return
libvirt_auth, _ = common.get_libvirt_params(**kwargs)
conn = libvirt.open(libvirt_auth)
if conn is None:
raise cfy_exc.NonRecoverableError(
'Failed to open connection to the hypervisor')
try:
# lookup the default pool by name
try:
pool = conn.storagePoolLookupByName(resource_id)
except libvirt.libvirtError as e:
raise cfy_exc.NonRecoverableError(
'Failed to find the pool: {}'.format(repr(e)))
if pool.delete() < 0:
raise cfy_exc.RecoverableError('Can not delete pool.')
if pool.undefine() < 0:
raise cfy_exc.NonRecoverableError('Can not undefine pool.')
ctx.instance.runtime_properties['resource_id'] = None
ctx.instance.runtime_properties['backups'] = {}
finally:
conn.close()
def configure(**kwargs):
ctx.logger.info("configure")
resource_id = ctx.instance.runtime_properties.get('resource_id')
if not resource_id:
# not uninstall workflow, raise exception
raise cfy_exc.NonRecoverableError("No pool for configure")
if ctx.instance.runtime_properties.get('use_external_resource'):
ctx.logger.info("External resource, skip")
return
libvirt_auth, _ = common.get_libvirt_params(**kwargs)
conn = libvirt.open(libvirt_auth)
if conn is None:
raise cfy_exc.NonRecoverableError(
'Failed to open connection to the hypervisor')
try:
try:
pool = conn.storagePoolLookupByName(resource_id)
except libvirt.libvirtError as e:
raise cfy_exc.NonRecoverableError(
'Failed to find the pool: {}'.format(repr(e)))
state, capacity, allocation, available = pool.info()
ctx.logger.info(
"State: {}, Capacity: {}, Allocation: {}, Available: {}".format(
repr(state), repr(capacity), repr(allocation),
repr(available)))
if state == libvirt.VIR_STORAGE_POOL_INACTIVE:
if pool.build(0) < 0:
raise cfy_exc.RecoverableError('Can not build guest pool.')
finally:
conn.close()
if __name__ == '__main__':
base_disk = ctx.instance.runtime_properties['disk_image']
ctx.logger.info("Base image: {}".format(repr(base_disk)))
cloud_init = ctx.instance.runtime_properties['cloud_init']
ctx.logger.debug("Cloud init: {}".format(repr(cloud_init)))
cwd = ctx.instance.runtime_properties.get('storage_path', os.getcwd())
ctx.logger.info("Current dir: {}".format(repr(cwd)))
copy_disk = "{}/{}.qcow2".format(cwd, ctx.instance.id)
if execute_command([
"qemu-img", "create", "-f", "qcow2", "-o",
"backing_file={}".format(base_disk), copy_disk
]) == False:
raise exceptions.RecoverableError('Failed create disk.')
if ctx.instance.runtime_properties.get('disk_size'):
if execute_command([
"qemu-img", "resize", copy_disk,
ctx.instance.runtime_properties.get('disk_size')
]) == False:
raise exceptions.RecoverableError('Failed resize disk.')
ctx.instance.runtime_properties["vm_image"] = copy_disk
seed_disk = "{}/{}_seed".format(cwd, ctx.instance.id)
os.mkdir(seed_disk)
with open("{}/meta-data".format(seed_disk), 'w') as meta_file:
meta_file.write("instance-id: {}\n".format(ctx.instance.id))
meta_file.write("local-hostname: {}\n".format(
_gen_hostname(ctx.instance.id)))
workdir = ctx.plugin.workdir
paths = diamond.get_paths(workdir.replace("script","diamond"))
name = 'SNMPProxyCollector'
collector_dir = os.path.join(paths['collectors'], name)
if not os.path.exists(collector_dir):
os.mkdir(collector_dir)
collector_file = os.path.join(collector_dir, '{0}.py'.format(name))
ctx.download_resource('scripts/monitoring/proxy_snmp/snmpproxy.py', collector_file)
config = ctx.target.instance.runtime_properties.get('snmp_collector_config', {})
config.update({'enabled': True,
'hostname': '{0}.{1}.{2}'.format(diamond.get_host_id(ctx.target),
ctx.target.node.name,
ctx.target.instance.id)
})
config_full_path = os.path.join(paths['collectors_config'], '{0}.conf'.format(name))
diamond.write_config(config_full_path, config)
try:
diamond.stop_diamond(paths['config'])
except:
pass
try:
diamond.start_diamond(paths['config'])
except:
exceptions.RecoverableError("Failed to start diamond", 30)
pass
def wrapper(*args, **kwargs):
ctx = _find_context_arg(args, kwargs, _is_cloudify_context)
if ctx is None:
ctx = {}
if not _is_cloudify_context(ctx):
ctx = context.CloudifyContext(ctx)
# remove __cloudify_context
raw_context = kwargs.pop(CLOUDIFY_CONTEXT_PROPERTY_KEY, {})
if ctx.task_target is None:
# task is local (not through celery) so we need to
# clone kwarg
kwargs = copy.deepcopy(kwargs)
if raw_context.get('has_intrinsic_functions') is True:
kwargs = ctx._endpoint.evaluate_functions(payload=kwargs)
kwargs['ctx'] = ctx
try:
current_ctx.set(ctx, kwargs)
result = func(*args, **kwargs)
except BaseException as e:
ctx.logger.error(
'Exception raised on operation [%s] invocation',
ctx.task_name, exc_info=True)
if ctx.task_target is None:
# local task execution
# no serialization issues
raise
# extract exception details
# type, value, traceback
tpe, value, tb = sys.exc_info()
# we re-create the exception here
# since it will be sent
# over the wire. And the original exception
# may cause de-serialization issues
# on the other side.
# preserve original type in the message
message = '{0}: {1}'.format(tpe.__name__, str(e))
# if the exception type is directly one of our exception
# than there is no need for conversion and we can just
# raise the original exception
if type(e) in [exceptions.OperationRetry,
exceptions.RecoverableError,
exceptions.NonRecoverableError,
exceptions.HttpException]:
raise
# if the exception inherits from our base exceptions, there
# still might be a de-serialization problem caused by one of
# the types in the inheritance tree.
if isinstance(e, exceptions.NonRecoverableError):
value = exceptions.NonRecoverableError(message)
elif isinstance(e, exceptions.OperationRetry):
value = exceptions.OperationRetry(message, e.retry_after)
elif isinstance(e, exceptions.RecoverableError):
value = exceptions.RecoverableError(message, e.retry_after)
else:
# convert pure user exceptions
# to a RecoverableError
value = exceptions.RecoverableError(message)
raise type(value), value, tb
finally:
current_ctx.clear()
if ctx.type == context.NODE_INSTANCE:
ctx.instance.update()
elif ctx.type == context.RELATIONSHIP_INSTANCE:
ctx.source.instance.update()
ctx.target.instance.update()
if ctx.operation._operation_retry:
raise ctx.operation._operation_retry
return result
def test_cleanup_after_waituntilstatus_throws_recoverable_error(self):
err = cfy_exc.RecoverableError('Some recoverable error')
with mock.patch.object(volume, 'wait_until_status',
side_effect=[err, (None, True)]) as wait_mock:
self._test_cleanup__after_attach_fails(type(err), True, wait_mock)
def test_delete_snapshot(self, cinder_m):
cinder_instance = cinder_m.return_value
volume_ctx, volume_id = self._simple_volume_ctx()
# remove any, nothing
cinder_instance.backups.list = mock.Mock(return_value=[])
cinder_instance.volume_snapshots.list = mock.Mock(return_value=[])
volume._delete_snapshot(cinder_instance, {'volume_id': volume_id})
cinder_instance.backups.list.assert_not_called()
cinder_instance.volume_snapshots.list.assert_has_calls([
mock.call(search_opts={'volume_id': volume_id})])
# remove any, but we have other snapshot
snapshot_mock = mock.Mock()
snapshot_mock.delete = mock.Mock()
snapshot_mock.name = 'snapshot_other'
snapshot_mock.id = 'snapshot_id'
cinder_instance.backups.list = mock.Mock(return_value=[])
cinder_instance.volume_snapshots.list = mock.Mock(
return_value=[snapshot_mock])
volume._delete_snapshot(cinder_instance, {
'volume_id': volume_id, 'display_name': 'snapshot_name'
})
cinder_instance.backups.list.assert_not_called()
cinder_instance.volume_snapshots.list.assert_has_calls([
mock.call(search_opts={'volume_id': volume_id,
'display_name': 'snapshot_name'}),
mock.call(search_opts={'volume_id': volume_id,
'display_name': 'snapshot_name'})])
snapshot_mock.delete.assert_not_called()
# can't delete snapshot
snapshot_mock = mock.Mock()
snapshot_mock.delete = mock.Mock()
snapshot_mock.name = 'snapshot_name'
snapshot_mock.id = 'snapshot_id'
snapshot_mock.status = 'available'
cinder_instance.backups.list = mock.Mock(return_value=[])
cinder_instance.volume_snapshots.list = mock.Mock(
return_value=[snapshot_mock])
volume_ctx.operation.retry = mock.Mock(
side_effect=cfy_exc.RecoverableError())
with self.assertRaises(cfy_exc.RecoverableError):
volume._delete_snapshot(cinder_instance, {
'volume_id': volume_id, 'display_name': 'snapshot_name'
})
cinder_instance.backups.list.assert_not_called()
volume_ctx.operation.retry.assert_called_with(
message='snapshot_name is still alive', retry_after=30)
cinder_instance.volume_snapshots.list.assert_has_calls([
mock.call(search_opts={'volume_id': volume_id,
'display_name': 'snapshot_name'}),
mock.call(search_opts={'volume_id': volume_id,
'display_name': 'snapshot_name'})])
snapshot_mock.delete.assert_called_once_with()
def set_url(**_):
try:
ctx.source.instance.runtime_properties['mongo_ip_address'] = ctx.target.instance.runtime_properties.host_ip
ctx.source.instance.runtime_properties['mongo_ip_port'] = ctx.target.instance.runtime_properties.host_port
except Exception as e:
raise exceptions.RecoverableError('Failed to set mongo ip & port')
def test_cleanup_after_waituntilstatus_throws_recoverable_error(self):
err = cfy_exc.RecoverableError('Some recoverable error')
self._test_cleanup_after_waituntilstatus_throws(err, True)
def _ondemand_login(self,
url,
username,
password,
token,
instance_id,
session_token=None,
org_url=None):
"""
login to ondemand service
"""
def get_instance(vca, instance_id):
instances = vca.get_instances() or []
for instance in instances:
if instance['id'] == instance_id:
return instance
version = '5.7'
if instance_id is None:
raise cfy_exc.NonRecoverableError(
"Instance ID should be specified for OnDemand login")
logined = False
instance_logined = False
vca = vcloudair.VCA(url,
username,
service_type=ONDEMAND_SERVICE_TYPE,
version=version)
if session_token:
if session_login(vca, org_url, session_token, version):
return vca
else:
raise cfy_exc.NonRecoverableError(
"Invalid session credentials")
# login with token
if token:
for _ in range(LOGIN_RETRY_NUM):
logined = vca.login(token=token)
if logined is False:
ctx.logger.info("Login using token failed.")
time.sleep(RELOGIN_TIMEOUT)
continue
else:
ctx.logger.info("Login using token successful.")
break
# outdated token, try login by password
if logined is False and password:
for _ in range(LOGIN_RETRY_NUM):
logined = vca.login(password)
if logined is False:
ctx.logger.info("Login using password failed. Retrying...")
time.sleep(RELOGIN_TIMEOUT)
continue
else:
ctx.logger.info("Login using password successful.")
break
# can't login to system at all
if logined is False:
raise cfy_exc.NonRecoverableError("Invalid login credentials")
instance = get_instance(vca, instance_id)
if instance is None:
raise cfy_exc.NonRecoverableError(
"Instance {0} could not be found.".format(instance_id))
for _ in range(LOGIN_RETRY_NUM):
instance_logined = vca.login_to_instance(instance_id, password,
token, None)
if instance_logined is False:
ctx.logger.info("Login to instance failed. Retrying...")
time.sleep(RELOGIN_TIMEOUT)
continue
else:
ctx.logger.info("Login to instance successful.")
break
for _ in range(LOGIN_RETRY_NUM):
instance_logined = vca.login_to_instance(
instance_id, None, vca.vcloud_session.token,
vca.vcloud_session.org_url)
if instance_logined is False:
ctx.logger.info("Login to instance failed. Retrying...")
time.sleep(RELOGIN_TIMEOUT)
continue
else:
ctx.logger.info("Login to instance successful.")
break
# we can login to system,
# but have some troubles with login to instance,
# lets retry later
if instance_logined is False:
raise cfy_exc.RecoverableError(
message="Could not login to instance",
retry_after=RELOGIN_TIMEOUT)
atexit.register(vca.logout)
return vca
def run(**kwargs):
"""main entry point for all calls"""
calls = kwargs.get('calls', [])
if not calls:
ctx.logger.info("No calls")
return
# credentials
properties = ctx.node.properties
terminal_auth = properties.get('terminal_auth', {})
terminal_auth.update(kwargs.get('terminal_auth', {}))
ip_list = terminal_auth.get('ip')
# if node contained in some other node, try to overwrite ip
if not ip_list:
ip_list = [ctx.instance.host_ip]
ctx.logger.info("Used host from container: %s" % str(ip_list))
if isinstance(ip_list, basestring):
ip_list = [ip_list]
user = terminal_auth.get('user')
password = terminal_auth.get('password')
key_content = terminal_auth.get('key_content')
port = terminal_auth.get('port', 22)
if not ip_list or not user:
raise cfy_exc.NonRecoverableError(
"please check your credentials, ip or user not set")
# additional settings
global_promt_check = terminal_auth.get('promt_check')
global_error_examples = terminal_auth.get('errors')
exit_command = terminal_auth.get('exit_command')
ctx.logger.info("---> exit_command: %s" % str(exit_command))
# save logs to debug file
log_file_name = None
if terminal_auth.get('store_logs'):
log_file_name = "/tmp/terminal-%s_%s_%s.log" % (str(
ctx.execution_id), str(ctx.instance.id), str(ctx.workflow_id))
ctx.logger.info("Communication logs will be saved to %s" %
log_file_name)
connection = terminal_connection.connection()
for ip in ip_list:
try:
prompt = connection.connect(ip,
user,
password,
key_content,
port,
global_promt_check,
logger=ctx.logger,
log_file_name=log_file_name)
ctx.logger.info("Will be used: " + ip)
break
except Exception as ex:
ctx.logger.info("Can't connect to %s with %s" %
(repr(ip), str(ex)))
raise cfy_exc.RecoverableError()
else:
raise cfy_exc.NonRecoverableError("please check your ip list")
ctx.logger.info("Device prompt: " + prompt)
for call in calls:
responses = call.get('responses', [])
promt_check = call.get('promt_check', global_promt_check)
error_examples = call.get('errors', global_error_examples)
# use action if exist
operation = call.get('action', "")
# use template if have
if not operation and 'template' in call:
template_name = call.get('template')
template_params = call.get('params')
template = ctx.get_resource(template_name)
if not template:
ctx.logger.info("Empty template.")
continue
template_engine = Template(template)
if not template_params:
template_params = {}
# save context for reuse in template
template_params['ctx'] = ctx
operation = template_engine.render(template_params)
# incase of template_text
if not operation and 'template_text' in call:
template_params = call.get('params')
template = call.get('template_text')
if not template:
ctx.logger.info("Empty template_text.")
continue
template_engine = Template(template)
if not template_params:
template_params = {}
# save context for reuse in template
template_params['ctx'] = ctx
operation = template_engine.render(template_params)
if not operation:
continue
if responses:
ctx.logger.info("We have predefined responses: " + str(responses))
ctx.logger.info("Template: \n" + str(operation))
result = ""
for op_line in operation.split("\n"):
# skip empty lines
if not op_line.strip():
continue
ctx.logger.info("Execute: " + op_line)
result_part = connection.run(op_line, promt_check, error_examples,
responses)
if result_part.strip():
ctx.logger.info(result_part.strip())
result += (result_part + "\n")
# save results to runtime properties
save_to = call.get('save_to')
if save_to:
ctx.logger.info("For save: " + result.strip())
ctx.instance.runtime_properties[save_to] = result.strip()
while not connection.is_closed() and exit_command:
ctx.logger.info("Execute close")
ctx.logger.info("---> exit_command: %s" % str(exit_command))
ctx.logger.info("---> promt_check: %s" % str(promt_check))
ctx.logger.info("---> error_examples: %s" % str(error_examples))
result = connection.run(exit_command, promt_check, error_examples)
ctx.logger.info("Result of close: " + result)
connection.close()
def _ondemand_login(self, url, username, password, token, instance_id,
session_token=None, org_url=None):
"""
login to ondemand service
"""
def get_instance(vca, instance_id):
instances = vca.get_instances() or []
for instance in instances:
if instance['id'] == instance_id:
return instance
version = '5.7'
if instance_id is None:
raise cfy_exc.NonRecoverableError(
"Instance ID should be specified for OnDemand login")
logined = False
instance_logined = False
vca = vcloudair.VCA(
url, username, service_type=ONDEMAND_SERVICE_TYPE, version=version)
if session_token:
vca = login_to_vca_with_token(vca, org_url, session_token, version)
if vca:
return vca
else:
raise cfy_exc.NonRecoverableError("Invalid session credentials")
global local_org_url
global local_session_token
if local_session_token:
vca = login_to_vca_with_token(vca, local_org_url,
local_session_token, version)
if vca:
return vca
# login with token
if token:
logined = login_with_retry(vca.login, [None, token],
"Login using token")
# outdated token, try login by password
if logined is False and password:
logined = login_with_retry(vca.login, [password, None],
"Login using password")
# can't login to system at all
if logined is False:
raise cfy_exc.NonRecoverableError("Invalid login credentials")
instance = get_instance(vca, instance_id)
if instance is None:
raise cfy_exc.NonRecoverableError(
"Instance {0} could not be found.".format(instance_id))
instance_logined = login_with_retry(vca.login_to_instance,
[instance_id, password],
"Login to instance with password")
if instance_logined:
instance_logined = login_with_retry(vca.login_to_instance,
[instance_id, None,
vca.vcloud_session.token,
vca.vcloud_session.org_url],
"Login to instance with token")
# we can login to system,
# but have some troubles with login to instance,
# lets retry later
if instance_logined:
local_session_token = vca.vcloud_session.token
local_org_url = vca.vcloud_session.org_url
else:
raise cfy_exc.RecoverableError(
message="Could not login to instance",
retry_after=RELOGIN_TIMEOUT)
atexit.register(vca.logout)
return vca
