def resource_by_id(_client, _id, _type):
_resources_client = getattr(_client, _type)
try:
_resources = _resources_client.list(_include=['id'])
except CloudifyClientError as ex:
raise NonRecoverableError('{0} list failed {1}.'.format(
_type, text_type(ex)))
else:
return [text_type(_r['id']) == _id for _r in _resources]
def test_dep_workflow_in_state_pollster_raises(self):
test_name = 'test_dep_workflow_in_state_pollster_raises'
_ctx = self.get_mock_ctx(test_name)
current_ctx.set(_ctx)
with mock.patch('cloudify.manager.get_rest_client') as mock_client:
cfy_mock_client = MockCloudifyRestClient()
response = cfy_mock_client.executions.get()
response['id'] = test_name
def mock_return(*args, **kwargs):
del args, kwargs
raise CloudifyClientError('Mistake')
cfy_mock_client.executions.get = mock_return
mock_client.return_value = cfy_mock_client
output = \
self.assertRaises(
NonRecoverableError,
dep_workflow_in_state_pollster,
cfy_mock_client,
test_name,
'terminated',
0)
self.assertIn('failed', text_type(output))
def dep_system_workflows_finished(_client, _check_all_in_deployment=False):
_offset = int(getenv('_PAGINATION_OFFSET', 0))
_size = int(getenv('_PAGINATION_SIZE', 1000))
while True:
try:
_execs = _client.executions.list(include_system_workflows=True,
_offset=_offset,
_size=_size)
except CloudifyClientError as ex:
raise NonRecoverableError('Executions list failed {0}.'.format(
text_type(ex)))
for _exec in _execs:
if _exec.get('is_system_workflow'):
if _exec.get('status') not in ('terminated', 'completed',
'failed', 'cancelled'):
return False
if _check_all_in_deployment:
if _check_all_in_deployment == _exec.get('deployment_id'):
if _exec.get('status') not in ('terminated', 'completed',
'failed', 'cancelled'):
return False
if _execs.metadata.pagination.total <= \
_execs.metadata.pagination.offset:
break
_offset = _offset + _size
return True
def download_file(url, destination=None, keep_name=False):
"""Download file.
:param url: Location of the file to download
:type url: str
:param destination:
Location where the file should be saved (autogenerated by default)
:param keep_name: use the filename from the url as destination filename
:type destination: str | None
:returns: Location where the file was saved
:rtype: str
"""
CHUNK_SIZE = 1024
if not destination:
if keep_name:
path = urlparse(url).path
name = os.path.basename(path)
destination = os.path.join(tempfile.mkdtemp(), name)
else:
fd, destination = tempfile.mkstemp()
os.close(fd)
ctx.logger.info('Downloading {0} to {1}...'.format(url, destination))
try:
response = requests.get(url, stream=True)
except requests.exceptions.RequestException as ex:
raise NonRecoverableError('Failed to download {0}. ({1})'.format(
url, text_type(ex)))
final_url = response.url
if final_url != url:
ctx.logger.debug('Redirected to {0}'.format(final_url))
try:
with open(destination, 'wb') as destination_file:
for chunk in response.iter_content(CHUNK_SIZE):
destination_file.write(chunk)
except IOError as ex:
raise NonRecoverableError('Failed to download {0}. ({1})'.format(
url, text_type(ex)))
return destination
def dep_workflow_in_state_pollster(_client,
_dep_id,
_state,
_workflow_id=None,
_log_redirect=False,
_execution_id=None):
exec_get_fields = \
['status', 'workflow_id', 'created_at', 'id']
try:
_exec = \
_client.executions.get(execution_id=_execution_id,
_include=exec_get_fields)
ctx.logger.debug('The exec get response form {0} is {1}'.format(
_dep_id, _exec))
except CloudifyClientError as ex:
raise NonRecoverableError('Executions get failed {0}.'.format(
text_type(ex)))
if _log_redirect and _exec.get('id'):
ctx.logger.debug('_exec info for _log_redirect is {0}'.format(_exec))
dep_logs_redirect(_client, _exec.get('id'))
if _exec.get('status') == _state:
ctx.logger.debug('The status for _exec info id'
' {0} is {1}'.format(_execution_id, _state))
return True
elif _exec.get('status') == 'failed':
raise NonRecoverableError('Execution {0} failed.'.format(
text_type(_exec)))
return False
def test_poll_workflow_after_execute_failed(self):
_ctx = self.get_mock_ctx('test_poll_workflow_after_execute_failed')
_ctx.logger.log = mock.MagicMock(return_value=None)
current_ctx.set(_ctx)
with mock.patch(
'cloudify_deployment_proxy.polling.poll_with_timeout') \
as mocked_fn:
mocked_fn.return_value = False
output = \
self.assertRaises(
NonRecoverableError,
poll_workflow_after_execute,
None, None, None, None, None, None, None)
self.assertIn('Execution timeout', text_type(output))
def test_execute_start_rest_client_error(self):
# Tests that execute start fails on rest client error
test_name = 'test_execute_start_rest_client_error'
_ctx = self.get_mock_ctx(test_name)
current_ctx.set(_ctx)
_ctx.instance.runtime_properties['deployment'] = {}
with mock.patch('cloudify.manager.get_rest_client') as mock_client:
cfy_mock_client = MockCloudifyRestClient()
cfy_mock_client.executions.start = REST_CLIENT_EXCEPTION
mock_client.return_value = cfy_mock_client
error = self.assertRaises(NonRecoverableError,
execute_start,
deployment_id=test_name,
workflow_id='install')
self.assertIn('action start failed', text_type(error))
del _ctx, mock_client
def dp_get_client_response(self,
_client,
_client_attr,
_client_args):
_generic_client = \
getattr(self.client, _client)
_special_client = \
getattr(_generic_client, _client_attr)
try:
response = _special_client(**_client_args)
except CloudifyClientError as ex:
raise NonRecoverableError(
'Client action {0} failed: {1}.'.format(_client_attr,
text_type(ex)))
else:
return response
def test_execute_start_timeout(self):
# Tests that execute start fails on timeout
test_name = 'test_execute_start_timeout'
_ctx = self.get_mock_ctx(test_name)
current_ctx.set(_ctx)
_ctx.instance.runtime_properties['deployment'] = {}
with mock.patch('cloudify.manager.get_rest_client') as mock_client:
mock_client.return_value = MockCloudifyRestClient()
poll_with_timeout_test = \
'cloudify_deployment_proxy.polling.poll_with_timeout'
with mock.patch(poll_with_timeout_test) as poll:
poll.return_value = False
error = self.assertRaises(NonRecoverableError,
execute_start,
deployment_id=test_name,
workflow_id='install',
timeout=.001)
self.assertIn('Execution timeout', text_type(error))
del _ctx, mock_client
def test_resource_by_id_client_error(self):
test_name = 'test_resource_by_id_client_error'
_ctx = self.get_mock_ctx(test_name)
current_ctx.set(_ctx)
def mock_return(*args, **kwargs):
del args, kwargs
raise CloudifyClientError('Mistake')
with mock.patch('cloudify.manager.get_rest_client') as mock_client:
cfy_mock_client = MockCloudifyRestClient()
cfy_mock_client.deployments.list = mock_return
mock_client.return_value = mock_return
output = \
self.assertRaises(
NonRecoverableError,
resource_by_id,
cfy_mock_client,
test_name,
'deployments')
self.assertIn('failed', text_type(output))
def test_dep_system_workflows_finished_raises(self):
test_name = 'test_dep_system_workflows_finished_raises'
_ctx = self.get_mock_ctx(test_name)
current_ctx.set(_ctx)
with mock.patch('cloudify.manager.get_rest_client') as mock_client:
cfy_mock_client = MockCloudifyRestClient()
list_response = cfy_mock_client.blueprints.list()
list_response[0]['id'] = test_name
def mock_return(*args, **kwargs):
del args, kwargs
raise CloudifyClientError('Mistake')
cfy_mock_client.executions.list = mock_return
mock_client.return_value = cfy_mock_client
output = \
self.assertRaises(
NonRecoverableError,
dep_system_workflows_finished,
cfy_mock_client)
self.assertIn('failed', text_type(output))
def dep_logs_redirect(_client, execution_id):
COUNT_EVENTS = "received_events"
if not ctx.instance.runtime_properties.get(COUNT_EVENTS):
ctx.instance.runtime_properties[COUNT_EVENTS] = {}
last_event = int(ctx.instance.runtime_properties[COUNT_EVENTS].get(
execution_id, 0))
full_count = last_event + 100
while full_count > last_event:
events, full_count = _client.events.get(execution_id, last_event, 250,
True)
for event in events:
ctx.logger.debug('Event {0} for execution_id {1}'.format(
event, execution_id))
instance_prompt = event.get('node_instance_id', "")
if instance_prompt:
if event.get('operation'):
instance_prompt += ("." +
event.get('operation').split('.')[-1])
if instance_prompt:
instance_prompt = "[" + instance_prompt + "] "
message = "%s %s%s" % (event.get('reported_timestamp', ""),
instance_prompt if instance_prompt else "",
event.get('message', ""))
message = text_type(message)
ctx.logger.debug(
'Message {0} for Event {1} for execution_id {1}'.format(
message, event))
level = event.get('level', logging.INFO)
# If the event dict had a 'level' key, then the value is
# a string. In that case, convert it to uppercase and get
# the matching Python logging constant.
if isinstance(level, text_type):
level = logging.getLevelName(level.upper())
# In the (very) odd case that the level is still not an int
# (can happen if the original level value wasn't recognized
# by Python's logging library), then use 'INFO'.
if not isinstance(level, int):
level = logging.INFO
ctx.logger.log(level, message)
last_event += len(events)
# returned infinite count
if full_count < 0:
full_count = last_event + 100
# returned nothing, let's do it next time
if len(events) == 0:
ctx.logger.log(
20, "Waiting for log messages "
"(execution: {0})...".format(execution_id))
break
ctx.instance.runtime_properties[COUNT_EVENTS][execution_id] = last_event
def test_upload_plugins(self):
# Tests that deployments upload plugins
test_name = 'test_delete_deployment_success'
_ctx = self.get_mock_ctx(test_name)
current_ctx.set(_ctx)
get_local_path = mock.Mock(return_value="some_path")
with mock.patch('cloudify.manager.get_rest_client') as mock_client:
plugin = mock.Mock()
plugin.id = "CustomPlugin"
cfy_mock_client = MockCloudifyRestClient()
cfy_mock_client.plugins.upload = mock.Mock(return_value=plugin)
mock_client.return_value = cfy_mock_client
with mock.patch('cloudify_deployment_proxy.get_local_path',
get_local_path):
zip_files = mock.Mock(return_value="_zip")
with mock.patch('cloudify_deployment_proxy.zip_files',
zip_files):
# empty plugins
deployment = DeploymentProxyBase({'plugins': []})
deployment._upload_plugins()
zip_files.assert_not_called()
get_local_path.assert_not_called()
# dist of plugins
deployment = DeploymentProxyBase({
'plugins': {
'base_plugin': {
'wagon_path': '_wagon_path',
'plugin_yaml_path': '_plugin_yaml_path'
}
}
})
os_mock = mock.Mock()
with mock.patch('cloudify_deployment_proxy.os', os_mock):
deployment._upload_plugins()
zip_files.assert_called_with(["some_path", "some_path"])
get_local_path.assert_has_calls([
mock.call('_wagon_path', create_temp=True),
mock.call('_plugin_yaml_path', create_temp=True)
])
os_mock.remove.assert_has_calls([
mock.call('some_path'),
mock.call('some_path'),
mock.call('_zip')
])
get_local_path = mock.Mock(return_value="some_path")
# zip_files = mock.Mock(return_value="_zip")
with mock.patch('cloudify_deployment_proxy.get_local_path',
get_local_path):
zip_files = mock.Mock(return_value="_zip")
with mock.patch('cloudify_deployment_proxy.zip_files',
zip_files):
# list of plugins
deployment = DeploymentProxyBase({
'plugins': [{
'wagon_path': '_wagon_path',
'plugin_yaml_path': '_plugin_yaml_path'
}]
})
os_mock = mock.Mock()
with mock.patch('cloudify_deployment_proxy.os', os_mock):
deployment._upload_plugins()
zip_files.assert_called_with(["some_path", "some_path"])
get_local_path.assert_has_calls([
mock.call('_wagon_path', create_temp=True),
mock.call('_plugin_yaml_path', create_temp=True)
])
os_mock.remove.assert_has_calls([
mock.call('some_path'),
mock.call('some_path'),
mock.call('_zip')
])
# raise error if wrong plugins list
deployment = DeploymentProxyBase({'plugins': True})
error = self.assertRaises(NonRecoverableError,
deployment._upload_plugins)
self.assertIn('Wrong type in plugins: True', str(error))
# raise error if wrong wagon/yaml values
deployment = DeploymentProxyBase(
{'plugins': [{
'wagon_path': '',
'plugin_yaml_path': ''
}]})
error = self.assertRaises(NonRecoverableError,
deployment._upload_plugins)
self.assertIn(
"You should provide both values wagon_path: '' "
"and plugin_yaml_path: ''", text_type(error))
def __init__(self, operation_inputs):
"""
Sets the properties that all operations need.
:param operation_inputs: The inputs from the operation.
"""
full_operation_name = ctx.operation.name
self.operation_name = full_operation_name.split('.').pop()
# These should not make their way into the Operation inputs.
os.environ['_PAGINATION_OFFSET'] = \
text_type(operation_inputs.pop('pagination_offset', 0))
os.environ['_PAGINATION_SIZE'] = \
text_type(operation_inputs.pop('pagination_size', 1000))
# cloudify client
self.client_config = get_desired_value(
'client', operation_inputs,
ctx.instance.runtime_properties,
ctx.node.properties
)
if self.client_config:
self.client = CloudifyClient(**self.client_config)
else:
self.client = manager.get_rest_client()
# plugins
self.plugins = get_desired_value(
'plugins', operation_inputs,
ctx.instance.runtime_properties,
ctx.node.properties
)
# secrets
self.secrets = get_desired_value(
'secrets', operation_inputs,
ctx.instance.runtime_properties,
ctx.node.properties
)
# resource_config
self.config = get_desired_value(
'resource_config', operation_inputs,
ctx.instance.runtime_properties,
ctx.node.properties)
# Blueprint-related properties
self.blueprint = self.config.get('blueprint', {})
self.blueprint_id = self.blueprint.get('id') or ctx.instance.id
self.blueprint_file_name = self.blueprint.get('main_file_name')
self.blueprint_archive = self.blueprint.get('blueprint_archive')
# Deployment-related properties
self.deployment = self.config.get('deployment', {})
self.deployment_id = self.deployment.get('id') or ctx.instance.id
self.deployment_inputs = self.deployment.get('inputs', {})
self.deployment_outputs = self.deployment.get('outputs')
self.deployment_all_outputs = self.deployment.get('all_outputs', True)
self.deployment_logs = self.deployment.get('logs', {})
# Node-instance-related properties
self.node_instance_proxy = self.config.get('node_instance')
# Execution-related properties
self.workflow_id = \
operation_inputs.get('workflow_id',
'create_deployment_environment')
self.workflow_state = \
operation_inputs.get(
'workflow_state',
'terminated')
self.reexecute = \
self.config.get('reexecute') \
or ctx.instance.runtime_properties.get('reexecute') \
or False
# Polling-related properties
self.interval = operation_inputs.get('interval', POLLING_INTERVAL)
self.state = operation_inputs.get('state', 'terminated')
self.timeout = operation_inputs.get('timeout', EXECUTIONS_TIMEOUT)
# This ``execution_id`` will be set once execute workflow done
# successfully
self.execution_id = None
