def func7(message): try: raise RuntimeError(message) except RuntimeError: _, ex, tb = sys.exc_info() raise NonRecoverableUserException(causes=[ utils.exception_to_error_cause(ex, tb) ])
def log(user_cause=False, **_): ctx.logger.info('INFO_MESSAGE') ctx.logger.debug('DEBUG_MESSAGE') causes = [] if user_cause: try: raise RuntimeError('ERROR_MESSAGE') except RuntimeError: _, ex, tb = sys.exc_info() causes.append(utils.exception_to_error_cause(ex, tb)) raise exceptions.NonRecoverableError('ERROR_MESSAGE', causes=causes)
def test_task_failure_causes(self): message = 'test_message' test_event = _event('cloudify_event', event_type='task_failed', message=message) self.assertEqual(test_event.text, message) causes = [] test_event = _event('cloudify_event', event_type='task_failed', message=message, causes=causes) self.assertEqual(test_event.text, message) try: raise RuntimeError() except RuntimeError: _, ex, tb = sys.exc_info() causes = [utils.exception_to_error_cause(ex, tb)] test_event = _event('cloudify_event', event_type='task_failed', message=message, causes=causes) self.assertEqual(test_event.text, message) test_event = _event('cloudify_event', event_type='task_failed', message=message, causes=causes, verbosity_level=event.LOW_VERBOSE) text = test_event.text self.assertIn(message, text) self.assertNotIn('Causes (most recent cause last):', text) self.assertEqual(1, text.count(causes[0]['traceback'])) causes = causes + causes test_event = _event('cloudify_event', event_type='task_failed', message=message, causes=causes, verbosity_level=event.LOW_VERBOSE) text = test_event.text self.assertIn(message, text) self.assertIn('Causes (most recent cause last):', text) self.assertEqual(2, text.count(causes[0]['traceback'])) # one test with task_rescheduled test_event = _event('cloudify_event', event_type='task_rescheduled', message=message, causes=causes, verbosity_level=event.LOW_VERBOSE) text = test_event.text self.assertIn(message, text) self.assertIn('Causes (most recent cause last):', text) self.assertEqual(2, text.count(causes[0]['traceback']))
def verify_server_is_up(port): for attempt in range(15): try: response = urllib2.urlopen("http://localhost:{0}".format(port)) response.read() break except BaseException: _, last_ex, last_tb = sys.exc_info() time.sleep(1) else: raise NonRecoverableError( "Failed to start HTTP webserver", causes=[exception_to_error_cause(last_ex, last_tb)])
def _handle_task_not_succeeded(self): """ Call handler for task which hasn't ended in 'succeeded' state (i.e. has either failed or been rescheduled) """ try: exception = self.async_result.result except Exception as e: exception = exceptions.NonRecoverableError( 'Could not de-serialize ' 'exception of task {0} --> {1}: {2}' .format(self.name, type(e).__name__, str(e))) if isinstance(exception, exceptions.OperationRetry): # operation explicitly requested a retry, so we ignore # the handler set on the task. handler_result = HandlerResult.retry() elif self.on_failure: handler_result = self.on_failure(self) else: handler_result = HandlerResult.retry() if handler_result.action == HandlerResult.HANDLER_RETRY: if isinstance(exception, exceptions.NonRecoverableError): handler_result = HandlerResult.fail() elif isinstance(exception, exceptions.RecoverableError): handler_result.retry_after = exception.retry_after if not self.is_subgraph: causes = [] if isinstance(exception, (exceptions.RecoverableError, exceptions.NonRecoverableError)): causes = exception.causes if isinstance(self, LocalWorkflowTask): tb = self.async_result._holder.error[1] causes.append(utils.exception_to_error_cause(exception, tb)) self.workflow_context.internal.send_task_event( state=self.get_state(), task=self, event={'exception': exception, 'causes': causes}) return handler_result
def make_client_call(self, client_method_name, client_method_args=None, log_response=True, fatal_handled_exceptions=FATAL_EXCEPTIONS): """ :param client_method_name: A method on self.client. :param client_method_args: Optional Args. :param log_response: Whether to log API response. :param fatal_handled_exceptions: exceptions to fail on. :return: Either Exception class or successful response content. """ type_name = getattr(self, 'type_name') self.logger.debug('Calling {0} method {1} with parameters: {2}'.format( type_name, client_method_name, client_method_args)) client_method = getattr(self.client, client_method_name) if not client_method: return try: if isinstance(client_method_args, dict): res = client_method(**client_method_args) elif isinstance(client_method_args, list): res = client_method(*client_method_args) else: res = client_method_args() except fatal_handled_exceptions as error: _, _, tb = sys.exc_info() if isinstance(error, ClientError) and hasattr(error, 'message'): message = error.message + NTP_NOTE else: message = 'API error encountered: {}'.format(error) raise NonRecoverableError( text_type(message), causes=[exception_to_error_cause(error, tb)]) else: if log_response: self.logger.debug('Response: {0}'.format(res)) return res
def wrapper_inner(**kwargs): # Get the context for the current task operation ctx = kwargs.pop('ctx', CloudifyContext) # Resolve the actual context which need to run operation, # the context could be belongs to relationship context or actual # node context ctx_node = resolve_ctx(ctx) # Get the current operation name operation_name = get_current_operation() try: # Prepare the openstack resource that need to execute the # current task operation resource = \ prepare_resource_instance(class_decl, ctx_node, kwargs) if use_external_resource(ctx_node, resource, existing_resource_handler, **existing_resource_kwargs): # Set external resource as runtime property set_external_resource(ctx_node, resource) return # check resource_id before stop/delete for already cleaned up if operation_name in (CLOUDIFY_STOP_OPERATION, CLOUDIFY_DELETE_OPERATION, CLOUDIFY_UNLINK_OPERATION): if not ctx_node.instance.runtime_properties.get( RESOURCE_ID): ctx.logger.info('Instance is already uninitialized.') return # run action kwargs['openstack_resource'] = resource func(**kwargs) update_runtime_properties_for_operation_task( operation_name, ctx_node, resource) except EXCEPTIONS as errors: _, _, tb = sys.exc_info() raise NonRecoverableError( 'Failure while trying to run operation:' '{0}: {1}'.format(operation_name, errors.message), causes=[exception_to_error_cause(errors, tb)])
def _amqp_client(self): # initialize an amqp client only when needed, ie. if the task is # not local with_amqp = bool(self.ctx.task_target) if with_amqp: try: amqp_client_utils.init_events_publisher() except Exception: _, ex, tb = sys.exc_info() # This one should never (!) raise an exception. amqp_client_utils.close_amqp_client() raise exceptions.RecoverableError( 'Failed initializing AMQP connection', causes=[utils.exception_to_error_cause(ex, tb)]) try: yield finally: if with_amqp: amqp_client_utils.close_amqp_client()
def wrapper_inner(**kwargs): # Get the context for the current task operation ctx = kwargs.pop('ctx', CloudifyContext) # Resolve the actual context which need to run operation, # the context could be belongs to relationship context or actual # node context ctx_node = resolve_ctx(ctx) # Get the current operation name operation_name = get_current_operation() # Prepare the openstack resource that need to execute the # current task operation resource = \ prepare_resource_instance(class_decl, ctx_node, kwargs) # Handle external resource when it is enabled if ctx_node.node.properties.get(USE_EXTERNAL_RESOURCE_PROPERTY): handle_external_resource(ctx_node, resource, existing_resource_handler, **existing_resource_kwargs) # Update runtime properties if not allow_to_run_operation_for_external_node( operation_name): # Update runtime properties for operation update_runtime_properties_for_operation_task( operation_name, ctx_node, resource) return try: kwargs['openstack_resource'] = resource func(**kwargs) update_runtime_properties_for_operation_task( operation_name, ctx_node, resource) except exceptions.SDKException as error: _, _, tb = sys.exc_info() raise NonRecoverableError( 'Failure while trying to request ' 'Openstack API: {}'.format(error.message), causes=[exception_to_error_cause(error, tb)])
def make_client_call(self, client_method_name, client_method_args=None, log_response=True, fatal_handled_exceptions=FATAL_EXCEPTIONS): """ :param client_method_name: A method on self.client. :param client_method_args: Optional Args. :param log_response: Whether to log API response. :param fatal_handled_exceptions: exceptions to fail on. :return: Either Exception class or successful response content. """ type_name = getattr(self, 'type_name') self.logger.debug( 'Calling {0} method {1} with parameters: {2}'.format( type_name, client_method_name, client_method_args)) client_method = getattr(self.client, client_method_name) if not client_method: return try: if isinstance(client_method_args, dict): res = client_method(**client_method_args) elif isinstance(client_method_args, list): res = client_method(*client_method_args) else: res = client_method_args() except fatal_handled_exceptions as error: _, _, tb = sys.exc_info() raise NonRecoverableError( str(error.message), causes=[exception_to_error_cause(error, tb)]) else: if log_response: self.logger.debug('Response: {0}'.format(res)) return res
def wrapper_inner(**kwargs): # Get the context for the current task operation ctx = kwargs.pop('ctx', CloudifyContext) # Resolve the actual context which need to run operation, # the context could be belongs to relationship context or actual # node context ctx_node = resolve_ctx(ctx) client_config = ctx_node.node.properties.get('client_config') cacert, cafile, cafilename = handle_cert_in_config(client_config) validate_auth_url(client_config['auth_url'], cacert, client_config.get('insecure')) resource_config = ctx_node.node.properties.get('resource_config') try: resource = class_decl(client_config=client_config, resource_config=resource_config, logger=ctx.logger) func(resource, ctx) except StarlingXException as errors: raise OperationRetry( 'Attempting WRCP registration again, ' 'due to invalid response from DC API. {e}'.format( e=errors)) except StarlingXFatalException as e: raise NonRecoverableError(e.message) except Exception as errors: _, _, tb = sys.exc_info() if hasattr(errors, 'message'): message = errors.message else: message = '' raise NonRecoverableError( 'Failure while trying to run operation:' '{0}: {1}'.format(ctx.operation.name, message), causes=[exception_to_error_cause(errors, tb)]) finally: if cafile and cafilename: os.close(cafile) os.remove(cafilename)
def generate_swift_access_config(auth_url, username, password): payload = dict() payload['X-Auth-User'] = username payload['X-Auth-Key'] = password # Try to generate the token and endpoint url to be used later on try: resp = requests.get(auth_url, headers=payload) resp.raise_for_status() except exceptions.HTTPError as error: _, _, tb = sys.exc_info() raise NonRecoverableError("Failed generating swift endpoint and token", causes=[exception_to_error_cause(error, tb)]) # Get the url which represent "endpoint_url" endpoint_url = urllib.parse.urljoin(resp.headers.get('X-Storage-Url'), '/') # This represent "aws_secret_access_key" which should be used with boto3 # client token = resp.headers['X-Auth-Token'] return endpoint_url, token
def lookup_remote_resource(_ctx, etcd_resource): """ This method will try to lookup etcd remote resource based on the instance type :param _ctx Cloudify node instance which is could be an instance of RelationshipSubjectContext or CloudifyContext :param etcd_resource: Instance derived from "EtcdResource", it could be "EtcdKeyValuePair" or "WatchKey" ..etc :return: list of etcd values stored remotely """ try: # Get the remote resource remote_resource_responses = map( lambda x: etcd_resource.get(x), get_keys_from_resource_config(_ctx) ) # remote_resource_responses are tuples of (value, KVMetadata) # with byte strings so conversion is needed remote_resources = list( map( lambda x: str(x[0]), remote_resource_responses ) ) except etcd3.exceptions.Etcd3Exception as error: _, _, tb = sys.exc_info() # If external resource does not exist then try to create it instead # of failed, when "create_if_missing" is set to "True" if is_create_if_missing(_ctx): _ctx.instance.runtime_properties[CONDITIONALLY_CREATED] = True etcd_resource.resource_id = None return None raise NonRecoverableError( 'Failure while trying to request ' 'etcd API: {}'.format(error.message), causes=[exception_to_error_cause(error, tb)]) return remote_resources
def _write_key_file(_key_file_path, _key_file_material, _private_key_permissions=False): expanded_key_path = os.path.expanduser(_key_file_path) with tempfile.NamedTemporaryFile('wb', delete=False) as temporary_file: temporary_file.write(_key_file_material) temporary_file.close() try: directory = os.path.dirname(expanded_key_path) if not os.path.exists(directory): os.makedirs(directory) shutil.move(temporary_file.name, expanded_key_path) except Exception: _, last_ex, last_tb = sys.exc_info() raise NonRecoverableError( "Failed moving private key", causes=[exception_to_error_cause(last_ex, last_tb)]) finally: if os.path.exists(temporary_file.name): os.remove(temporary_file.name) if _private_key_permissions: os.chmod(os.path.expanduser(_key_file_path), 0o600)
def generate_swift_access_config(auth_url, username, password): payload = dict() payload['X-Auth-User'] = username payload['X-Auth-Key'] = password # Try to generate the token and endpoint url to be used later on try: resp = requests.get(auth_url, headers=payload) resp.raise_for_status() except exceptions.HTTPError as error: _, _, tb = sys.exc_info() raise NonRecoverableError( "Failed generating swift endpoint and token", causes=[exception_to_error_cause(error, tb)]) # Get the url which represent "endpoint_url" endpoint_url = urllib.parse.urljoin(resp.headers.get('X-Storage-Url'), '/') # This represent "aws_secret_access_key" which should be used with boto3 # client token = resp.headers['X-Auth-Token'] return endpoint_url, token
def generate_traceback_exception(): _, exc_value, exc_traceback = sys.exc_info() response = exception_to_error_cause(exc_value, exc_traceback) ctx.logger.error( 'Error traceback {0} with message {1}'.format( response['traceback'], response['message']))
def generate_traceback_exception(): _, exc_value, exc_traceback = sys.exc_info() response = exception_to_error_cause(exc_value, exc_traceback) return response
def wrapper(**kwargs): ctx = kwargs['ctx'] node_type = ctx.node.type if node_type and node_type.startswith(SWIFT_NODE_PREFIX): response = None swift_config = ctx.node.properties.get('swift_config') username = swift_config.get('swift_username') password = swift_config.get('swift_password') auth_url = swift_config.get('swift_auth_url') region_name = swift_config.get('swift_region_name') aws_config = {} # Only Generate the token if it is not generated before if not ctx.instance.runtime_properties.get('aws_config'): endpoint_url, token = \ utils.generate_swift_access_config(auth_url, username, password) aws_config['aws_access_key_id'] = username aws_config['aws_secret_access_key'] = token aws_config['region_name'] = region_name aws_config['endpoint_url'] = endpoint_url ctx.instance.runtime_properties['aws_config'] = aws_config try: kwargs['aws_config'] = aws_config kwargs['ctx'] = ctx response = func(**kwargs) except ClientError as err: _, _, tb = sys.exc_info() error = err.response.get('Error') error_code = error.get('Code', 'Unknown') if error_code == SWIFT_ERROR_TOKEN_CODE: endpoint_url, token = \ utils.generate_swift_access_config(auth_url, username, password) # Reset the old "aws_config" and generate new one del ctx.instance.runtime_properties['aws_config'] aws_config = {} aws_config['aws_access_key_id'] = username aws_config['aws_secret_access_key'] = token aws_config['region_name'] = region_name aws_config['endpoint_url'] = endpoint_url ctx.instance.runtime_properties['aws_config'] =\ aws_config raise OperationRetry( 'Re-try the operation and generate new token' ' and endpoint url for swift connection', retry_after=10, causes=[exception_to_error_cause(error, tb)]) except Exception as error: error_traceback = utils.get_traceback_exception() raise NonRecoverableError('{0}'.format(str(error)), causes=[error_traceback]) return response return func(**kwargs)
def create(ctx, iface, resource_config, **_): """Creates an AWS S3 Bucket Object""" # Create a copy of the resource config for clean manipulation. params = utils.clean_params( dict() if not resource_config else resource_config.copy()) # Get the bucket object key from params object_key = params.get(OBJECT_KEY) if not object_key: raise NonRecoverableError('{0} param is required'.format(OBJECT_KEY)) utils.update_resource_id(ctx.instance, object_key) source_type = ctx.node.properties.get(OBJECT_SOURCE_TYPE) cloudify_path = None object_body = None # If "source_type" is either local or remote then we need to download # the file from remote or local directory and then parse the data as # bytes and prepared it to be sent on the "Body" param for "put_object" # method if source_type in [OBJECT_LOCAL_SOURCE, OBJECT_REMOTE_SOURCE]: path = ctx.node.properties.get(OBJECT_PATH) if not path: raise NonRecoverableError( 'path param must be provided when ' 'source_type is selected as remote or local') if source_type == OBJECT_LOCAL_SOURCE: cloudify_path = _download_local_file(path) elif source_type == OBJECT_REMOTE_SOURCE: cloudify_path = _download_remote_file(path) try: object_body = open(cloudify_path, 'rb') except IOError as error: _, _, tb = sys.exc_info() raise NonRecoverableError( 'Failed to open file {0},' ' with error message {1}' ''.format(path, error.strerror, causes=[exception_to_error_cause(error, tb)])) # Set the updated path url so that it can # be uploaded to the AWS S3 bucket params[BUCKET_OBJECT_BODY] = object_body # If the "source_type" is "bytes" then the body should provided from the # blueprint and follow the boto3 API documents elif source_type == OBJECT_BYTES_SOURCE: if not params.get(BUCKET_OBJECT_BODY): raise NonRecoverableError('Body param must be provided when ' 'source_type is selected as bytes') # Get the bucket name from either params or a relationship. bucket_name = params.get(BUCKET) if not bucket_name: targ = utils.find_rel_by_node_type(ctx.instance, BUCKET_TYPE) bucket_name = \ targ.target.instance.runtime_properties.get( EXTERNAL_RESOURCE_ID ) params[BUCKET] = bucket_name iface.bucket_name = bucket_name ctx.instance.runtime_properties[BUCKET] = bucket_name # Actually create the resource iface.create(params)
def generate_traceback_exception(): _, exc_value, exc_traceback = sys.exc_info() response = exception_to_error_cause(exc_value, exc_traceback) return response
def handle_external_resource(ctx_node_instance, openstack_resource, existing_resource_handler=None, **kwargs): """ :param ctx_node_instance: Cloudify context cloudify.context.CloudifyContext :param openstack_resource: Openstack resource instance :param existing_resource_handler: Callback handler that used to be called in order to execute custom operation when "use_external_resource" is enabled :param kwargs: Any extra param passed to the existing_resource_handler """ # Get the current operation name operation_name = get_current_operation() # Validate if the "is_external" is set and the resource # identifier (id|name) for the Openstack is invalid raise error and # abort the operation error_message = openstack_resource.validate_resource_identifier() # Raise error when validation failed if error_message: raise NonRecoverableError(error_message) # Cannot delete/create resource when it is external if operation_name in [ CLOUDIFY_CREATE_OPERATION, CLOUDIFY_DELETE_OPERATION ]: ctx.logger.info('Using external resource {0}'.format(RESOURCE_ID)) try: # Get the remote resource remote_resource = openstack_resource.get() except openstack.exceptions.SDKException as error: _, _, tb = sys.exc_info() raise NonRecoverableError( 'Failure while trying to request ' 'Openstack API: {}'.format(error.message), causes=[exception_to_error_cause(error, tb)]) # Check the operation type and based on that decide what to do if operation_name == CLOUDIFY_CREATE_OPERATION: ctx.logger.info('not creating resource {0}' ' since an external resource is being used' ''.format(remote_resource.name)) ctx_node_instance.instance.runtime_properties[RESOURCE_ID] \ = remote_resource.id # Just log message that we cannot delete resource elif operation_name == CLOUDIFY_DELETE_OPERATION: ctx.logger.info('not deleting resource {0}' ' since an external resource is being used' ''.format(remote_resource.name)) # Check if we need to run custom operation for already existed # resource for operation task if existing_resource_handler: # We may need to send the "openstack_resource" to the # existing resource handler and in order to do that we may # need to check if the resource is already there or not func_args = inspect.getargspec(existing_resource_handler).args if 'openstack_resource' in func_args: kwargs['openstack_resource'] = openstack_resource existing_resource_handler(**kwargs)