def _CreateUniqueJobKeyForExistingJob(job, project): """Creates a key from the proto job instance's attributes passed as input. Args: job: An instance of job fetched from the backend. project: The base name of the project. Returns: A tuple of attributes used as a key to identify this job. """ return ( job.schedule.schedule, job.schedule.timeZone, job.appEngineHttpTarget.relativeUrl, job.description, convertors.CheckAndConvertStringToFloatIfApplicable( job.retryConfig.minBackoffDuration) if job.retryConfig else None, convertors.CheckAndConvertStringToFloatIfApplicable( job.retryConfig.maxBackoffDuration) if job.retryConfig else None, job.retryConfig.maxDoublings if job.retryConfig else None, convertors.CheckAndConvertStringToFloatIfApplicable( job.retryConfig.maxRetryDuration) if job.retryConfig else None, job.retryConfig.retryCount if job.retryConfig else None, parsers.ExtractTargetFromAppEngineHostUrl( job.appEngineHttpTarget.appEngineRouting.host, project), )
def _DoesAttributeNeedToBeUpdated(cur_queue_state, attribute, new_value): """Checks whether the attribute & value provided need to be updated. Note: We only check if the attribute exists in `queue.rateLimits` and `queue.retryConfig` since those are the only attributes we verify here. The only attribute we do not verify here is app-engine routing override which we handle separately. Args: cur_queue_state: apis.cloudtasks.<ver>.cloudtasks_<ver>_messages.Queue, The Queue instance fetched from the backend. attribute: Snake case representation of the CT API attribute name. One example is 'max_burst_size'. new_value: The value we are trying to set this attribute to. Returns: True if the attribute needs to be updated to the new value, False otherwise. """ proto_attribute_name = convertors.ConvertStringToCamelCase(attribute) if ( hasattr(cur_queue_state, 'rateLimits') and hasattr(cur_queue_state.rateLimits, proto_attribute_name) ): old_value = getattr(cur_queue_state.rateLimits, proto_attribute_name) elif hasattr(cur_queue_state.retryConfig, proto_attribute_name): old_value = getattr(cur_queue_state.retryConfig, proto_attribute_name) else: # Unable to get old attribute value. return True if old_value == new_value: return False if ( old_value is None and attribute != 'max_concurrent_dispatches' and attribute in constants.PUSH_QUEUES_APP_DEPLOY_DEFAULT_VALUES and new_value == constants.PUSH_QUEUES_APP_DEPLOY_DEFAULT_VALUES[attribute] ): return False if attribute == 'max_dispatches_per_second' and not new_value: # No need to set rate if rate specified is 0. Instead, we will pause the # queue if it is not already paused or blocked. return False if old_value is None or new_value is None: return True old_value = convertors.CheckAndConvertStringToFloatIfApplicable(old_value) new_value = convertors.CheckAndConvertStringToFloatIfApplicable(new_value) if ( isinstance(old_value, float) and isinstance(new_value, float) ): return not IsClose(old_value, new_value) return old_value != new_value
def _PostProcessMinMaxBackoff(cloud_task_args, used_default_value_for_min_backoff, cur_queue_state): """Checks min and max backoff values and updates the other value if needed. When uploading via queue.yaml files, if only one of the backoff values is specified, the other value will automatically be updated to the default value. If the default value does not satisfy the condition min_backoff <= max_backoff, then it is set equal to the other backoff value. Args: cloud_task_args: argparse.Namespace, A placeholder args namespace built to pass on forwards to Cloud Tasks API. used_default_value_for_min_backoff: A boolean value telling us if we used a default value for min_backoff or if it was specified explicitly in the YAML file. cur_queue_state: apis.cloudtasks.<ver>.cloudtasks_<ver>_messages.Queue, The Queue instance fetched from the backend if it exists, None otherwise. """ if cloud_task_args.type == 'pull': return min_backoff = convertors.CheckAndConvertStringToFloatIfApplicable( cloud_task_args.min_backoff) max_backoff = convertors.CheckAndConvertStringToFloatIfApplicable( cloud_task_args.max_backoff) if min_backoff > max_backoff: if used_default_value_for_min_backoff: min_backoff = max_backoff cloud_task_args.min_backoff = cloud_task_args.max_backoff _SetSpecifiedArg(cloud_task_args, 'min_backoff', cloud_task_args.max_backoff) else: max_backoff = min_backoff cloud_task_args.max_backoff = cloud_task_args.min_backoff _SetSpecifiedArg(cloud_task_args, 'max_backoff', cloud_task_args.min_backoff) # Check if the backend values match with what we are trying to set if cur_queue_state and cur_queue_state.retryConfig: old_min_backoff = convertors.CheckAndConvertStringToFloatIfApplicable( cur_queue_state.retryConfig.minBackoff) old_max_backoff = convertors.CheckAndConvertStringToFloatIfApplicable( cur_queue_state.retryConfig.maxBackoff) if max_backoff == old_max_backoff and min_backoff == old_min_backoff: _DeleteSpecifiedArg(cloud_task_args, 'min_backoff') cloud_task_args.min_backoff = None _DeleteSpecifiedArg(cloud_task_args, 'max_backoff') cloud_task_args.max_backoff = None
def _PostProcessMinMaxBackoff(cloud_task_args, used_default_value_for_min_backoff): """Checks min and max backoff values and updates the other value if needed. When uploading via queue.yaml files, if only one of the backoff values is specified, the other value will automatically be updated to the default value. If the default value does not satisfy the condition min_backoff <= max_backoff, then it is set equal to the other backoff value. Args: cloud_task_args: argparse.Namespace, A dummy args namespace built to pass on forwards to Cloud Tasks API. used_default_value_for_min_backoff: A boolean value telling us if we used a default value for min_backoff or if it was specified explicitly in the YAML file. Returns: argparse.Namespace, The same dummy args namespace but with min_backoff and max_backoff values set appropriately. """ min_backoff_specified = cloud_task_args.IsSpecified('min_backoff') max_backoff_specified = cloud_task_args.IsSpecified('max_backoff') min_backoff = convertors.CheckAndConvertStringToFloatIfApplicable( cloud_task_args.min_backoff) max_backoff = convertors.CheckAndConvertStringToFloatIfApplicable( cloud_task_args.max_backoff) if min_backoff_specified and max_backoff_specified: if min_backoff > max_backoff: if used_default_value_for_min_backoff: cloud_task_args.min_backoff = cloud_task_args.max_backoff _SetSpecifiedArg(cloud_task_args, 'min_backoff', cloud_task_args.max_backoff) else: cloud_task_args.max_backoff = cloud_task_args.min_backoff _SetSpecifiedArg(cloud_task_args, 'max_backoff', cloud_task_args.min_backoff) elif min_backoff_specified and not max_backoff_specified: if min_backoff > 3600: cloud_task_args.max_backoff = cloud_task_args.min_backoff _SetSpecifiedArg(cloud_task_args, 'max_backoff', cloud_task_args.min_backoff) elif max_backoff_specified and not min_backoff_specified: if max_backoff < 0.1: cloud_task_args.min_backoff = cloud_task_args.max_backoff _SetSpecifiedArg(cloud_task_args, 'min_backoff', cloud_task_args.max_backoff)
def ValidateCronYamlFileConfig(config): """Validates jobs configuration parameters in the cron YAML file. The purpose of this function is to mimick the behaviour of the old implementation of `gcloud app deploy cron.yaml` before migrating away from console-admin-hr. The errors generated are the same as the ones previously seen when gcloud sent the batch-request for updating jobs to the Zeus backend. Args: config: A yaml_parsing.ConfigYamlInfo object for the parsed YAML file we are going to process. Raises: HTTPError: Various different scenarios defined in the function can cause this exception to be raised. """ cron_yaml = config.parsed if not cron_yaml.cron: return for job in cron_yaml.cron: # Retry Parameters if job.retry_parameters: # Job Retry Limit if ( job.retry_parameters.job_retry_limit and job.retry_parameters.job_retry_limit > 5 ): _RaiseHTTPException( 'Invalid Cron retry parameters: Cannot set retry limit to more ' 'than 5 (currently set to {}).'.format( job.retry_parameters.job_retry_limit)) # Job Age Limit if ( job.retry_parameters.job_age_limit and int(convertors.CheckAndConvertStringToFloatIfApplicable( job.retry_parameters.job_age_limit)) <= 0 ): _RaiseHTTPException( 'Invalid Cron retry parameters: Job age limit must be greater ' 'than zero seconds.') # Min & Max backoff comparison if ( job.retry_parameters.min_backoff_seconds is not None and job.retry_parameters.max_backoff_seconds is not None ): min_backoff = job.retry_parameters.min_backoff_seconds max_backoff = job.retry_parameters.max_backoff_seconds if max_backoff < min_backoff: _RaiseHTTPException( 'Invalid Cron retry parameters: Min backoff sec must not be ' 'greater than than max backoff sec.')
def testCheckAndConvertStringToFloatIfApplicable(self): combinations_to_test = ( ('2m', 120), ('1.5h', 5400), ('8.5s', 8.5), ('1d', 86400), ('max_retry_duration', 'max_retry_duration'), (17.4, 17.4), ) for input_str, output in combinations_to_test: self.assertEqual( convertors.CheckAndConvertStringToFloatIfApplicable(input_str), output)
def _CreateUniqueJobKeyForYamlJob(job): """Creates a key from the YAML job instance's attributes passed as input. Args: job: An instance of a parsed YAML job object. Returns: A tuple of attributes used as a key to identify this job. """ retry_params = job.retry_parameters return ( job.schedule, job.timezone if job.timezone else 'UTC', job.url, job.description, retry_params.min_backoff_seconds if retry_params else None, retry_params.max_backoff_seconds if retry_params else None, retry_params.max_doublings if retry_params else None, convertors.CheckAndConvertStringToFloatIfApplicable( retry_params.job_age_limit) if retry_params else None, retry_params.job_retry_limit if retry_params else None, job.target, )
def ValidateQueueYamlFileConfig(config): """Validates queue configuration parameters in the queue YAML file. The purpose of this function is to mimick the behaviour of the old implementation of `gcloud app deploy queue.yaml` before migrating away from console-admin-hr. The errors generated are the same as the ones previously seen when gcloud sent the batch-request for updating queues to the Zeus backend. Args: config: A yaml_parsing.ConfigYamlInfo object for the parsed YAML file we are going to process. Raises: HTTPError: Various different scenarios defined in the function can cause this exception to be raised. """ queue_yaml = config.parsed if not queue_yaml.queue: return for queue in queue_yaml.queue: # Push queues if not queue.mode or queue.mode == constants.PUSH_QUEUE: # Rate if not queue.rate: _RaiseHTTPException( 'Invalid queue configuration. Refill rate must be specified for ' 'push-based queue.') else: rate_in_seconds = convertors.ConvertRate(queue.rate) if rate_in_seconds > constants.MAX_RATE: _RaiseHTTPException( 'Invalid queue configuration. Refill rate must not exceed ' '{} per second (is {:.1f}).'.format( constants.MAX_RATE, rate_in_seconds)) # Retry Parameters if queue.retry_parameters: # Task Retry Limit _ValidateTaskRetryLimit(queue) # Task Age Limit if ( queue.retry_parameters.task_age_limit and int(convertors.CheckAndConvertStringToFloatIfApplicable( queue.retry_parameters.task_age_limit)) <= 0 ): _RaiseHTTPException( 'Invalid queue configuration. Task age limit must be greater ' 'than zero.') # Min backoff if ( queue.retry_parameters.min_backoff_seconds and queue.retry_parameters.min_backoff_seconds < 0 ): _RaiseHTTPException( 'Invalid queue configuration. Min backoff seconds must not be ' 'less than zero.') # Max backoff if ( queue.retry_parameters.max_backoff_seconds and queue.retry_parameters.max_backoff_seconds < 0 ): _RaiseHTTPException( 'Invalid queue configuration. Max backoff seconds must not be ' 'less than zero.') # Max Doublings if ( queue.retry_parameters.max_doublings and queue.retry_parameters.max_doublings < 0 ): _RaiseHTTPException( 'Invalid queue configuration. Max doublings must not be less ' 'than zero.') # Min & Max backoff comparison if ( queue.retry_parameters.min_backoff_seconds is not None and queue.retry_parameters.max_backoff_seconds is not None ): min_backoff = queue.retry_parameters.min_backoff_seconds max_backoff = queue.retry_parameters.max_backoff_seconds if max_backoff < min_backoff: _RaiseHTTPException( 'Invalid queue configuration. Min backoff sec must not be ' 'greater than than max backoff sec.') # Bucket size if queue.bucket_size: if queue.bucket_size < 0: _RaiseHTTPException( 'Error updating queue "{}": The queue rate is invalid.'.format( queue.name)) elif queue.bucket_size > constants.MAX_BUCKET_SIZE: _RaiseHTTPException( 'Error updating queue "{}": Maximum bucket size is {}.'.format( queue.name, constants.MAX_BUCKET_SIZE)) # Pull Queues else: # Rate if queue.rate: _RaiseHTTPException( 'Invalid queue configuration. Refill rate must not be specified ' 'for pull-based queue.') # Retry Parameters if queue.retry_parameters: # Task Retry Limit _ValidateTaskRetryLimit(queue) # Task Age Limit if queue.retry_parameters.task_age_limit is not None: _RaiseHTTPException( "Invalid queue configuration. Can't specify task_age_limit " "for a pull queue.") # Min backoff if queue.retry_parameters.min_backoff_seconds is not None: _RaiseHTTPException( "Invalid queue configuration. Can't specify min_backoff_seconds " "for a pull queue.") # Max backoff if queue.retry_parameters.max_backoff_seconds is not None: _RaiseHTTPException( "Invalid queue configuration. Can't specify max_backoff_seconds " "for a pull queue.") # Max doublings if queue.retry_parameters.max_doublings is not None: _RaiseHTTPException( "Invalid queue configuration. Can't specify max_doublings " "for a pull queue.") # Max concurrent requests if queue.max_concurrent_requests is not None: _RaiseHTTPException( 'Invalid queue configuration. Max concurrent requests must not ' 'be specified for pull-based queue.') # Bucket size if queue.bucket_size is not None: _RaiseHTTPException( 'Invalid queue configuration. Bucket size must not be specified ' 'for pull-based queue.') # Target if queue.target: _RaiseHTTPException( 'Invalid queue configuration. Target must not be specified for ' 'pull-based queue.')