class MapReduceYaml(validation.Validated):
"""Root class for mapreduce.yaml.
File format:
mapreduce:
- name: <mapreduce_name>
mapper:
- input_reader: google.appengine.ext.mapreduce.DatastoreInputReader
- handler: path_to_my.MapperFunction
- params:
- name: foo
default: bar
- name: blah
default: stuff
- params_validator: path_to_my.ValidatorFunction
Where
mapreduce_name: The name of the mapreduce. Used for UI purposes.
mapper_handler_spec: Full <module_name>.<function_name/class_name> of
mapper handler. See MapreduceSpec class documentation for full handler
specification.
input_reader: Full <module_name>.<function_name/class_name> of the
InputReader sub-class to use for the mapper job.
params: A list of optional parameter names and optional default values
that may be supplied or overridden by the user running the job.
params_validator is full <module_name>.<function_name/class_name> of
a callable to validate the mapper_params after they are input by the
user running the job.
"""
ATTRIBUTES = {
"mapreduce": validation.Optional(validation.Repeated(MapreduceInfo))
}
@staticmethod
def to_dict(mapreduce_yaml):
"""Converts a MapReduceYaml file into a JSON-encodable dictionary.
For use in user-visible UI and internal methods for interfacing with
user code (like param validation). as a list
Args:
mapreduce_yaml: The Pyton representation of the mapreduce.yaml document.
Returns:
A list of configuration dictionaries.
"""
all_configs = []
for config in mapreduce_yaml.mapreduce:
out = {
"name": config.name,
"mapper_input_reader": config.mapper.input_reader,
"mapper_handler": config.mapper.handler,
}
if config.mapper.params_validator:
out["mapper_params_validator"] = config.mapper.params_validator
if config.mapper.params:
param_defaults = {}
for param in config.mapper.params:
param_defaults[param.name] = param.default or param.value
out["mapper_params"] = param_defaults
if config.params:
param_defaults = {}
for param in config.params:
param_defaults[param.name] = param.default or param.value
out["params"] = param_defaults
if config.mapper.output_writer:
out["mapper_output_writer"] = config.mapper.output_writer
all_configs.append(out)
return all_configs
class BackendInfoExternal(validation.Validated):
"""BackendInfoExternal describes all backend entries for an application."""
ATTRIBUTES = {
BACKENDS: validation.Optional(validation.Repeated(BackendEntry)),
}
class DispatchInfoExternal(validation.Validated):
"""Describes the format of a dispatch.yaml file."""
ATTRIBUTES = {
APPLICATION: validation.Optional(appinfo.APPLICATION_RE_STRING),
DISPATCH: validation.Optional(validation.Repeated(DispatchEntry)),
}
class CronInfoExternal(validation.Validated):
"""CronInfoExternal describes all cron entries for an application."""
ATTRIBUTES = {
CRON: validation.Optional(validation.Repeated(CronEntry))
}
class CronInfoExternal(validation.Validated):
"""CronInfoExternal describes all cron entries for an application."""
ATTRIBUTES = {
appinfo.APPLICATION: validation.Optional(appinfo.APPLICATION_RE_STRING),
CRON: validation.Optional(validation.Repeated(CronEntry))
}
class DosInfoExternal(validation.Validated):
"""Describes the format of a dos.yaml file."""
ATTRIBUTES = {
appinfo.APPLICATION: validation.Optional(appinfo.APPLICATION_RE_STRING),
BLACKLIST: validation.Optional(validation.Repeated(BlacklistEntry)),
}
class AppInfoExternal(validation.Validated):
"""Class representing users application info.
This class is passed to a yaml_object builder to provide the validation
for the application information file format parser.
Attributes:
application: Unique identifier for application.
version: Application's major version number.
runtime: Runtime used by application.
api_version: Which version of APIs to use.
handlers: List of URL handlers.
default_expiration: Default time delta to use for cache expiration for
all static files, unless they have their own specific 'expiration' set.
See the URLMap.expiration field's documentation for more information.
skip_files: An re object. Files that match this regular expression will
not be uploaded by appcfg.py. For example:
skip_files: |
.svn.*|
#.*#
"""
ATTRIBUTES = {
APPLICATION: APPLICATION_RE_STRING,
VERSION: VERSION_RE_STRING,
RUNTIME: RUNTIME_RE_STRING,
API_VERSION: API_VERSION_RE_STRING,
BUILTINS: validation.Optional(validation.Repeated(BuiltinHandler)),
INCLUDES: validation.Optional(validation.Type(list)),
HANDLERS: validation.Optional(validation.Repeated(URLMap)),
SERVICES: validation.Optional(validation.Repeated(
validation.Regex(_SERVICE_RE_STRING))),
DEFAULT_EXPIRATION: validation.Optional(_EXPIRATION_REGEX),
SKIP_FILES: validation.RegexStr(default=DEFAULT_SKIP_FILES),
DERIVED_FILE_TYPE: validation.Optional(validation.Repeated(
validation.Options(JAVA_PRECOMPILED, PYTHON_PRECOMPILED))),
ADMIN_CONSOLE: validation.Optional(AdminConsole),
ERROR_HANDLERS: validation.Optional(validation.Repeated(ErrorHandlers)),
THREADSAFE: validation.Optional(bool),
}
def CheckInitialized(self):
"""Performs non-regex-based validation.
Ensures that at least one url mapping is provided in the URL mappers
Also ensures that the major version doesn't contain the string
-dot-.
Raises:
MissingURLMapping when no URLMap objects are present in object.
TooManyURLMappings when there are too many URLMap entries.
"""
super(AppInfoExternal, self).CheckInitialized()
if not self.handlers and not self.builtins and not self.includes:
raise appinfo_errors.MissingURLMapping(
'No URLMap entries found in application configuration')
if self.handlers and len(self.handlers) > MAX_URL_MAPS:
raise appinfo_errors.TooManyURLMappings(
'Found more than %d URLMap entries in application configuration' %
MAX_URL_MAPS)
if self.version.find(ALTERNATE_HOSTNAME_SEPARATOR) != -1:
raise validation.ValidationError(
'App version "%s" cannot contain the string "%s"' % (
self.version, ALTERNATE_HOSTNAME_SEPARATOR))
class DosInfoExternal(validation.Validated):
ATTRIBUTES = {
BLACKLIST: validation.Optional(validation.Repeated(BlacklistEntry)),
}
class AppInclude(validation.Validated):
"""Class representing the contents of an included app.yaml file.
Used for both builtins and includes directives.
"""
ATTRIBUTES = {
BUILTINS: validation.Optional(validation.Repeated(BuiltinHandler)),
INCLUDES: validation.Optional(validation.Type(list)),
HANDLERS: validation.Optional(validation.Repeated(URLMap)),
ADMIN_CONSOLE: validation.Optional(AdminConsole),
}
@classmethod
def MergeAppYamlAppInclude(cls, appyaml, appinclude):
"""This function merges an app.yaml file with referenced builtins/includes.
"""
if not appinclude:
return appyaml
if appinclude.handlers:
tail = appyaml.handlers or []
appyaml.handlers = []
for h in appinclude.handlers:
if not h.position or h.position == 'head':
appyaml.handlers.append(h)
else:
tail.append(h)
appyaml.handlers.extend(tail)
appyaml.admin_console = AdminConsole.Merge(appyaml.admin_console,
appinclude.admin_console)
return appyaml
@classmethod
def MergeAppIncludes(cls, appinclude_one, appinclude_two):
"""This function merges the non-referential state of the provided AppInclude
objects. That is, builtins and includes directives are not preserved, but
any static objects are copied into an aggregate AppInclude object that
preserves the directives of both provided AppInclude objects.
Args:
appinclude_one: object one to merge
appinclude_two: object two to merge
Returns:
AppInclude object that is the result of merging the static directives of
appinclude_one and appinclude_two.
"""
if not appinclude_one or not appinclude_two:
return appinclude_one or appinclude_two
if appinclude_one.handlers:
if appinclude_two.handlers:
appinclude_one.handlers.extend(appinclude_two.handlers)
else:
appinclude_one.handlers = appinclude_two.handlers
appinclude_one.admin_console = (
AdminConsole.Merge(appinclude_one.admin_console,
appinclude_two.admin_console))
return appinclude_one
class QueueInfoExternal(validation.Validated):
"""QueueInfoExternal describes all queue entries for an application."""
ATTRIBUTES = {
TOTAL_STORAGE_LIMIT: validation.Optional(_TOTAL_STORAGE_LIMIT_REGEX),
QUEUE: validation.Optional(validation.Repeated(QueueEntry)),
}
class AppInfoExternal(validation.Validated):
"""Class representing users application info.
This class is passed to a yaml_object builder to provide the validation
for the application information file format parser.
Attributes:
application: Unique identifier for application.
version: Application's major version.
runtime: Runtime used by application.
api_version: Which version of APIs to use.
handlers: List of URL handlers.
default_expiration: Default time delta to use for cache expiration for
all static files, unless they have their own specific 'expiration' set.
See the URLMap.expiration field's documentation for more information.
skip_files: An re object. Files that match this regular expression will
not be uploaded by appcfg.py. For example:
skip_files: |
.svn.*|
#.*#
nobuild_files: An re object. Files that match this regular expression will
not be built into the app. Go only.
api_config: URL root and script/servlet path for enhanced api serving
"""
ATTRIBUTES = {
APPLICATION:
APPLICATION_RE_STRING,
VERSION:
validation.Optional(VERSION_RE_STRING),
RUNTIME:
RUNTIME_RE_STRING,
API_VERSION:
API_VERSION_RE_STRING,
BUILTINS:
validation.Optional(validation.Repeated(BuiltinHandler)),
INCLUDES:
validation.Optional(validation.Type(list)),
HANDLERS:
validation.Optional(validation.Repeated(URLMap)),
LIBRARIES:
validation.Optional(validation.Repeated(Library)),
SERVICES:
validation.Optional(
validation.Repeated(validation.Regex(_SERVICE_RE_STRING))),
DEFAULT_EXPIRATION:
validation.Optional(_EXPIRATION_REGEX),
SKIP_FILES:
validation.RegexStr(default=DEFAULT_SKIP_FILES),
NOBUILD_FILES:
validation.RegexStr(default=DEFAULT_NOBUILD_FILES),
DERIVED_FILE_TYPE:
validation.Optional(
validation.Repeated(
validation.Options(JAVA_PRECOMPILED, PYTHON_PRECOMPILED))),
ADMIN_CONSOLE:
validation.Optional(AdminConsole),
ERROR_HANDLERS:
validation.Optional(validation.Repeated(ErrorHandlers)),
BACKENDS:
validation.Optional(validation.Repeated(backendinfo.BackendEntry)),
THREADSAFE:
validation.Optional(bool),
API_CONFIG:
validation.Optional(ApiConfigHandler),
CODE_LOCK:
validation.Optional(bool),
}
def CheckInitialized(self):
"""Performs non-regex-based validation.
The following are verified:
- At least one url mapping is provided in the URL mappers.
- Number of url mappers doesn't exceed MAX_URL_MAPS.
- Major version does not contain the string -dot-.
- If api_endpoints are defined, an api_config stanza must be defined.
- If the runtime is python27 and threadsafe is set, then no CGI handlers
can be used.
- That the version name doesn't start with BUILTIN_NAME_PREFIX
Raises:
DuplicateLibrary: if the name library name is specified more than once.
MissingURLMapping: if no URLMap object is present in the object.
TooManyURLMappings: if there are too many URLMap entries.
MissingApiConfig: if api_endpoints exist without an api_config.
MissingThreadsafe: if threadsafe is not set but the runtime requires it.
ThreadsafeWithCgiHandler: if the runtime is python27, threadsafe is set
and CGI handlers are specified.
"""
super(AppInfoExternal, self).CheckInitialized()
if not self.handlers and not self.builtins and not self.includes:
raise appinfo_errors.MissingURLMapping(
'No URLMap entries found in application configuration')
if self.handlers and len(self.handlers) > MAX_URL_MAPS:
raise appinfo_errors.TooManyURLMappings(
'Found more than %d URLMap entries in application configuration'
% MAX_URL_MAPS)
if self.threadsafe is None and self.runtime == 'python27':
raise appinfo_errors.MissingThreadsafe(
'threadsafe must be present and set to either "yes" or "no"')
if self.libraries:
if self.runtime != 'python27':
raise appinfo_errors.RuntimeDoesNotSupportLibraries(
'libraries entries are only supported by the "python27" runtime'
)
library_names = [library.name for library in self.libraries]
for library_name in library_names:
if library_names.count(library_name) > 1:
raise appinfo_errors.DuplicateLibrary(
'Duplicate library entry for %s' % library_name)
if self.version and self.version.find(
ALTERNATE_HOSTNAME_SEPARATOR) != -1:
raise validation.ValidationError(
'Version "%s" cannot contain the string "%s"' %
(self.version, ALTERNATE_HOSTNAME_SEPARATOR))
if self.version and self.version.startswith(BUILTIN_NAME_PREFIX):
raise validation.ValidationError(
('Version "%s" cannot start with "%s" because it is a '
'reserved version name prefix.') %
(self.version, BUILTIN_NAME_PREFIX))
if self.handlers:
api_endpoints = [
handler.url for handler in self.handlers
if handler.GetHandlerType() == HANDLER_API_ENDPOINT
]
if api_endpoints and not self.api_config:
raise appinfo_errors.MissingApiConfig(
'An api_endpoint handler was specified, but the required '
'api_config stanza was not configured.')
if self.threadsafe and self.runtime == 'python27':
for handler in self.handlers:
if (handler.script and (handler.script.endswith('.py')
or '/' in handler.script)):
raise appinfo_errors.ThreadsafeWithCgiHandler(
'threadsafe cannot be enabled with CGI handler: %s'
% handler.script)
def GetAllLibraries(self):
"""Returns a list of all Library instances active for this configuration.
Returns:
The list of active Library instances for this configuration. This includes
directly-specified libraries as well as any required dependencies.
"""
if not self.libraries:
return []
library_names = set(library.name for library in self.libraries)
required_libraries = []
for library in self.libraries:
for required_name, required_version in REQUIRED_LIBRARIES.get(
(library.name, library.version), []):
if required_name not in library_names:
required_libraries.append(
Library(name=required_name, version=required_version))
return self.libraries + required_libraries
def ApplyBackendSettings(self, backend_name):
"""Applies settings from the indicated backend to the AppInfoExternal.
Backend entries may contain directives that modify other parts of the
app.yaml, such as the 'start' directive, which adds a handler for the start
request. This method performs those modifications.
Args:
backend_name: The name of a backend defined in 'backends'.
Raises:
BackendNotFound: If the indicated backend was not listed in 'backends'.
"""
if backend_name is None:
return
if self.backends is None:
raise appinfo_errors.BackendNotFound
self.version = backend_name
match = None
for backend in self.backends:
if backend.name != backend_name:
continue
if match:
raise appinfo_errors.DuplicateBackend
else:
match = backend
if match is None:
raise appinfo_errors.BackendNotFound
if match.start is None:
return
start_handler = URLMap(url=_START_PATH, script=match.start)
self.handlers.insert(0, start_handler)
class AppInfoExternal(validation.Validated):
"""Class representing users application info.
This class is passed to a yaml_object builder to provide the validation
for the application information file format parser.
Attributes:
application: Unique identifier for application.
version: Application's major version number.
runtime: Runtime used by application.
api_version: Which version of APIs to use.
handlers: List of URL handlers.
default_expiration: Default time delta to use for cache expiration for
all static files, unless they have their own specific 'expiration' set.
See the URLMap.expiration field's documentation for more information.
skip_files: An re object. Files that match this regular expression will
not be uploaded by appcfg.py. For example:
skip_files: |
.svn.*|
#.*#
"""
ATTRIBUTES = {
APPLICATION:
APPLICATION_RE_STRING,
VERSION:
VERSION_RE_STRING,
RUNTIME:
RUNTIME_RE_STRING,
API_VERSION:
API_VERSION_RE_STRING,
HANDLERS:
validation.Optional(validation.Repeated(URLMap)),
SERVICES:
validation.Optional(
validation.Repeated(validation.Regex(_SERVICE_RE_STRING))),
DEFAULT_EXPIRATION:
validation.Optional(_EXPIRATION_REGEX),
SKIP_FILES:
validation.RegexStr(default=DEFAULT_SKIP_FILES),
DERIVED_FILE_TYPE:
validation.Optional(
validation.Repeated(
validation.Options(JAVA_PRECOMPILED, PYTHON_PRECOMPILED))),
ADMIN_CONSOLE:
validation.Optional(AdminConsole),
}
def CheckInitialized(self):
"""Ensures that at least one url mapping is provided.
Raises:
MissingURLMapping when no URLMap objects are present in object.
TooManyURLMappings when there are too many URLMap entries.
"""
super(AppInfoExternal, self).CheckInitialized()
if not self.handlers:
raise appinfo_errors.MissingURLMapping(
'No URLMap entries found in application configuration')
if len(self.handlers) > MAX_URL_MAPS:
raise appinfo_errors.TooManyURLMappings(
'Found more than %d URLMap entries in application configuration'
% MAX_URL_MAPS)
def FixSecureDefaults(self):
"""Force omitted 'secure: ...' handler fields to 'secure: optional'.
The effect is that handler.secure is never equal to the (nominal)
default.
See http://b/issue?id=2073962.
"""
if self.handlers:
for handler in self.handlers:
if handler.secure == SECURE_DEFAULT:
handler.secure = SECURE_HTTP_OR_HTTPS
class AdminConsole(validation.Validated):
"""Class representing admin console directives in application info.
"""
ATTRIBUTES = {
PAGES: validation.Optional(validation.Repeated(AdminConsolePage)),
}
