class MapReduceYaml(validation.Validated): """Root class for mapreduce.yaml. File format: mapreduce: - name: <mapreduce_name> mapper: - input_reader: google.appengine.ext.mapreduce.DatastoreInputReader - handler: path_to_my.MapperFunction - params: - name: foo default: bar - name: blah default: stuff - params_validator: path_to_my.ValidatorFunction Where mapreduce_name: The name of the mapreduce. Used for UI purposes. mapper_handler_spec: Full <module_name>.<function_name/class_name> of mapper handler. See MapreduceSpec class documentation for full handler specification. input_reader: Full <module_name>.<function_name/class_name> of the InputReader sub-class to use for the mapper job. params: A list of optional parameter names and optional default values that may be supplied or overridden by the user running the job. params_validator is full <module_name>.<function_name/class_name> of a callable to validate the mapper_params after they are input by the user running the job. """ ATTRIBUTES = { "mapreduce": validation.Optional(validation.Repeated(MapreduceInfo)) } @staticmethod def to_dict(mapreduce_yaml): """Converts a MapReduceYaml file into a JSON-encodable dictionary. For use in user-visible UI and internal methods for interfacing with user code (like param validation). as a list Args: mapreduce_yaml: The Pyton representation of the mapreduce.yaml document. Returns: A list of configuration dictionaries. """ all_configs = [] for config in mapreduce_yaml.mapreduce: out = { "name": config.name, "mapper_input_reader": config.mapper.input_reader, "mapper_handler": config.mapper.handler, } if config.mapper.params_validator: out["mapper_params_validator"] = config.mapper.params_validator if config.mapper.params: param_defaults = {} for param in config.mapper.params: param_defaults[param.name] = param.default or param.value out["mapper_params"] = param_defaults if config.params: param_defaults = {} for param in config.params: param_defaults[param.name] = param.default or param.value out["params"] = param_defaults if config.mapper.output_writer: out["mapper_output_writer"] = config.mapper.output_writer all_configs.append(out) return all_configs
class BackendInfoExternal(validation.Validated): """BackendInfoExternal describes all backend entries for an application.""" ATTRIBUTES = { BACKENDS: validation.Optional(validation.Repeated(BackendEntry)), }
class DispatchInfoExternal(validation.Validated): """Describes the format of a dispatch.yaml file.""" ATTRIBUTES = { APPLICATION: validation.Optional(appinfo.APPLICATION_RE_STRING), DISPATCH: validation.Optional(validation.Repeated(DispatchEntry)), }
class CronInfoExternal(validation.Validated): """CronInfoExternal describes all cron entries for an application.""" ATTRIBUTES = { CRON: validation.Optional(validation.Repeated(CronEntry)) }
class CronInfoExternal(validation.Validated): """CronInfoExternal describes all cron entries for an application.""" ATTRIBUTES = { appinfo.APPLICATION: validation.Optional(appinfo.APPLICATION_RE_STRING), CRON: validation.Optional(validation.Repeated(CronEntry)) }
class DosInfoExternal(validation.Validated): """Describes the format of a dos.yaml file.""" ATTRIBUTES = { appinfo.APPLICATION: validation.Optional(appinfo.APPLICATION_RE_STRING), BLACKLIST: validation.Optional(validation.Repeated(BlacklistEntry)), }
class AppInfoExternal(validation.Validated): """Class representing users application info. This class is passed to a yaml_object builder to provide the validation for the application information file format parser. Attributes: application: Unique identifier for application. version: Application's major version number. runtime: Runtime used by application. api_version: Which version of APIs to use. handlers: List of URL handlers. default_expiration: Default time delta to use for cache expiration for all static files, unless they have their own specific 'expiration' set. See the URLMap.expiration field's documentation for more information. skip_files: An re object. Files that match this regular expression will not be uploaded by appcfg.py. For example: skip_files: | .svn.*| #.*# """ ATTRIBUTES = { APPLICATION: APPLICATION_RE_STRING, VERSION: VERSION_RE_STRING, RUNTIME: RUNTIME_RE_STRING, API_VERSION: API_VERSION_RE_STRING, BUILTINS: validation.Optional(validation.Repeated(BuiltinHandler)), INCLUDES: validation.Optional(validation.Type(list)), HANDLERS: validation.Optional(validation.Repeated(URLMap)), SERVICES: validation.Optional(validation.Repeated( validation.Regex(_SERVICE_RE_STRING))), DEFAULT_EXPIRATION: validation.Optional(_EXPIRATION_REGEX), SKIP_FILES: validation.RegexStr(default=DEFAULT_SKIP_FILES), DERIVED_FILE_TYPE: validation.Optional(validation.Repeated( validation.Options(JAVA_PRECOMPILED, PYTHON_PRECOMPILED))), ADMIN_CONSOLE: validation.Optional(AdminConsole), ERROR_HANDLERS: validation.Optional(validation.Repeated(ErrorHandlers)), THREADSAFE: validation.Optional(bool), } def CheckInitialized(self): """Performs non-regex-based validation. Ensures that at least one url mapping is provided in the URL mappers Also ensures that the major version doesn't contain the string -dot-. Raises: MissingURLMapping when no URLMap objects are present in object. TooManyURLMappings when there are too many URLMap entries. """ super(AppInfoExternal, self).CheckInitialized() if not self.handlers and not self.builtins and not self.includes: raise appinfo_errors.MissingURLMapping( 'No URLMap entries found in application configuration') if self.handlers and len(self.handlers) > MAX_URL_MAPS: raise appinfo_errors.TooManyURLMappings( 'Found more than %d URLMap entries in application configuration' % MAX_URL_MAPS) if self.version.find(ALTERNATE_HOSTNAME_SEPARATOR) != -1: raise validation.ValidationError( 'App version "%s" cannot contain the string "%s"' % ( self.version, ALTERNATE_HOSTNAME_SEPARATOR))
class DosInfoExternal(validation.Validated): ATTRIBUTES = { BLACKLIST: validation.Optional(validation.Repeated(BlacklistEntry)), }
class AppInclude(validation.Validated): """Class representing the contents of an included app.yaml file. Used for both builtins and includes directives. """ ATTRIBUTES = { BUILTINS: validation.Optional(validation.Repeated(BuiltinHandler)), INCLUDES: validation.Optional(validation.Type(list)), HANDLERS: validation.Optional(validation.Repeated(URLMap)), ADMIN_CONSOLE: validation.Optional(AdminConsole), } @classmethod def MergeAppYamlAppInclude(cls, appyaml, appinclude): """This function merges an app.yaml file with referenced builtins/includes. """ if not appinclude: return appyaml if appinclude.handlers: tail = appyaml.handlers or [] appyaml.handlers = [] for h in appinclude.handlers: if not h.position or h.position == 'head': appyaml.handlers.append(h) else: tail.append(h) appyaml.handlers.extend(tail) appyaml.admin_console = AdminConsole.Merge(appyaml.admin_console, appinclude.admin_console) return appyaml @classmethod def MergeAppIncludes(cls, appinclude_one, appinclude_two): """This function merges the non-referential state of the provided AppInclude objects. That is, builtins and includes directives are not preserved, but any static objects are copied into an aggregate AppInclude object that preserves the directives of both provided AppInclude objects. Args: appinclude_one: object one to merge appinclude_two: object two to merge Returns: AppInclude object that is the result of merging the static directives of appinclude_one and appinclude_two. """ if not appinclude_one or not appinclude_two: return appinclude_one or appinclude_two if appinclude_one.handlers: if appinclude_two.handlers: appinclude_one.handlers.extend(appinclude_two.handlers) else: appinclude_one.handlers = appinclude_two.handlers appinclude_one.admin_console = ( AdminConsole.Merge(appinclude_one.admin_console, appinclude_two.admin_console)) return appinclude_one
class QueueInfoExternal(validation.Validated): """QueueInfoExternal describes all queue entries for an application.""" ATTRIBUTES = { TOTAL_STORAGE_LIMIT: validation.Optional(_TOTAL_STORAGE_LIMIT_REGEX), QUEUE: validation.Optional(validation.Repeated(QueueEntry)), }
class AppInfoExternal(validation.Validated): """Class representing users application info. This class is passed to a yaml_object builder to provide the validation for the application information file format parser. Attributes: application: Unique identifier for application. version: Application's major version. runtime: Runtime used by application. api_version: Which version of APIs to use. handlers: List of URL handlers. default_expiration: Default time delta to use for cache expiration for all static files, unless they have their own specific 'expiration' set. See the URLMap.expiration field's documentation for more information. skip_files: An re object. Files that match this regular expression will not be uploaded by appcfg.py. For example: skip_files: | .svn.*| #.*# nobuild_files: An re object. Files that match this regular expression will not be built into the app. Go only. api_config: URL root and script/servlet path for enhanced api serving """ ATTRIBUTES = { APPLICATION: APPLICATION_RE_STRING, VERSION: validation.Optional(VERSION_RE_STRING), RUNTIME: RUNTIME_RE_STRING, API_VERSION: API_VERSION_RE_STRING, BUILTINS: validation.Optional(validation.Repeated(BuiltinHandler)), INCLUDES: validation.Optional(validation.Type(list)), HANDLERS: validation.Optional(validation.Repeated(URLMap)), LIBRARIES: validation.Optional(validation.Repeated(Library)), SERVICES: validation.Optional( validation.Repeated(validation.Regex(_SERVICE_RE_STRING))), DEFAULT_EXPIRATION: validation.Optional(_EXPIRATION_REGEX), SKIP_FILES: validation.RegexStr(default=DEFAULT_SKIP_FILES), NOBUILD_FILES: validation.RegexStr(default=DEFAULT_NOBUILD_FILES), DERIVED_FILE_TYPE: validation.Optional( validation.Repeated( validation.Options(JAVA_PRECOMPILED, PYTHON_PRECOMPILED))), ADMIN_CONSOLE: validation.Optional(AdminConsole), ERROR_HANDLERS: validation.Optional(validation.Repeated(ErrorHandlers)), BACKENDS: validation.Optional(validation.Repeated(backendinfo.BackendEntry)), THREADSAFE: validation.Optional(bool), API_CONFIG: validation.Optional(ApiConfigHandler), CODE_LOCK: validation.Optional(bool), } def CheckInitialized(self): """Performs non-regex-based validation. The following are verified: - At least one url mapping is provided in the URL mappers. - Number of url mappers doesn't exceed MAX_URL_MAPS. - Major version does not contain the string -dot-. - If api_endpoints are defined, an api_config stanza must be defined. - If the runtime is python27 and threadsafe is set, then no CGI handlers can be used. - That the version name doesn't start with BUILTIN_NAME_PREFIX Raises: DuplicateLibrary: if the name library name is specified more than once. MissingURLMapping: if no URLMap object is present in the object. TooManyURLMappings: if there are too many URLMap entries. MissingApiConfig: if api_endpoints exist without an api_config. MissingThreadsafe: if threadsafe is not set but the runtime requires it. ThreadsafeWithCgiHandler: if the runtime is python27, threadsafe is set and CGI handlers are specified. """ super(AppInfoExternal, self).CheckInitialized() if not self.handlers and not self.builtins and not self.includes: raise appinfo_errors.MissingURLMapping( 'No URLMap entries found in application configuration') if self.handlers and len(self.handlers) > MAX_URL_MAPS: raise appinfo_errors.TooManyURLMappings( 'Found more than %d URLMap entries in application configuration' % MAX_URL_MAPS) if self.threadsafe is None and self.runtime == 'python27': raise appinfo_errors.MissingThreadsafe( 'threadsafe must be present and set to either "yes" or "no"') if self.libraries: if self.runtime != 'python27': raise appinfo_errors.RuntimeDoesNotSupportLibraries( 'libraries entries are only supported by the "python27" runtime' ) library_names = [library.name for library in self.libraries] for library_name in library_names: if library_names.count(library_name) > 1: raise appinfo_errors.DuplicateLibrary( 'Duplicate library entry for %s' % library_name) if self.version and self.version.find( ALTERNATE_HOSTNAME_SEPARATOR) != -1: raise validation.ValidationError( 'Version "%s" cannot contain the string "%s"' % (self.version, ALTERNATE_HOSTNAME_SEPARATOR)) if self.version and self.version.startswith(BUILTIN_NAME_PREFIX): raise validation.ValidationError( ('Version "%s" cannot start with "%s" because it is a ' 'reserved version name prefix.') % (self.version, BUILTIN_NAME_PREFIX)) if self.handlers: api_endpoints = [ handler.url for handler in self.handlers if handler.GetHandlerType() == HANDLER_API_ENDPOINT ] if api_endpoints and not self.api_config: raise appinfo_errors.MissingApiConfig( 'An api_endpoint handler was specified, but the required ' 'api_config stanza was not configured.') if self.threadsafe and self.runtime == 'python27': for handler in self.handlers: if (handler.script and (handler.script.endswith('.py') or '/' in handler.script)): raise appinfo_errors.ThreadsafeWithCgiHandler( 'threadsafe cannot be enabled with CGI handler: %s' % handler.script) def GetAllLibraries(self): """Returns a list of all Library instances active for this configuration. Returns: The list of active Library instances for this configuration. This includes directly-specified libraries as well as any required dependencies. """ if not self.libraries: return [] library_names = set(library.name for library in self.libraries) required_libraries = [] for library in self.libraries: for required_name, required_version in REQUIRED_LIBRARIES.get( (library.name, library.version), []): if required_name not in library_names: required_libraries.append( Library(name=required_name, version=required_version)) return self.libraries + required_libraries def ApplyBackendSettings(self, backend_name): """Applies settings from the indicated backend to the AppInfoExternal. Backend entries may contain directives that modify other parts of the app.yaml, such as the 'start' directive, which adds a handler for the start request. This method performs those modifications. Args: backend_name: The name of a backend defined in 'backends'. Raises: BackendNotFound: If the indicated backend was not listed in 'backends'. """ if backend_name is None: return if self.backends is None: raise appinfo_errors.BackendNotFound self.version = backend_name match = None for backend in self.backends: if backend.name != backend_name: continue if match: raise appinfo_errors.DuplicateBackend else: match = backend if match is None: raise appinfo_errors.BackendNotFound if match.start is None: return start_handler = URLMap(url=_START_PATH, script=match.start) self.handlers.insert(0, start_handler)
class AppInfoExternal(validation.Validated): """Class representing users application info. This class is passed to a yaml_object builder to provide the validation for the application information file format parser. Attributes: application: Unique identifier for application. version: Application's major version number. runtime: Runtime used by application. api_version: Which version of APIs to use. handlers: List of URL handlers. default_expiration: Default time delta to use for cache expiration for all static files, unless they have their own specific 'expiration' set. See the URLMap.expiration field's documentation for more information. skip_files: An re object. Files that match this regular expression will not be uploaded by appcfg.py. For example: skip_files: | .svn.*| #.*# """ ATTRIBUTES = { APPLICATION: APPLICATION_RE_STRING, VERSION: VERSION_RE_STRING, RUNTIME: RUNTIME_RE_STRING, API_VERSION: API_VERSION_RE_STRING, HANDLERS: validation.Optional(validation.Repeated(URLMap)), SERVICES: validation.Optional( validation.Repeated(validation.Regex(_SERVICE_RE_STRING))), DEFAULT_EXPIRATION: validation.Optional(_EXPIRATION_REGEX), SKIP_FILES: validation.RegexStr(default=DEFAULT_SKIP_FILES), DERIVED_FILE_TYPE: validation.Optional( validation.Repeated( validation.Options(JAVA_PRECOMPILED, PYTHON_PRECOMPILED))), ADMIN_CONSOLE: validation.Optional(AdminConsole), } def CheckInitialized(self): """Ensures that at least one url mapping is provided. Raises: MissingURLMapping when no URLMap objects are present in object. TooManyURLMappings when there are too many URLMap entries. """ super(AppInfoExternal, self).CheckInitialized() if not self.handlers: raise appinfo_errors.MissingURLMapping( 'No URLMap entries found in application configuration') if len(self.handlers) > MAX_URL_MAPS: raise appinfo_errors.TooManyURLMappings( 'Found more than %d URLMap entries in application configuration' % MAX_URL_MAPS) def FixSecureDefaults(self): """Force omitted 'secure: ...' handler fields to 'secure: optional'. The effect is that handler.secure is never equal to the (nominal) default. See http://b/issue?id=2073962. """ if self.handlers: for handler in self.handlers: if handler.secure == SECURE_DEFAULT: handler.secure = SECURE_HTTP_OR_HTTPS
class AdminConsole(validation.Validated): """Class representing admin console directives in application info. """ ATTRIBUTES = { PAGES: validation.Optional(validation.Repeated(AdminConsolePage)), }