class DosInfoExternal(validation.Validated): """Describes the format of a dos.yaml file.""" ATTRIBUTES = { appinfo.APPLICATION: validation.Optional(appinfo.APPLICATION_RE_STRING), BLACKLIST: validation.Optional(validation.Repeated(BlacklistEntry)), }
class CronInfoExternal(validation.Validated): """CronInfoExternal describes all cron entries for an application.""" ATTRIBUTES = { appinfo.APPLICATION: validation.Optional(appinfo.APPLICATION_RE_STRING), CRON: validation.Optional(validation.Repeated(CronEntry)) }
class DispatchEntry(validation.Validated): """A Dispatch entry describes a mapping from a URL pattern to a module.""" ATTRIBUTES = { URL: DispatchEntryURLValidator(), MODULE: validation.Optional(appinfo.MODULE_ID_RE_STRING), SERVICE: validation.Optional(appinfo.MODULE_ID_RE_STRING) }
class OptionalStuff(validation.Validated): ATTRIBUTES = { 'optional': validation.Optional(validation.TYPE_STR), 'implicit_type': validation.Optional(int, default=2001), 'implicit_regex': validation.Optional('[0-9]+'), 'implicit_options': validation.Optional(('a', 'b', 'c')), }
class PropertyEntry(validation.Validated): """Describes the transform for a single property.""" ATTRIBUTES = { 'property': validation.Type(str), 'import_transform': OPTIONAL_EVALUATED_CALLABLE, 'import_template': validation.Optional(validation.TYPE_STR), 'default_value': validation.Optional(validation.TYPE_STR), 'export': validation.Optional(validation.Repeated(ExportEntry)), } ATTRIBUTES.update(ExportEntry.ATTRIBUTES) def CheckInitialized(self): """Check that all required (combinations) of fields are set. Also fills in computed properties. Raises: InvalidConfiguration: If the config is invalid. """ super(PropertyEntry, self).CheckInitialized() if not (self.external_name or self.import_template or self.export): raise bulkloader_errors.InvalidConfiguration( 'Neither external_name nor import_template nor export specified for ' 'property %s.' % self.property)
class PagespeedEntry(validation.Validated): """Describes the format of a pagespeed configuration from a yaml file. URL blacklist entries are patterns (with '?' and '*' as wildcards). Any URLs that match a pattern on the blacklist will not be optimized by PageSpeed. Rewriter names are strings (like 'CombineCss' or 'RemoveComments') describing individual PageSpeed rewriters. A full list of valid rewriter names can be found in the PageSpeed documentation. The domains-to-rewrite list is a whitelist of domain name patterns with '*' as a wildcard, optionally starting with 'http://' or 'https://'. If no protocol is given, 'http://' is assumed. A resource will only be rewritten if it is on the same domain as the HTML that references it, or if its domain is on the domains-to-rewrite list. """ ATTRIBUTES = { URL_BLACKLIST: validation.Optional( validation.Repeated(validation.Regex(_URL_BLACKLIST_REGEX))), ENABLED_REWRITERS: validation.Optional( validation.Repeated(validation.Regex(_REWRITER_NAME_REGEX))), DISABLED_REWRITERS: validation.Optional( validation.Repeated(validation.Regex(_REWRITER_NAME_REGEX))), DOMAINS_TO_REWRITE: validation.Optional( validation.Repeated(validation.Regex(_DOMAINS_TO_REWRITE_REGEX))), }
class Property(validation.Validated): """Representation for an individual property of an index. Attributes: name: Name of attribute to sort by. direction: Direction of sort. mode: How the property is indexed. Either 'geospatial' or None (unspecified). """ ATTRIBUTES = { 'name': validation.Type(str, convert=False), 'direction': validation.Optional([('asc', ('ascending', )), ('desc', ('descending', ))]), 'mode': validation.Optional(['geospatial']) } def IsAscending(self): return self.direction != 'desc' def CheckInitialized(self): if self.direction is not None and self.mode is not None: raise validation.ValidationError( 'direction and mode are mutually exclusive') super(Property, self).CheckInitialized()
class ConnectorSubOptions(validation.Validated): """Connector options.""" ATTRIBUTES = { 'delimiter': validation.Optional(validation.TYPE_STR), 'dialect': validation.Optional(validation.TYPE_STR), }
class Pet(v.Validated): """Pet is used to test repeated fields.""" ATTRIBUTES = { 'name': v.Type(str), 'age': v.Optional(int), 'color': v.Optional(['black', 'brown', 'white']), }
class ErrorHandlers(validation.Validated): """Class representing error handler directives in application info. """ ATTRIBUTES = { ERROR_CODE: validation.Optional(_ERROR_CODE_REGEX), FILE: _FILES_REGEX, MIME_TYPE: validation.Optional(str), }
class QueueInfoExternal(validation.Validated): """Describes all of the queue entries for an application.""" ATTRIBUTES = { appinfo.APPLICATION: validation.Optional(appinfo.APPLICATION_RE_STRING), TOTAL_STORAGE_LIMIT: validation.Optional(_TOTAL_STORAGE_LIMIT_REGEX), QUEUE: validation.Optional(validation.Repeated(QueueEntry)), }
class UserParam(validation.Validated): """A user-supplied parameter to a mapreduce job.""" ATTRIBUTES = { "name": r"[a-zA-Z0-9_\.]+", "default": validation.Optional(r".*"), "value": validation.Optional(r".*"), }
class MapperInfo(validation.Validated): """Configuration parameters for the mapper part of the job.""" ATTRIBUTES = { "handler": r".+", "input_reader": r".+", "params": validation.Optional(validation.Repeated(UserParam)), "params_validator": validation.Optional(r".+"), }
class MapreduceInfo(validation.Validated): """Mapreduce description in mapreduce.yaml.""" ATTRIBUTES = { "name": r".+", "mapper": MapperInfo, "params": validation.Optional(validation.Repeated(UserParam)), "params_validator": validation.Optional(r".+"), }
class QueueEntry(validation.Validated): """A queue entry describes a single task queue.""" ATTRIBUTES = { NAME: _NAME_REGEX, RATE: _RATE_REGEX, BUCKET_SIZE: validation.Optional(validation.TYPE_INT), MAX_CONCURRENT_REQUESTS: validation.Optional(validation.TYPE_INT), RETRY_PARAMETERS: validation.Optional(RetryParameters), }
class PetStore(v.Validated): """Each test document maps to a PetStore.""" ATTRIBUTES = { 'name': v.Type(str), 'address': v.Optional(str), 'pets': v.Optional(v.Repeated(Pet)), 'mascot': v.Optional(v.Type(Pet, False)), 'phones': v.Optional(PhoneNumberDict), }
class RetryParameters(validation.Validated): """Specifies the retry parameters for a single task queue.""" ATTRIBUTES = { TASK_RETRY_LIMIT: validation.Optional(validation.TYPE_INT), TASK_AGE_LIMIT: validation.Optional(validation.TimeValue()), MIN_BACKOFF_SECONDS: validation.Optional(validation.TYPE_FLOAT), MAX_BACKOFF_SECONDS: validation.Optional(validation.TYPE_FLOAT), MAX_DOUBLINGS: validation.Optional(validation.TYPE_INT), }
class CronEntry(validation.Validated): """A cron entry describes a single cron job.""" ATTRIBUTES = { URL: _URL_REGEX, SCHEDULE: GrocValidator(), TIMEZONE: TimezoneValidator(), DESCRIPTION: validation.Optional(_DESCRIPTION_REGEX), TARGET: validation.Optional(_VERSION_REGEX), }
class ClientDeployInfoExternal(validation.Validated): """Describes the format of a client_deployinfo.yaml file.""" ATTRIBUTES = { RUNTIME: appinfo.RUNTIME_RE_STRING, START_TIME_USEC: validation.TYPE_LONG, END_TIME_USEC: validation.TYPE_LONG, REQUESTS: validation.Optional(validation.Repeated(Request)), SUCCESS: validation.TYPE_BOOL, SDK_VERSION: validation.Optional(validation.TYPE_STR) }
class TransformerEntry(validation.Validated): """Describes the transform for an entity (or model) kind.""" ATTRIBUTES = { 'name': validation.Optional(validation.TYPE_STR), 'kind': validation.Optional(validation.TYPE_STR), 'model': OPTIONAL_EVALUATED_CALLABLE, 'connector': validation.TYPE_STR, 'connector_options': validation.Optional(ConnectorOptions, {}), 'use_model_on_export': validation.Optional(validation.TYPE_BOOL), 'sort_key_from_entity': OPTIONAL_EVALUATED_CALLABLE, 'post_import_function': OPTIONAL_EVALUATED_CALLABLE, 'post_export_function': OPTIONAL_EVALUATED_CALLABLE, 'property_map': validation.Repeated(PropertyEntry, default=[]), } def CheckInitialized(self): """Check that all required (combinations) of fields are set. Also fills in computed properties. Raises: InvalidConfiguration: if the config is invalid. """ if not self.kind and not self.model: raise bulkloader_errors.InvalidConfiguration( 'Neither kind nor model specified for transformer.') if self.kind and self.model: raise bulkloader_errors.InvalidConfiguration( 'Both kind and model specified for transformer.') if self.model: self.kind = self.model.method.kind() else: if self.use_model_on_export: raise bulkloader_errors.InvalidConfiguration( 'No model class specified but use_model_on_export is true.' ) if not self.name: self.name = self.kind if not self.connector: raise bulkloader_errors.InvalidConfiguration( 'No connector specified.') property_names = set() for prop in self.property_map: if prop.property in property_names: raise bulkloader_errors.InvalidConfiguration( 'Duplicate property specified for property %s in transform %s' % (prop.property, self.name)) property_names.add(prop.property)
class IndexDefinitions(validation.Validated): """Top level for index definition file. Attributes: indexes: List of Index definitions. """ ATTRIBUTES = { appinfo.APPLICATION: validation.Optional(appinfo.APPLICATION_RE_STRING), 'indexes': validation.Optional(validation.Repeated(Index)), }
class RetryParameters(validation.Validated): """Retry parameters for a single cron job.""" ATTRIBUTES = { JOB_RETRY_LIMIT: validation.Optional( validation.Range(0, None, range_type=int)), JOB_AGE_LIMIT: validation.Optional(validation.TimeValue()), MIN_BACKOFF_SECONDS: validation.Optional( validation.Range(0.0, None, range_type=float)), MAX_BACKOFF_SECONDS: validation.Optional( validation.Range(0.0, None, range_type=float)), MAX_DOUBLINGS: validation.Optional( validation.Range(0, None, range_type=int)), }
class Index(validation.Validated): """Individual index definition. Order of the properties determines a given index's sort priority. Attributes: kind: Datastore kind that index belongs to. ancestors: Include ancestors in index. properties: Properties to be included. """ ATTRIBUTES = { 'kind': validation.Type(str, convert=False), 'ancestor': validation.Type(bool, convert=False, default=False), 'properties': validation.Optional(validation.Repeated(Property)), } def CheckInitialized(self): self._Normalize() super(Index, self).CheckInitialized() def _Normalize(self): if self.properties is None: return is_geo = any(x.mode == 'geospatial' for x in self.properties) for p in self.properties: if is_geo: if p.direction is not None: raise validation.ValidationError( 'direction not supported in a geospatial index') else: if p.IsAscending(): p.direction = 'asc'
class ExportEntry(validation.Validated): """Describes the optional export transform for a single property.""" ATTRIBUTES = { 'external_name': validation.Optional(validation.TYPE_STR), 'export_transform': OPTIONAL_EVALUATED_CALLABLE, }
class AdminConsole(validation.Validated): """Class representing admin console directives in application info. """ ATTRIBUTES = { PAGES: validation.Optional(validation.Repeated(AdminConsolePage)), } @classmethod def Merge(cls, adminconsole_one, adminconsole_two): """Return the result of merging two AdminConsole objects.""" if not adminconsole_one or not adminconsole_two: return adminconsole_one or adminconsole_two if adminconsole_one.pages: if adminconsole_two.pages: adminconsole_one.pages.extend(adminconsole_two.pages) else: adminconsole_one.pages = adminconsole_two.pages return adminconsole_one
class QueueEntry(validation.Validated): """A queue entry describes a single task queue.""" ATTRIBUTES = { NAME: _NAME_REGEX, RATE: _RATE_REGEX, BUCKET_SIZE: validation.Optional(validation.TYPE_INT), }
class FetcherPolicyInfo(validation.Validated): """Configuration parameters for the fetcher_policy part of the job.""" ATTRIBUTES = { "agent_name": r".+", "email_address": r".+", "web_address": r".+", "min_response_rate": "[0-9]+", "max_content_size": validation.Optional(validation.Repeated(MaxContentSizeInfo)), "crawl_end_time": "[0-9]+", "crawl_delay": "[0-9]+", "max_redirects": "[0-9]+", "accept_language": r".+", "valid_mime_types": r".+", "redirect_mode": validation.Options(FOLLOW_ALL, FOLLOW_NONE, default=FOLLOW_ALL), "request_timeout": r".+", }
class NagFile(validation.Validated): """A validated YAML class to represent the user's nag preferences. Attributes: timestamp: The timestamp of the last nag. opt_in: True if the user wants to check for updates on dev_appserver start. False if not. May be None if we have not asked the user yet. """ ATTRIBUTES = { 'timestamp': validation.TYPE_FLOAT, 'opt_in': validation.Optional(validation.TYPE_BOOL), } @staticmethod def Load(nag_file): """Load a single NagFile object where one and only one is expected. Args: nag_file: A file-like object or string containing the yaml data to parse. Returns: A NagFile instance. """ return yaml_object.BuildSingleObject(NagFile, nag_file)
class AppInfoExternal(validation.Validated): """Class representing users application info. This class is passed to a yaml_object builder to provide the validation for the application information file format parser. Attributes: application: Unique identifier for application. version: Application's major version number. runtime: Runtime used by application. api_version: Which version of APIs to use. handlers: List of URL handlers. default_expiration: Default time delta to use for cache expiration for all static files, unless they have their own specific 'expiration' set. See the URLMap.expiration field's documentation for more information. skip_files: An re object. Files that match this regular expression will not be uploaded by appcfg.py. For example: skip_files: | .svn.*| #.*# """ ATTRIBUTES = { APPLICATION: APPLICATION_RE_STRING, VERSION: VERSION_RE_STRING, RUNTIME: RUNTIME_RE_STRING, API_VERSION: API_VERSION_RE_STRING, HANDLERS: validation.Optional(validation.Repeated(URLMap)), DEFAULT_EXPIRATION: validation.Optional(_EXPIRATION_REGEX), SKIP_FILES: validation.RegexStr(default=DEFAULT_SKIP_FILES) } def CheckInitialized(self): """Ensures that at least one url mapping is provided. Raises: MissingURLMapping when no URLMap objects are present in object. TooManyURLMappings when there are too many URLMap entries. """ super(AppInfoExternal, self).CheckInitialized() if not self.handlers: raise appinfo_errors.MissingURLMapping( 'No URLMap entries found in application configuration') if len(self.handlers) > MAX_URL_MAPS: raise appinfo_errors.TooManyURLMappings( 'Found more than %d URLMap entries in application configuration' % MAX_URL_MAPS)
class BulkloaderEntry(validation.Validated): """Root of the bulkloader configuration.""" ATTRIBUTES = { 'python_preamble': validation.Optional(validation.Repeated(PythonPreambleEntry)), 'transformers': validation.Repeated(TransformerEntry), }