class DosInfoExternal(validation.Validated):
"""Describes the format of a dos.yaml file."""
ATTRIBUTES = {
appinfo.APPLICATION:
validation.Optional(appinfo.APPLICATION_RE_STRING),
BLACKLIST: validation.Optional(validation.Repeated(BlacklistEntry)),
}
class CronInfoExternal(validation.Validated):
"""CronInfoExternal describes all cron entries for an application."""
ATTRIBUTES = {
appinfo.APPLICATION:
validation.Optional(appinfo.APPLICATION_RE_STRING),
CRON: validation.Optional(validation.Repeated(CronEntry))
}
class DispatchEntry(validation.Validated):
"""A Dispatch entry describes a mapping from a URL pattern to a module."""
ATTRIBUTES = {
URL: DispatchEntryURLValidator(),
MODULE: validation.Optional(appinfo.MODULE_ID_RE_STRING),
SERVICE: validation.Optional(appinfo.MODULE_ID_RE_STRING)
}
class OptionalStuff(validation.Validated):
ATTRIBUTES = {
'optional': validation.Optional(validation.TYPE_STR),
'implicit_type': validation.Optional(int, default=2001),
'implicit_regex': validation.Optional('[0-9]+'),
'implicit_options': validation.Optional(('a', 'b', 'c')),
}
class PropertyEntry(validation.Validated):
"""Describes the transform for a single property."""
ATTRIBUTES = {
'property': validation.Type(str),
'import_transform': OPTIONAL_EVALUATED_CALLABLE,
'import_template': validation.Optional(validation.TYPE_STR),
'default_value': validation.Optional(validation.TYPE_STR),
'export': validation.Optional(validation.Repeated(ExportEntry)),
}
ATTRIBUTES.update(ExportEntry.ATTRIBUTES)
def CheckInitialized(self):
"""Check that all required (combinations) of fields are set.
Also fills in computed properties.
Raises:
InvalidConfiguration: If the config is invalid.
"""
super(PropertyEntry, self).CheckInitialized()
if not (self.external_name or self.import_template or self.export):
raise bulkloader_errors.InvalidConfiguration(
'Neither external_name nor import_template nor export specified for '
'property %s.' % self.property)
class PagespeedEntry(validation.Validated):
"""Describes the format of a pagespeed configuration from a yaml file.
URL blacklist entries are patterns (with '?' and '*' as wildcards). Any URLs
that match a pattern on the blacklist will not be optimized by PageSpeed.
Rewriter names are strings (like 'CombineCss' or 'RemoveComments') describing
individual PageSpeed rewriters. A full list of valid rewriter names can be
found in the PageSpeed documentation.
The domains-to-rewrite list is a whitelist of domain name patterns with '*' as
a wildcard, optionally starting with 'http://' or 'https://'. If no protocol
is given, 'http://' is assumed. A resource will only be rewritten if it is on
the same domain as the HTML that references it, or if its domain is on the
domains-to-rewrite list.
"""
ATTRIBUTES = {
URL_BLACKLIST:
validation.Optional(
validation.Repeated(validation.Regex(_URL_BLACKLIST_REGEX))),
ENABLED_REWRITERS:
validation.Optional(
validation.Repeated(validation.Regex(_REWRITER_NAME_REGEX))),
DISABLED_REWRITERS:
validation.Optional(
validation.Repeated(validation.Regex(_REWRITER_NAME_REGEX))),
DOMAINS_TO_REWRITE:
validation.Optional(
validation.Repeated(validation.Regex(_DOMAINS_TO_REWRITE_REGEX))),
}
class Property(validation.Validated):
"""Representation for an individual property of an index.
Attributes:
name: Name of attribute to sort by.
direction: Direction of sort.
mode: How the property is indexed. Either 'geospatial'
or None (unspecified).
"""
ATTRIBUTES = {
'name':
validation.Type(str, convert=False),
'direction':
validation.Optional([('asc', ('ascending', )),
('desc', ('descending', ))]),
'mode':
validation.Optional(['geospatial'])
}
def IsAscending(self):
return self.direction != 'desc'
def CheckInitialized(self):
if self.direction is not None and self.mode is not None:
raise validation.ValidationError(
'direction and mode are mutually exclusive')
super(Property, self).CheckInitialized()
class ConnectorSubOptions(validation.Validated):
"""Connector options."""
ATTRIBUTES = {
'delimiter': validation.Optional(validation.TYPE_STR),
'dialect': validation.Optional(validation.TYPE_STR),
}
class Pet(v.Validated):
"""Pet is used to test repeated fields."""
ATTRIBUTES = {
'name': v.Type(str),
'age': v.Optional(int),
'color': v.Optional(['black', 'brown', 'white']),
}
class ErrorHandlers(validation.Validated):
"""Class representing error handler directives in application info.
"""
ATTRIBUTES = {
ERROR_CODE: validation.Optional(_ERROR_CODE_REGEX),
FILE: _FILES_REGEX,
MIME_TYPE: validation.Optional(str),
}
class QueueInfoExternal(validation.Validated):
"""Describes all of the queue entries for an application."""
ATTRIBUTES = {
appinfo.APPLICATION:
validation.Optional(appinfo.APPLICATION_RE_STRING),
TOTAL_STORAGE_LIMIT: validation.Optional(_TOTAL_STORAGE_LIMIT_REGEX),
QUEUE: validation.Optional(validation.Repeated(QueueEntry)),
}
class UserParam(validation.Validated):
"""A user-supplied parameter to a mapreduce job."""
ATTRIBUTES = {
"name": r"[a-zA-Z0-9_\.]+",
"default": validation.Optional(r".*"),
"value": validation.Optional(r".*"),
}
class MapperInfo(validation.Validated):
"""Configuration parameters for the mapper part of the job."""
ATTRIBUTES = {
"handler": r".+",
"input_reader": r".+",
"params": validation.Optional(validation.Repeated(UserParam)),
"params_validator": validation.Optional(r".+"),
}
class MapreduceInfo(validation.Validated):
"""Mapreduce description in mapreduce.yaml."""
ATTRIBUTES = {
"name": r".+",
"mapper": MapperInfo,
"params": validation.Optional(validation.Repeated(UserParam)),
"params_validator": validation.Optional(r".+"),
}
class QueueEntry(validation.Validated):
"""A queue entry describes a single task queue."""
ATTRIBUTES = {
NAME: _NAME_REGEX,
RATE: _RATE_REGEX,
BUCKET_SIZE: validation.Optional(validation.TYPE_INT),
MAX_CONCURRENT_REQUESTS: validation.Optional(validation.TYPE_INT),
RETRY_PARAMETERS: validation.Optional(RetryParameters),
}
class PetStore(v.Validated):
"""Each test document maps to a PetStore."""
ATTRIBUTES = {
'name': v.Type(str),
'address': v.Optional(str),
'pets': v.Optional(v.Repeated(Pet)),
'mascot': v.Optional(v.Type(Pet, False)),
'phones': v.Optional(PhoneNumberDict),
}
class RetryParameters(validation.Validated):
"""Specifies the retry parameters for a single task queue."""
ATTRIBUTES = {
TASK_RETRY_LIMIT: validation.Optional(validation.TYPE_INT),
TASK_AGE_LIMIT: validation.Optional(validation.TimeValue()),
MIN_BACKOFF_SECONDS: validation.Optional(validation.TYPE_FLOAT),
MAX_BACKOFF_SECONDS: validation.Optional(validation.TYPE_FLOAT),
MAX_DOUBLINGS: validation.Optional(validation.TYPE_INT),
}
class CronEntry(validation.Validated):
"""A cron entry describes a single cron job."""
ATTRIBUTES = {
URL: _URL_REGEX,
SCHEDULE: GrocValidator(),
TIMEZONE: TimezoneValidator(),
DESCRIPTION: validation.Optional(_DESCRIPTION_REGEX),
TARGET: validation.Optional(_VERSION_REGEX),
}
class ClientDeployInfoExternal(validation.Validated):
"""Describes the format of a client_deployinfo.yaml file."""
ATTRIBUTES = {
RUNTIME: appinfo.RUNTIME_RE_STRING,
START_TIME_USEC: validation.TYPE_LONG,
END_TIME_USEC: validation.TYPE_LONG,
REQUESTS: validation.Optional(validation.Repeated(Request)),
SUCCESS: validation.TYPE_BOOL,
SDK_VERSION: validation.Optional(validation.TYPE_STR)
}
class TransformerEntry(validation.Validated):
"""Describes the transform for an entity (or model) kind."""
ATTRIBUTES = {
'name': validation.Optional(validation.TYPE_STR),
'kind': validation.Optional(validation.TYPE_STR),
'model': OPTIONAL_EVALUATED_CALLABLE,
'connector': validation.TYPE_STR,
'connector_options': validation.Optional(ConnectorOptions, {}),
'use_model_on_export': validation.Optional(validation.TYPE_BOOL),
'sort_key_from_entity': OPTIONAL_EVALUATED_CALLABLE,
'post_import_function': OPTIONAL_EVALUATED_CALLABLE,
'post_export_function': OPTIONAL_EVALUATED_CALLABLE,
'property_map': validation.Repeated(PropertyEntry, default=[]),
}
def CheckInitialized(self):
"""Check that all required (combinations) of fields are set.
Also fills in computed properties.
Raises:
InvalidConfiguration: if the config is invalid.
"""
if not self.kind and not self.model:
raise bulkloader_errors.InvalidConfiguration(
'Neither kind nor model specified for transformer.')
if self.kind and self.model:
raise bulkloader_errors.InvalidConfiguration(
'Both kind and model specified for transformer.')
if self.model:
self.kind = self.model.method.kind()
else:
if self.use_model_on_export:
raise bulkloader_errors.InvalidConfiguration(
'No model class specified but use_model_on_export is true.'
)
if not self.name:
self.name = self.kind
if not self.connector:
raise bulkloader_errors.InvalidConfiguration(
'No connector specified.')
property_names = set()
for prop in self.property_map:
if prop.property in property_names:
raise bulkloader_errors.InvalidConfiguration(
'Duplicate property specified for property %s in transform %s'
% (prop.property, self.name))
property_names.add(prop.property)
class IndexDefinitions(validation.Validated):
"""Top level for index definition file.
Attributes:
indexes: List of Index definitions.
"""
ATTRIBUTES = {
appinfo.APPLICATION: validation.Optional(appinfo.APPLICATION_RE_STRING),
'indexes': validation.Optional(validation.Repeated(Index)),
}
class RetryParameters(validation.Validated):
"""Retry parameters for a single cron job."""
ATTRIBUTES = {
JOB_RETRY_LIMIT: validation.Optional(
validation.Range(0, None, range_type=int)),
JOB_AGE_LIMIT: validation.Optional(validation.TimeValue()),
MIN_BACKOFF_SECONDS: validation.Optional(
validation.Range(0.0, None, range_type=float)),
MAX_BACKOFF_SECONDS: validation.Optional(
validation.Range(0.0, None, range_type=float)),
MAX_DOUBLINGS: validation.Optional(
validation.Range(0, None, range_type=int)),
}
class Index(validation.Validated):
"""Individual index definition.
Order of the properties determines a given index's sort priority.
Attributes:
kind: Datastore kind that index belongs to.
ancestors: Include ancestors in index.
properties: Properties to be included.
"""
ATTRIBUTES = {
'kind': validation.Type(str, convert=False),
'ancestor': validation.Type(bool, convert=False, default=False),
'properties': validation.Optional(validation.Repeated(Property)),
}
def CheckInitialized(self):
self._Normalize()
super(Index, self).CheckInitialized()
def _Normalize(self):
if self.properties is None:
return
is_geo = any(x.mode == 'geospatial' for x in self.properties)
for p in self.properties:
if is_geo:
if p.direction is not None:
raise validation.ValidationError(
'direction not supported in a geospatial index')
else:
if p.IsAscending():
p.direction = 'asc'
class ExportEntry(validation.Validated):
"""Describes the optional export transform for a single property."""
ATTRIBUTES = {
'external_name': validation.Optional(validation.TYPE_STR),
'export_transform': OPTIONAL_EVALUATED_CALLABLE,
}
class AdminConsole(validation.Validated):
"""Class representing admin console directives in application info.
"""
ATTRIBUTES = {
PAGES: validation.Optional(validation.Repeated(AdminConsolePage)),
}
@classmethod
def Merge(cls, adminconsole_one, adminconsole_two):
"""Return the result of merging two AdminConsole objects."""
if not adminconsole_one or not adminconsole_two:
return adminconsole_one or adminconsole_two
if adminconsole_one.pages:
if adminconsole_two.pages:
adminconsole_one.pages.extend(adminconsole_two.pages)
else:
adminconsole_one.pages = adminconsole_two.pages
return adminconsole_one
class QueueEntry(validation.Validated):
"""A queue entry describes a single task queue."""
ATTRIBUTES = {
NAME: _NAME_REGEX,
RATE: _RATE_REGEX,
BUCKET_SIZE: validation.Optional(validation.TYPE_INT),
}
class FetcherPolicyInfo(validation.Validated):
"""Configuration parameters for the fetcher_policy part of the job."""
ATTRIBUTES = {
"agent_name":
r".+",
"email_address":
r".+",
"web_address":
r".+",
"min_response_rate":
"[0-9]+",
"max_content_size":
validation.Optional(validation.Repeated(MaxContentSizeInfo)),
"crawl_end_time":
"[0-9]+",
"crawl_delay":
"[0-9]+",
"max_redirects":
"[0-9]+",
"accept_language":
r".+",
"valid_mime_types":
r".+",
"redirect_mode":
validation.Options(FOLLOW_ALL, FOLLOW_NONE, default=FOLLOW_ALL),
"request_timeout":
r".+",
}
class NagFile(validation.Validated):
"""A validated YAML class to represent the user's nag preferences.
Attributes:
timestamp: The timestamp of the last nag.
opt_in: True if the user wants to check for updates on dev_appserver
start. False if not. May be None if we have not asked the user yet.
"""
ATTRIBUTES = {
'timestamp': validation.TYPE_FLOAT,
'opt_in': validation.Optional(validation.TYPE_BOOL),
}
@staticmethod
def Load(nag_file):
"""Load a single NagFile object where one and only one is expected.
Args:
nag_file: A file-like object or string containing the yaml data to parse.
Returns:
A NagFile instance.
"""
return yaml_object.BuildSingleObject(NagFile, nag_file)
class AppInfoExternal(validation.Validated):
"""Class representing users application info.
This class is passed to a yaml_object builder to provide the validation
for the application information file format parser.
Attributes:
application: Unique identifier for application.
version: Application's major version number.
runtime: Runtime used by application.
api_version: Which version of APIs to use.
handlers: List of URL handlers.
default_expiration: Default time delta to use for cache expiration for
all static files, unless they have their own specific 'expiration' set.
See the URLMap.expiration field's documentation for more information.
skip_files: An re object. Files that match this regular expression will
not be uploaded by appcfg.py. For example:
skip_files: |
.svn.*|
#.*#
"""
ATTRIBUTES = {
APPLICATION: APPLICATION_RE_STRING,
VERSION: VERSION_RE_STRING,
RUNTIME: RUNTIME_RE_STRING,
API_VERSION: API_VERSION_RE_STRING,
HANDLERS: validation.Optional(validation.Repeated(URLMap)),
DEFAULT_EXPIRATION: validation.Optional(_EXPIRATION_REGEX),
SKIP_FILES: validation.RegexStr(default=DEFAULT_SKIP_FILES)
}
def CheckInitialized(self):
"""Ensures that at least one url mapping is provided.
Raises:
MissingURLMapping when no URLMap objects are present in object.
TooManyURLMappings when there are too many URLMap entries.
"""
super(AppInfoExternal, self).CheckInitialized()
if not self.handlers:
raise appinfo_errors.MissingURLMapping(
'No URLMap entries found in application configuration')
if len(self.handlers) > MAX_URL_MAPS:
raise appinfo_errors.TooManyURLMappings(
'Found more than %d URLMap entries in application configuration'
% MAX_URL_MAPS)
class BulkloaderEntry(validation.Validated):
"""Root of the bulkloader configuration."""
ATTRIBUTES = {
'python_preamble':
validation.Optional(validation.Repeated(PythonPreambleEntry)),
'transformers':
validation.Repeated(TransformerEntry),
}
