def LoadSingleDos(dos_info, open_fn=None):
"""Load a dos.yaml file or string and return a DosInfoExternal object.
Args:
dos_info: The contents of a dos.yaml file as a string, or an open file
object.
open_fn: Function for opening files. Unused.
Returns:
A DosInfoExternal instance which represents the contents of the parsed yaml
file.
Raises:
MalformedDosConfiguration: The yaml file contains multiple blacklist
sections.
yaml_errors.EventError: An error occured while parsing the yaml file.
"""
builder = yaml_object.ObjectBuilder(DosInfoExternal)
handler = yaml_builder.BuilderHandler(builder)
listener = yaml_listener.EventListener(handler)
listener.Parse(dos_info)
parsed_yaml = handler.GetResults()
if not parsed_yaml:
return DosInfoExternal()
if len(parsed_yaml) > 1:
raise MalformedDosConfiguration('Multiple blacklist: sections '
'in configuration.')
return parsed_yaml[0]
def LoadSingleClientDeployInfo(client_deploy_info):
"""Returns a ClientDeployInfoExternal from a deploy_info.yaml file or string.
Args:
client_deploy_info: The contents of a client_deploy_info.yaml file or
string, or an open file object.
Returns:
A ClientDeployInfoExternal instance which represents the contents of the
parsed yaml.
Raises:
EmptyYaml: when there are no documents in yaml.
MultipleClientDeployInfo: when there are multiple documents in yaml.
yaml_errors.EventError: when an error occurs while parsing the yaml.
"""
builder = yaml_object.ObjectBuilder(ClientDeployInfoExternal)
handler = yaml_builder.BuilderHandler(builder)
listener = yaml_listener.EventListener(handler)
listener.Parse(client_deploy_info)
parsed_yaml = handler.GetResults()
if not parsed_yaml:
raise EmptyYaml()
if len(parsed_yaml) > 1:
raise MultipleClientDeployInfo()
return parsed_yaml[0]
def LoadSingleAppInfo(app_info):
"""Load a single AppInfo object where one and only one is expected.
Args:
app_info: A file-like object or string. If it is a string, parse it as
a configuration file. If it is a file-like object, read in data and
parse.
Returns:
An instance of AppInfoExternal as loaded from a YAML file.
Raises:
ValueError: if a specified service is not valid.
EmptyConfigurationFile: when there are no documents in YAML file.
MultipleConfigurationFile: when there is more than one document in YAML
file.
"""
builder = yaml_object.ObjectBuilder(AppInfoExternal)
handler = yaml_builder.BuilderHandler(builder)
listener = yaml_listener.EventListener(handler)
listener.Parse(app_info)
app_infos = handler.GetResults()
if len(app_infos) < 1:
raise appinfo_errors.EmptyConfigurationFile()
if len(app_infos) > 1:
raise appinfo_errors.MultipleConfigurationFile()
appyaml = app_infos[0]
ValidateHandlers(appyaml.handlers)
if appyaml.builtins:
BuiltinHandler.Validate(appyaml.builtins)
return appyaml
def LoadSingleDos(dos_info):
"""Load a dos.yaml file or string and return a DosInfoExternal object.
Args:
dos_info: The contents of a dos.yaml file, as a string.
Returns:
A DosInfoExternal instance which represents the contents of the parsed yaml
file.
Raises:
MalformedDosConfiguration if the yaml file contains multiple blacklist
sections.
yaml_errors.EventError if any errors occured while parsing the yaml file.
"""
builder = yaml_object.ObjectBuilder(DosInfoExternal)
handler = yaml_builder.BuilderHandler(builder)
listener = yaml_listener.EventListener(handler)
listener.Parse(dos_info)
dos_info = handler.GetResults()
if len(dos_info) < 1:
return DosInfoExternal()
if len(dos_info) > 1:
raise MalformedDosConfiguration('Multiple blacklist: sections '
'in configuration.')
return dos_info[0]
def LoadBackendInfo(backend_info, open_fn=None):
"""Parses a BackendInfoExternal object from a string.
Args:
backend_info: a backends stanza (list of backends) as a string
open_fn: Function for opening files. Unused.
Returns:
A BackendInfoExternal object.
"""
builder = yaml_object.ObjectBuilder(BackendInfoExternal)
handler = yaml_builder.BuilderHandler(builder)
listener = yaml_listener.EventListener(handler)
listener.Parse(backend_info)
backend_info = handler.GetResults()
if len(backend_info) < 1:
return BackendInfoExternal(backends=[])
if len(backend_info) > 1:
raise BadConfig("Only one 'backends' clause is allowed.")
info = backend_info[0]
if not info.backends: # The 'backends' clause was empty.
return BackendInfoExternal(backends=[])
for backend in info.backends:
backend.Init()
return info
def parse_mapreduce_yaml(contents):
"""Parses mapreduce.yaml file contents.
Args:
contents: mapreduce.yaml file contents.
Returns:
MapReduceYaml object with all the data from original file.
Raises:
errors.BadYamlError: when contents is not a valid mapreduce.yaml file.
"""
try:
builder = yaml_object.ObjectBuilder(MapReduceYaml)
handler = yaml_builder.BuilderHandler(builder)
listener = yaml_listener.EventListener(handler)
listener.Parse(contents)
mr_info = handler.GetResults()
except (ValueError, yaml_errors.EventError) as e:
raise errors.BadYamlError(e)
if len(mr_info) < 1:
raise errors.BadYamlError("No configs found in mapreduce.yaml")
if len(mr_info) > 1:
raise errors.MultipleDocumentsInMrYaml("Found %d YAML documents" %
len(mr_info))
jobs = mr_info[0]
job_names = set(j.name for j in jobs.mapreduce)
if len(jobs.mapreduce) != len(job_names):
raise errors.BadYamlError(
"Overlapping mapreduce names; names must be unique")
return jobs
def load_config(stream, config_globals):
"""Load a configuration file and generate importer and exporter classes.
Args:
stream: Stream containing config YAML.
config_globals: Dict to use to reference globals for code in the config.
Returns:
BulkloaderEntry
Raises:
InvalidConfiguration: If the config is invalid.
"""
builder = yaml_object.ObjectBuilder(BulkloaderEntry)
handler = yaml_builder.BuilderHandler(builder)
listener = yaml_listener.EventListener(handler)
global _global_temp_globals
_global_temp_globals = config_globals
try:
listener.Parse(stream)
finally:
_global_temp_globals = None
bulkloader_infos = handler.GetResults()
if len(bulkloader_infos) < 1:
raise bulkloader_errors.InvalidConfiguration(
'No configuration specified.')
if len(bulkloader_infos) > 1:
raise bulkloader_errors.InvalidConfiguration(
'Multiple sections in configuration.')
bulkloader_info = bulkloader_infos[0]
if not bulkloader_info.transformers:
raise bulkloader_errors.InvalidConfiguration(
'No transformers specified.')
return bulkloader_info
def LoadSingleDispatch(dispatch_info, open_fn=None):
"""Load a dispatch.yaml file or string and return a DispatchInfoExternal.
Args:
dispatch_info: The contents of a dispatch.yaml file as a string, or an open
file object.
open_fn: Function for opening files. Unused here, needed to provide
a polymorphic API used by appcfg.py yaml parsing.
Returns:
A DispatchInfoExternal instance which represents the contents of the parsed
yaml file.
Raises:
MalformedDispatchConfigurationError: The yaml file contains multiple
dispatch sections.
yaml_errors.EventError: An error occured while parsing the yaml file.
"""
builder = yaml_object.ObjectBuilder(DispatchInfoExternal)
handler = yaml_builder.BuilderHandler(builder)
listener = yaml_listener.EventListener(handler)
listener.Parse(dispatch_info)
parsed_yaml = handler.GetResults()
if not parsed_yaml:
return DispatchInfoExternal()
if len(parsed_yaml) > 1:
raise MalformedDispatchConfigurationError(
'Multiple dispatch: sections '
'in configuration.')
return parsed_yaml[0]
def LoadPagespeedEntry(pagespeed_entry, open_fn=None):
"""Load a yaml file or string and return a PagespeedEntry.
Args:
pagespeed_entry: The contents of a pagespeed entry from a yaml file
as a string, or an open file object.
open_fn: Function for opening files. Unused.
Returns:
A PagespeedEntry instance which represents the contents of the parsed yaml.
Raises:
yaml_errors.EventError: An error occured while parsing the yaml.
MalformedPagespeedConfiguration: The configuration is parseable but invalid.
"""
builder = yaml_object.ObjectBuilder(PagespeedEntry)
handler = yaml_builder.BuilderHandler(builder)
listener = yaml_listener.EventListener(handler)
listener.Parse(pagespeed_entry)
parsed_yaml = handler.GetResults()
if not parsed_yaml:
return PagespeedEntry()
if len(parsed_yaml) > 1:
raise MalformedPagespeedConfiguration(
'Multiple configuration sections in the yaml')
return parsed_yaml[0]
def LoadSinglePagespeed(pagespeed_info):
"""Load a pagespeed.yaml file or string and return a PagespeedInfoExternal.
Args:
pagespeed_info: The contents of a pagespeed.yaml file as a string, or an
open file object.
Returns:
A PagespeedInfoExternal instance which represents the contents of the parsed
yaml file.
Raises:
yaml_errors.EventError: An error occured while parsing the yaml file.
MalformedPagespeedConfiguration: The configuration is parseable but invalid.
"""
builder = yaml_object.ObjectBuilder(PagespeedInfoExternal)
handler = yaml_builder.BuilderHandler(builder)
listener = yaml_listener.EventListener(handler)
listener.Parse(pagespeed_info)
parsed_yaml = handler.GetResults()
if not parsed_yaml:
return PagespeedInfoExternal()
if len(parsed_yaml) > 1:
raise MalformedPagespeedConfiguration(
'Multiple configuration sections in pagespeed.yaml')
return parsed_yaml[0]
def setUp(self):
"""Set up test harness.
Create BuilderHandler and connect to FakeBuilder. Create loader
to pass in to methods which require it.
"""
self.builder = yaml_test_util.FakeBuilder()
self.handler = yaml_builder.BuilderHandler(self.builder)
self.loader = yaml.loader.Loader('')
def LoadSingleCron(cron_info, open_fn=None):
"""Load a cron.yaml file or string and return a CronInfoExternal object."""
builder = yaml_object.ObjectBuilder(CronInfoExternal)
handler = yaml_builder.BuilderHandler(builder)
listener = yaml_listener.EventListener(handler)
listener.Parse(cron_info)
cron_info = handler.GetResults()
if len(cron_info) < 1:
raise MalformedCronfigurationFile('Empty cron configuration.')
if len(cron_info) > 1:
raise MalformedCronfigurationFile('Multiple cron sections '
'in configuration.')
return cron_info[0]
def LoadSingleDispatch(dispatch_info, open_fn=None):
"""Load a dispatch.yaml file or string and return a DispatchInfoExternal.
Args:
dispatch_info: The contents of a dispatch.yaml file as a string, or an open
file object.
open_fn: Function for opening files. Unused here, needed to provide
a polymorphic API used by appcfg.py yaml parsing.
Returns:
A DispatchInfoExternal instance which represents the contents of the parsed
yaml file.
Raises:
MalformedDispatchConfigurationError: The yaml file contains multiple
dispatch sections or is missing a required value.
yaml_errors.EventError: An error occured while parsing the yaml file.
"""
builder = yaml_object.ObjectBuilder(DispatchInfoExternal)
handler = yaml_builder.BuilderHandler(builder)
listener = yaml_listener.EventListener(handler)
listener.Parse(dispatch_info)
parsed_yaml = handler.GetResults()
if not parsed_yaml:
return DispatchInfoExternal()
if len(parsed_yaml) > 1:
raise MalformedDispatchConfigurationError(
'Multiple dispatch: sections '
'in configuration.')
# The validation framework doesn't allow validating multiple fields at once,
# so we have to check here.
dispatch_info_external = parsed_yaml[0]
for dispatch in getattr(dispatch_info_external, DISPATCH) or []:
if dispatch.module and dispatch.service:
raise MalformedDispatchConfigurationError(
'Both module: and service: in dispatch entry. Please use only one.'
)
if not (dispatch.module or dispatch.service):
raise MalformedDispatchConfigurationError(
"Missing required value 'service'.")
# The important part is that just 'module' is sent to the API for now;
# eventually, we'll switch this to just check service.
dispatch.module = dispatch.module or dispatch.service
dispatch.service = None
return dispatch_info_external
def parse_fetcher_policy_yaml(contents):
"""Parses fetcher_policy.yaml file contents.
Args:
contents: fetcher_policy.yaml file contents.
Returns:
FetcherPolicyYaml object with all the data from original file.
Raises:
errors.BadYamlError: when contents is not a valid fetcher_policy.yaml file.
"""
try:
builder = yaml_object.ObjectBuilder(FetcherPolicyYaml)
handler = yaml_builder.BuilderHandler(builder)
listener = yaml_listener.EventListener(handler)
listener.Parse(contents)
fp_info = handler.GetResults()
except (ValueError, yaml_errors.EventError), e:
raise errors.BadYamlError(e)
def parse_mapreduce_yaml(contents):
"""Parses mapreduce.yaml file contents.
Args:
contents: mapreduce.yaml file contents.
Returns:
MapReduceYaml object with all the data from original file.
Raises:
errors.BadYamlError: when contents is not a valid mapreduce.yaml file.
"""
try:
builder = yaml_object.ObjectBuilder(MapReduceYaml)
handler = yaml_builder.BuilderHandler(builder)
listener = yaml_listener.EventListener(handler)
listener.Parse(contents)
mr_info = handler.GetResults()
except (ValueError, yaml_errors.EventError), e:
raise errors.BadYamlError(e)
def LoadBackendEntry(backend_entry):
"""Parses a BackendEntry object from a string.
Args:
backend_entry: a backend entry, as a string
Returns:
A BackendEntry object.
"""
builder = yaml_object.ObjectBuilder(BackendEntry)
handler = yaml_builder.BuilderHandler(builder)
listener = yaml_listener.EventListener(handler)
listener.Parse(backend_entry)
entries = handler.GetResults()
if len(entries) < 1:
raise BadConfig('Empty backend configuration.')
if len(entries) > 1:
raise BadConfig('Multiple backend entries were found in configuration.')
return entries[0].Init()
def LoadSingleQueue(queue_info):
"""Load a queue.yaml file or string and return a QueueInfoExternal object.
Args:
queue_info: the contents of a queue.yaml file, as a string.
Returns:
A QueueInfoExternal object.
"""
builder = yaml_object.ObjectBuilder(QueueInfoExternal)
handler = yaml_builder.BuilderHandler(builder)
listener = yaml_listener.EventListener(handler)
listener.Parse(queue_info)
queue_info = handler.GetResults()
if len(queue_info) < 1:
raise MalformedQueueConfiguration('Empty queue configuration.')
if len(queue_info) > 1:
raise MalformedQueueConfiguration('Multiple queue: sections '
'in configuration.')
return queue_info[0]
def LoadSingleQueue(queue_info, open_fn=None):
"""Loads a `queue.yaml` file/string and returns a `QueueInfoExternal` object.
Args:
queue_info: The contents of a `queue.yaml` file, as a string.
open_fn: Function for opening files. Unused.
Returns:
A `QueueInfoExternal` object.
"""
builder = yaml_object.ObjectBuilder(QueueInfoExternal)
handler = yaml_builder.BuilderHandler(builder)
listener = yaml_listener.EventListener(handler)
listener.Parse(queue_info)
queue_info = handler.GetResults()
if len(queue_info) < 1:
raise MalformedQueueConfiguration('Empty queue configuration.')
if len(queue_info) > 1:
raise MalformedQueueConfiguration('Multiple queue: sections '
'in configuration.')
return queue_info[0]
def BuildObjects(default_class, stream, loader=yaml.loader.SafeLoader):
"""Build objects from stream.
Handles the basic case of loading all the objects from a stream.
Args:
default_class: Class that is instantiated upon the detection of a new
document. An instance of this class will act as the document itself.
stream: String document or open file object to process as per the
yaml.parse method. Any object that implements a 'read()' method which
returns a string document will work with the YAML parser.
loader_class: Used for dependency injection.
Returns:
List of default_class instances parsed from the stream.
"""
builder = ObjectBuilder(default_class)
handler = yaml_builder.BuilderHandler(builder)
listener = yaml_listener.EventListener(handler)
listener.Parse(stream, loader)
return handler.GetResults()
def LoadSingleDispatch(dispatch_info, open_fn=None):
"""Load a dispatch.yaml file or string and return a DispatchInfoExternal.
Args:
dispatch_info: The contents of a dispatch.yaml file as a string, or an open
file object.
open_fn: Function for opening files. Unused here, needed to provide
a polymorphic API used by appcfg.py yaml parsing.
Returns:
A DispatchInfoExternal instance which represents the contents of the parsed
yaml file.
Raises:
MalformedDispatchConfigurationError: The yaml file contains multiple
dispatch sections or is missing a required value.
yaml_errors.EventError: An error occured while parsing the yaml file.
"""
builder = yaml_object.ObjectBuilder(DispatchInfoExternal)
handler = yaml_builder.BuilderHandler(builder)
listener = yaml_listener.EventListener(handler)
listener.Parse(dispatch_info)
parsed_yaml = handler.GetResults()
if not parsed_yaml:
return DispatchInfoExternal()
if len(parsed_yaml) > 1:
raise MalformedDispatchConfigurationError(
'Multiple dispatch: sections '
'in configuration.')
# The validation framework doesn't allow validating multiple fields at once,
# so we have to check here.
dispatch_info_external = parsed_yaml[0]
dispatch_info_external.CheckInitialized()
return dispatch_info_external
def LoadAppInclude(app_include):
"""Load a single AppInclude object where one and only one is expected.
Args:
app_include: A file-like object or string. If it is a string, parse it as
a configuration file. If it is a file-like object, read in data and
parse.
Returns:
An instance of AppInclude as loaded from a YAML file.
Raises:
EmptyConfigurationFile: when there are no documents in YAML file.
MultipleConfigurationFile: when there is more than one document in YAML
file.
"""
builder = yaml_object.ObjectBuilder(AppInclude)
handler = yaml_builder.BuilderHandler(builder)
listener = yaml_listener.EventListener(handler)
listener.Parse(app_include)
includes = handler.GetResults()
if len(includes) < 1:
raise appinfo_errors.EmptyConfigurationFile()
if len(includes) > 1:
raise appinfo_errors.MultipleConfigurationFile()
includeyaml = includes[0]
if includeyaml.handlers:
for handler in includeyaml.handlers:
handler.FixSecureDefaults()
handler.WarnReservedURLs()
if includeyaml.builtins:
BuiltinHandler.Validate(includeyaml.builtins)
return includeyaml
def setUp(self):
"""Set up parsing frame work."""
self.builder = yaml_object.ObjectBuilder(PetStore)
self.handler = yaml_builder.BuilderHandler(self.builder)
self.listener = yaml_listener.EventListener(self.handler)
