def ReadYaml(message_type, stream, schema_path=None):
"""Read yaml from a stream as a message.
Args:
message_type: Type of message to interpret the yaml as.
stream: Stream from which yaml should be read.
schema_path: Path to schema used to validate yaml, relative to schemas dir.
Returns:
Message that was read.
Raises:
ParseError: if yaml could not be parsed as the given message type.
"""
parsed_yaml = yaml.load(stream)
if schema_path:
# If a schema is provided, validate against it.
try:
_ValidateYaml(parsed_yaml, schema_path)
except jsonschema_exceptions.ValidationError as e:
raise exceptions.ParseError('Validation Error: [{0}]'.format(
e.message))
try:
message = encoding.PyValueToMessage(message_type, parsed_yaml)
except Exception as e:
raise exceptions.ParseError('Cannot parse YAML: [{0}]'.format(e))
return message
def ReadYaml(file_path, message_type):
parsed_yaml = yaml.load_path(file_path)
try:
message = encoding.PyValueToMessage(message_type, parsed_yaml)
except Exception as e:
raise exceptions.ParseError(
'Cannot parse YAML from file {0}: [{1}]'.format(file_path, e))
return message
def ParseKerberosConfigFile(dataproc, kerberos_config_file):
"""Parse a kerberos-config-file into the KerberosConfig message."""
data = console_io.ReadFromFileOrStdin(kerberos_config_file, binary=False)
try:
kerberos_config_data = yaml.load(data)
except Exception as e:
raise exceptions.ParseError('Cannot parse YAML:[{0}]'.format(e))
ssl_config = kerberos_config_data.get('ssl', {})
keystore_uri = ssl_config.get('keystore_uri')
truststore_uri = ssl_config.get('truststore_uri')
keystore_password_uri = ssl_config.get('keystore_password_uri')
key_password_uri = ssl_config.get('key_password_uri')
truststore_password_uri = ssl_config.get('truststore_password_uri')
cross_realm_trust_config = kerberos_config_data.get(
'cross_realm_trust', {})
cross_realm_trust_realm = cross_realm_trust_config.get('realm')
cross_realm_trust_kdc = cross_realm_trust_config.get('kdc')
cross_realm_trust_admin_server = cross_realm_trust_config.get(
'admin_server')
cross_realm_trust_shared_password_uri = cross_realm_trust_config.get(
'shared_password_uri')
kerberos_config_msg = dataproc.messages.KerberosConfig(
# Unless user explicitly disable kerberos in kerberos config,
# consider the existence of the kerberos config is enabling
# kerberos, explicitly or implicitly.
enableKerberos=kerberos_config_data.get('enable_kerberos', True),
rootPrincipalPasswordUri=kerberos_config_data.get(
'root_principal_password_uri'),
kmsKeyUri=kerberos_config_data.get('kms_key_uri'),
kdcDbKeyUri=kerberos_config_data.get('kdc_db_key_uri'),
tgtLifetimeHours=kerberos_config_data.get('tgt_lifetime_hours'),
realm=kerberos_config_data.get('realm'),
keystoreUri=keystore_uri,
keystorePasswordUri=keystore_password_uri,
keyPasswordUri=key_password_uri,
truststoreUri=truststore_uri,
truststorePasswordUri=truststore_password_uri,
crossRealmTrustRealm=cross_realm_trust_realm,
crossRealmTrustKdc=cross_realm_trust_kdc,
crossRealmTrustAdminServer=cross_realm_trust_admin_server,
crossRealmTrustSharedPasswordUri=cross_realm_trust_shared_password_uri)
return kerberos_config_msg
def BuildJobProperties(arg_properties, properties_file):
"""Build job properties.
Merges properties from the arg_properties and properties_file. If a property
is set in both, the value in arg_properties is used.
Args:
arg_properties: A dictionary of property=value pairs.
properties_file: Path or URI to a text file with property=value lines
and/or comments. File can be a local file or a gs:// file.
Returns:
A dictionary merged properties
Example:
BuildJobProperties({'foo':'bar'}, 'gs://test-bucket/job_properties.conf')
"""
job_properties = {}
if properties_file:
try:
if properties_file.startswith('gs://'):
data = storage_helpers.ReadObject(properties_file)
else:
data = console_io.ReadFromFileOrStdin(properties_file,
binary=False)
except Exception as e:
raise exceptions.Error(
'Cannot read properties-file: {0}'.format(e))
try:
yaml.allow_duplicate_keys = True
key_values = yaml.load(data.strip().replace('=', ': '),
round_trip=True)
if key_values:
for key, value in key_values.items():
job_properties[key] = value
except Exception:
raise exceptions.ParseError(
'Cannot parse properties-file: {0}, '.format(properties_file) +
'make sure file format is a text file with list of key=value')
if arg_properties:
job_properties.update(arg_properties)
return job_properties
def Import(message_type, stream, schema_path=None):
"""Reads YAML from a stream as a message.
Args:
message_type: Type of message to load YAML into.
stream: Input stream or buffer containing the YAML.
schema_path: JSON schema file path. None for no YAML validation.
Raises:
ParseError: if yaml could not be parsed as the given message type.
Returns:
message_type object.
"""
parsed_yaml = yaml.load(stream)
if schema_path:
# If a schema is provided, validate against it.
yaml_validator.Validator(schema_path).Validate(parsed_yaml)
try:
message = api_encoding.PyValueToMessage(message_type, parsed_yaml)
except Exception as e:
raise exceptions.ParseError('Cannot parse YAML: [{0}]'.format(e))
return message
