def _ParseGcsDestinationConfig(self, gcs_destination_config_file):
"""Parses a gcs_destination_config into the GcsDestinationConfig message."""
data = console_io.ReadFromFileOrStdin(
gcs_destination_config_file, binary=False)
try:
gcs_destination_head_config_data = yaml.load(data)
except Exception as e:
raise ds_exceptions.ParseError('Cannot parse YAML:[{0}]'.format(e))
gcs_destination_config_data_object = gcs_destination_head_config_data.get(
'gcs_destination_config')
gcs_destination_config_data = gcs_destination_config_data_object if gcs_destination_config_data_object else gcs_destination_head_config_data
path = gcs_destination_config_data.get('path', '')
file_rotation_mb = gcs_destination_config_data.get('file_rotation_mb', {})
file_rotation_interval = gcs_destination_config_data.get(
'file_rotation_interval', {})
gcs_dest_config_msg = self._messages.GcsDestinationConfig(
path=path, fileRotationMb=file_rotation_mb,
fileRotationInterval=file_rotation_interval)
if 'avro_file_format' in gcs_destination_config_data:
gcs_dest_config_msg.avroFileFormat = self._messages.AvroFileFormat()
elif 'json_file_format' in gcs_destination_config_data:
json_file_format_data = gcs_destination_config_data.get(
'json_file_format')
gcs_dest_config_msg.jsonFileFormat = self._messages.JsonFileFormat(
compression=json_file_format_data.get('compression'),
schemaFileFormat=json_file_format_data.get('schema_file_format'))
else:
raise ds_exceptions.ParseError(
'Cannot parse YAML: missing file format.')
return gcs_dest_config_msg
def _ParseMysqlSourceConfig(self, mysql_source_config_file, release_track):
"""Parses a mysql_sorce_config into the MysqlSourceConfig message."""
data = console_io.ReadFromFileOrStdin(
mysql_source_config_file, binary=False)
try:
mysql_sorce_config_head_data = yaml.load(data)
except Exception as e:
raise ds_exceptions.ParseError('Cannot parse YAML:[{0}]'.format(e))
mysql_sorce_config_data_object = mysql_sorce_config_head_data.get(
'mysql_source_config')
mysql_rdbms_data = mysql_sorce_config_data_object if mysql_sorce_config_data_object else mysql_sorce_config_head_data
include_objects_raw = mysql_rdbms_data.get(
util.GetRDBMSV1alpha1ToV1FieldName('allowlist', release_track), {})
include_objects_data = util.ParseMysqlSchemasListToMysqlRdbmsMessage(
self._messages, include_objects_raw, release_track)
exclude_objects_raw = mysql_rdbms_data.get(
util.GetRDBMSV1alpha1ToV1FieldName('rejectlist', release_track), {})
exclude_objects_data = util.ParseMysqlSchemasListToMysqlRdbmsMessage(
self._messages, exclude_objects_raw, release_track)
mysql_sourec_config_msg = self._messages.MysqlSourceConfig(
includeObjects=include_objects_data,
excludeObjects=exclude_objects_data)
return mysql_sourec_config_msg
def _ParseConnectionProfileObjectFile(self,
connection_profile_object_file):
"""Parses a connection-profile-file into the ConnectionProfile message."""
data = console_io.ReadFromFileOrStdin(connection_profile_object_file,
binary=False)
try:
connection_profile_data = yaml.load(data)
except Exception as e:
raise ds_exceptions.ParseError('Cannot parse YAML:[{0}]'.format(e))
display_name = connection_profile_data.get('display_name')
labels = connection_profile_data.get('labels')
connection_profile_msg = self._messages.ConnectionProfile(
displayName=display_name, labels=labels)
oracle_profile = self._ParseOracleProfile(
connection_profile_data.get('oracle_profile', {}))
mysql_profile = self._ParseMySqlProfile(
connection_profile_data.get('mysql_profile', {}))
gcs_profile = self._ParseGCSProfile(
connection_profile_data.get('gcs_profile', {}))
if oracle_profile:
connection_profile_msg.oracleProfile = oracle_profile
elif mysql_profile:
connection_profile_msg.mysqlProfile = mysql_profile
elif gcs_profile:
connection_profile_msg.gcsProfile = gcs_profile
if 'static_service_ip_connectivity' in connection_profile_data:
connection_profile_msg.staticServiceIpConnectivity = connection_profile_data.get(
'static_service_ip_connectivity')
elif 'forward_ssh_connectivity' in connection_profile_data:
connection_profile_msg.forwardSshConnectivity = connection_profile_data.get(
'forward_ssh_connectivity')
elif 'private_connectivity' in connection_profile_data:
connection_profile_msg.privateConnectivity = connection_profile_data.get(
'private_connectivity')
else:
raise ds_exceptions.ParseError(
'Cannot parse YAML: missing connectivity method.')
return connection_profile_msg
def ParseOracleSchema(messages, oracle_schema_object, release_track):
"""Parses a raw oracle schema json/yaml into the OracleSchema message."""
oracle_tables_msg_list = []
for table in oracle_schema_object.get('oracle_tables', []):
oracle_tables_msg_list.append(
ParseOracleTable(messages, table, release_track))
schema_key = _GetRDBMSFieldName('schema', release_track)
schema_name = oracle_schema_object.get(schema_key)
if not schema_name:
raise ds_exceptions.ParseError('Cannot parse YAML: missing key "%s".' %
schema_key)
return messages.OracleSchema(schema=schema_name,
oracleTables=oracle_tables_msg_list)
def ParseOracleTable(messages, oracle_table_object, release_track):
"""Parses a raw oracle table json/yaml into the OracleTable message."""
oracle_columns_msg_list = []
for column in oracle_table_object.get('oracle_columns', []):
oracle_columns_msg_list.append(
ParseOracleColumn(messages, column, release_track))
table_key = _GetRDBMSFieldName('table', release_track)
table_name = oracle_table_object.get(table_key)
if not table_name:
raise ds_exceptions.ParseError('Cannot parse YAML: missing key "%s".' %
table_key)
return messages.OracleTable(table=table_name,
oracleColumns=oracle_columns_msg_list)
def ParseMysqlDatabase(messages, mysql_database_object, release_track):
"""Parses a raw mysql database json/yaml into the MysqlDatabase message."""
mysql_tables_msg_list = []
for table in mysql_database_object.get('mysql_tables', []):
mysql_tables_msg_list.append(
ParseMysqlTable(messages, table, release_track))
database_key = _GetRDBMSFieldName('database', release_track)
database_name = mysql_database_object.get(database_key)
if not database_name:
raise ds_exceptions.ParseError('Cannot parse YAML: missing key "%s".' %
database_key)
return messages.MysqlDatabase(database=database_name,
mysqlTables=mysql_tables_msg_list)
def ParseMysqlTable(messages, mysql_table_object, release_track):
"""Parses a raw mysql table json/yaml into the MysqlTable message."""
mysql_column_msg_list = []
for column in mysql_table_object.get('mysql_columns', []):
mysql_column_msg_list.append(
ParseMysqlColumn(messages, column, release_track))
table_key = _GetRDBMSFieldName('table', release_track)
table_name = mysql_table_object.get(table_key)
if not table_name:
raise ds_exceptions.ParseError('Cannot parse YAML: missing key "%s".' %
table_key)
return messages.MysqlTable(table=table_name,
mysqlColumns=mysql_column_msg_list)
def ParseMysqlRdbmsFile(messages,
mysql_rdbms_file,
release_track=base.ReleaseTrack.BETA):
"""Parses a mysql_rdbms_file into the MysqlRdbms message."""
data = console_io.ReadFromFileOrStdin(mysql_rdbms_file, binary=False)
try:
mysql_rdbms_head_data = yaml.load(data)
except Exception as e:
raise ds_exceptions.ParseError('Cannot parse YAML:[{0}]'.format(e))
mysql_rdbms_data = mysql_rdbms_head_data.get('mysql_rdbms',
mysql_rdbms_head_data)
return ParseMysqlSchemasListToMysqlRdbmsMessage(messages, mysql_rdbms_data,
release_track)