def _ParseGcsDestinationConfig(self, gcs_destination_config_file): """Parses a gcs_destination_config into the GcsDestinationConfig message.""" data = console_io.ReadFromFileOrStdin( gcs_destination_config_file, binary=False) try: gcs_destination_head_config_data = yaml.load(data) except Exception as e: raise ds_exceptions.ParseError('Cannot parse YAML:[{0}]'.format(e)) gcs_destination_config_data_object = gcs_destination_head_config_data.get( 'gcs_destination_config') gcs_destination_config_data = gcs_destination_config_data_object if gcs_destination_config_data_object else gcs_destination_head_config_data path = gcs_destination_config_data.get('path', '') file_rotation_mb = gcs_destination_config_data.get('file_rotation_mb', {}) file_rotation_interval = gcs_destination_config_data.get( 'file_rotation_interval', {}) gcs_dest_config_msg = self._messages.GcsDestinationConfig( path=path, fileRotationMb=file_rotation_mb, fileRotationInterval=file_rotation_interval) if 'avro_file_format' in gcs_destination_config_data: gcs_dest_config_msg.avroFileFormat = self._messages.AvroFileFormat() elif 'json_file_format' in gcs_destination_config_data: json_file_format_data = gcs_destination_config_data.get( 'json_file_format') gcs_dest_config_msg.jsonFileFormat = self._messages.JsonFileFormat( compression=json_file_format_data.get('compression'), schemaFileFormat=json_file_format_data.get('schema_file_format')) else: raise ds_exceptions.ParseError( 'Cannot parse YAML: missing file format.') return gcs_dest_config_msg
def _ParseMysqlSourceConfig(self, mysql_source_config_file, release_track): """Parses a mysql_sorce_config into the MysqlSourceConfig message.""" data = console_io.ReadFromFileOrStdin( mysql_source_config_file, binary=False) try: mysql_sorce_config_head_data = yaml.load(data) except Exception as e: raise ds_exceptions.ParseError('Cannot parse YAML:[{0}]'.format(e)) mysql_sorce_config_data_object = mysql_sorce_config_head_data.get( 'mysql_source_config') mysql_rdbms_data = mysql_sorce_config_data_object if mysql_sorce_config_data_object else mysql_sorce_config_head_data include_objects_raw = mysql_rdbms_data.get( util.GetRDBMSV1alpha1ToV1FieldName('allowlist', release_track), {}) include_objects_data = util.ParseMysqlSchemasListToMysqlRdbmsMessage( self._messages, include_objects_raw, release_track) exclude_objects_raw = mysql_rdbms_data.get( util.GetRDBMSV1alpha1ToV1FieldName('rejectlist', release_track), {}) exclude_objects_data = util.ParseMysqlSchemasListToMysqlRdbmsMessage( self._messages, exclude_objects_raw, release_track) mysql_sourec_config_msg = self._messages.MysqlSourceConfig( includeObjects=include_objects_data, excludeObjects=exclude_objects_data) return mysql_sourec_config_msg
def _ParseConnectionProfileObjectFile(self, connection_profile_object_file): """Parses a connection-profile-file into the ConnectionProfile message.""" data = console_io.ReadFromFileOrStdin(connection_profile_object_file, binary=False) try: connection_profile_data = yaml.load(data) except Exception as e: raise ds_exceptions.ParseError('Cannot parse YAML:[{0}]'.format(e)) display_name = connection_profile_data.get('display_name') labels = connection_profile_data.get('labels') connection_profile_msg = self._messages.ConnectionProfile( displayName=display_name, labels=labels) oracle_profile = self._ParseOracleProfile( connection_profile_data.get('oracle_profile', {})) mysql_profile = self._ParseMySqlProfile( connection_profile_data.get('mysql_profile', {})) gcs_profile = self._ParseGCSProfile( connection_profile_data.get('gcs_profile', {})) if oracle_profile: connection_profile_msg.oracleProfile = oracle_profile elif mysql_profile: connection_profile_msg.mysqlProfile = mysql_profile elif gcs_profile: connection_profile_msg.gcsProfile = gcs_profile if 'static_service_ip_connectivity' in connection_profile_data: connection_profile_msg.staticServiceIpConnectivity = connection_profile_data.get( 'static_service_ip_connectivity') elif 'forward_ssh_connectivity' in connection_profile_data: connection_profile_msg.forwardSshConnectivity = connection_profile_data.get( 'forward_ssh_connectivity') elif 'private_connectivity' in connection_profile_data: connection_profile_msg.privateConnectivity = connection_profile_data.get( 'private_connectivity') else: raise ds_exceptions.ParseError( 'Cannot parse YAML: missing connectivity method.') return connection_profile_msg
def ParseOracleSchema(messages, oracle_schema_object, release_track): """Parses a raw oracle schema json/yaml into the OracleSchema message.""" oracle_tables_msg_list = [] for table in oracle_schema_object.get('oracle_tables', []): oracle_tables_msg_list.append( ParseOracleTable(messages, table, release_track)) schema_key = _GetRDBMSFieldName('schema', release_track) schema_name = oracle_schema_object.get(schema_key) if not schema_name: raise ds_exceptions.ParseError('Cannot parse YAML: missing key "%s".' % schema_key) return messages.OracleSchema(schema=schema_name, oracleTables=oracle_tables_msg_list)
def ParseOracleTable(messages, oracle_table_object, release_track): """Parses a raw oracle table json/yaml into the OracleTable message.""" oracle_columns_msg_list = [] for column in oracle_table_object.get('oracle_columns', []): oracle_columns_msg_list.append( ParseOracleColumn(messages, column, release_track)) table_key = _GetRDBMSFieldName('table', release_track) table_name = oracle_table_object.get(table_key) if not table_name: raise ds_exceptions.ParseError('Cannot parse YAML: missing key "%s".' % table_key) return messages.OracleTable(table=table_name, oracleColumns=oracle_columns_msg_list)
def ParseMysqlDatabase(messages, mysql_database_object, release_track): """Parses a raw mysql database json/yaml into the MysqlDatabase message.""" mysql_tables_msg_list = [] for table in mysql_database_object.get('mysql_tables', []): mysql_tables_msg_list.append( ParseMysqlTable(messages, table, release_track)) database_key = _GetRDBMSFieldName('database', release_track) database_name = mysql_database_object.get(database_key) if not database_name: raise ds_exceptions.ParseError('Cannot parse YAML: missing key "%s".' % database_key) return messages.MysqlDatabase(database=database_name, mysqlTables=mysql_tables_msg_list)
def ParseMysqlTable(messages, mysql_table_object, release_track): """Parses a raw mysql table json/yaml into the MysqlTable message.""" mysql_column_msg_list = [] for column in mysql_table_object.get('mysql_columns', []): mysql_column_msg_list.append( ParseMysqlColumn(messages, column, release_track)) table_key = _GetRDBMSFieldName('table', release_track) table_name = mysql_table_object.get(table_key) if not table_name: raise ds_exceptions.ParseError('Cannot parse YAML: missing key "%s".' % table_key) return messages.MysqlTable(table=table_name, mysqlColumns=mysql_column_msg_list)
def ParseMysqlRdbmsFile(messages, mysql_rdbms_file, release_track=base.ReleaseTrack.BETA): """Parses a mysql_rdbms_file into the MysqlRdbms message.""" data = console_io.ReadFromFileOrStdin(mysql_rdbms_file, binary=False) try: mysql_rdbms_head_data = yaml.load(data) except Exception as e: raise ds_exceptions.ParseError('Cannot parse YAML:[{0}]'.format(e)) mysql_rdbms_data = mysql_rdbms_head_data.get('mysql_rdbms', mysql_rdbms_head_data) return ParseMysqlSchemasListToMysqlRdbmsMessage(messages, mysql_rdbms_data, release_track)