def parse_content(cls, content, **kwargs):
crc, compute_crc, pb_str = unwrap_pb_frame(content)
if crc != compute_crc:
raise DatahubException('Parse pb response body fail, error: crc check error. crc: %s, compute crc: %s'
% (crc, compute_crc))
pb_get_record_response = GetRecordsResponse()
pb_get_record_response.ParseFromString(pb_str)
next_cursor = pb_get_record_response.next_cursor
record_count = pb_get_record_response.record_count
start_sequence = pb_get_record_response.start_sequence
records = []
sequence = start_sequence
for pb_record in pb_get_record_response.records:
record_schema = kwargs['record_schema']
if record_schema:
values = [bp_field_data.value for bp_field_data in pb_record.data.data]
record = TupleRecord(schema=record_schema, values=values)
else:
record = BlobRecord(blob_data=pb_record.data.data[0].value)
record._attributes = {
attribute.key: attribute.value for attribute in pb_record.attributes.attributes
}
record.system_time = pb_record.system_time
record.sequence = sequence
sequence += 1
records.append(record)
return cls(next_cursor, record_count, start_sequence, records)
def parse_content(cls, content, **kwargs):
crc, compute_crc, pb_str = unwrap_pb_frame(content)
if crc != compute_crc:
raise DatahubException('Parse pb response body fail, error: crc check error. crc: %s, compute crc: %s'
% (crc, compute_crc))
pb_put_record_response = PutRecordsResponse()
pb_put_record_response.ParseFromString(pb_str)
pb_failed_records = pb_put_record_response.failed_records
failed_records = [FailedRecord.from_pb_message(pb_failed_record) for pb_failed_record in pb_failed_records]
return cls(pb_put_record_response.failed_count, failed_records)
def get_connector_builder_by_type(connector_type):
builder = connector_config_dict.get(connector_type, None)
if not builder:
raise DatahubException('unsupported connector type')
return builder