def to_iterator(self, raw_data_fd):
raw_data = decompress_raw_data(raw_data_fd, add_bracket=True).decode(
'utf-8', 'replace')
if not self._is_raw_binary_iterator:
ret = json.loads(raw_data)
elif not self._use_ijson:
ret = iter(json.loads(raw_data))
else:
ret = split_rows_from_stream(StringIO(raw_data))
return ret
def to_iterator(self, raw_data_fd, download_time):
parse_start_time = get_time_millis()
raw_data = decompress_raw_data(raw_data_fd, add_bracket=True).decode(
'utf-8', 'replace')
if not self._is_raw_binary_iterator:
ret = json.loads(raw_data)
elif not self._use_ijson:
ret = iter(json.loads(raw_data))
else:
ret = split_rows_from_stream(StringIO(raw_data))
parse_end_time = get_time_millis()
timing_metrics = {
ResultIterWithTimings.DOWNLOAD: download_time,
ResultIterWithTimings.PARSE: parse_end_time - parse_start_time
}
return ResultIterWithTimings(ret, timing_metrics)