def _event_load(self, file_handler):
"""
Load binary string to event string.
Args:
file_handler (FileHandler): A file handler.
Returns:
bytes, MindSpore event in bytes.
"""
# read the header
header_str = file_handler.read(HEADER_SIZE)
if not header_str:
logger.info("End of file, file_path=%s.", file_handler.file_path)
return None
header_crc_str = file_handler.read(CRC_STR_SIZE)
if not header_crc_str:
header_crc_str = ''
if len(header_str) != HEADER_SIZE or len(
header_crc_str) != CRC_STR_SIZE:
logger.warning(
"Check header size and crc, record truncated at offset %s, "
"file_path=%s.", file_handler.offset, file_handler.file_path)
return None
if not crc32.CheckValueAgainstData(header_crc_str, header_str,
HEADER_SIZE):
raise exceptions.CRCFailedError()
# read the event body if integrity of header is verified
header = struct.unpack('Q', header_str)
event_len = int(header[0])
event_str = file_handler.read(event_len)
if not event_str:
event_str = ''
event_crc_str = file_handler.read(CRC_STR_SIZE)
if not event_crc_str:
event_crc_str = ''
if len(event_str) != event_len or len(event_crc_str) != CRC_STR_SIZE:
logger.warning(
"Check event crc, record truncated at offset %d, file_path: %s.",
file_handler.offset, file_handler.file_path)
return None
if not crc32.CheckValueAgainstData(event_crc_str, event_str,
event_len):
raise exceptions.CRCFailedError()
return event_str
def event_load(file_handler):
"""
Load binary string to event string.
Args:
file_handler (FileHandler): A file handler.
Returns:
bytes, MindSpore event in bytes.
"""
# read the header
header_str = file_handler.read(HEADER_SIZE)
if not header_str:
logger.info("Load summary file finished, file_path=%s.",
file_handler.file_path)
return None
header_crc_str = file_handler.read(CRC_STR_SIZE)
if not header_crc_str:
header_crc_str = ''
if len(header_str) != HEADER_SIZE or len(
header_crc_str) != CRC_STR_SIZE:
raise exceptions.CRCLengthFailedError(
"CRC header length or event header length is incorrect.")
if not crc32.CheckValueAgainstData(header_crc_str, header_str,
HEADER_SIZE):
raise exceptions.CRCFailedError(
"The header of event crc is failed.")
# read the event body if integrity of header is verified
header = struct.unpack('Q', header_str)
event_len = int(header[0])
event_str = file_handler.read(event_len)
if not event_str:
event_str = ''
event_crc_str = file_handler.read(CRC_STR_SIZE)
if not event_crc_str:
event_crc_str = ''
if len(event_str) != event_len or len(event_crc_str) != CRC_STR_SIZE:
raise exceptions.CRCLengthFailedError(
"The event sting length or crc length is incorrect.")
if not crc32.CheckValueAgainstData(event_crc_str, event_str,
event_len):
raise exceptions.CRCFailedError(
"The event string crc is incorrect.")
return event_str
def _check_crc(source_str, crc_str):
"""
Check the integrity of source string.
Args:
source_str (bytes): Source string in bytes.
crc_str (bytes): CRC string of source string in bytes.
Raises:
LineageVerificationException: Raise when verification failed.
"""
if not crc32.CheckValueAgainstData(crc_str, source_str, len(source_str)):
log.error("The CRC verification failed.")
raise LineageVerificationException("The CRC verification failed.")