def _get_single_bs(
cls,
data: OutputData,
datastd: OutputDataConfig,
stm: io.RawIOBase,
enc: str = "utf-8",
) -> bytes:
"""输出带文件体的数据类型"""
res: bytes = None
try:
if not isinstance(data,
OutputData) or stm is None or not stm.readable():
cls._logger.error(
"Invalid OutputData object or stream for output single")
return res
for seg in data.get_output_segs():
# seg: OutputDataSeg = data.get_output_segs()
fields: dict = cls._parse_fields(seg, datastd)
if not isinstance(fields, dict) or len(fields) < 1:
cls._logger.error(
"Invalid fields after check output segment fields:\nplatform:{}\ndatatype:{}"
.format(data._platform, data._datatype.name))
return res
bs: bytes = cls._fields_to_bytes(fields, enc)
if bs is None or not any(bs):
return res
res = bs
return res
except Exception:
res = None
cls._logger.error(
"Output single data segment error:\nplatform:{}\ndatatype:{}\nerror:{}"
.format(data._platform, data._datatype.name,
traceback.format_exc()))
def _output_to_file(
cls,
data: OutputData,
bs: bytes,
datastd: OutputDataConfig,
targetdir: str,
stm: io.RawIOBase = None,
) -> bool:
"""输出到指定目录\n
bs:要输出的数据\n
datastd:数据对应的数据标准，用于构建文件名等\n
targetdir:目标目录\n
stm: 附带的数据流"""
res: bool = False
tmppath: str = None
outfi: str = None
try:
with cls.tmpdir_locker:
# 临时路径
tmppath: str = cls._get_datapath(cls._tmpdir, datastd)
if not isinstance(tmppath, str) or tmppath == "":
return res
with open(tmppath, mode="wb") as fs:
fs.write(bs)
if not stm is None and stm.readable():
# stm.readinto(fs)
readlen = 1024 * 1024 * 1024
while True:
buf = stm.read(readlen)
if buf is None:
break
readcount = len(buf)
fs.write(buf)
if readcount < readlen:
break
# 加了一个验证步骤..
# 后面如果要搞扩展输出方式，
# 应吧输出到临时，和输出到目标分成两个函数，
# 在两个函数调用的中间加一个验证步骤，各自实现
if not data.validate_file(tmppath):
# 不打日志了，错误数据直接不输出
cls._logger.debug("Corrupted data: {}".format(tmppath))
if os.path.isfile(tmppath):
os.remove(tmppath)
return res
with cls.outdir_locker:
outfi: str = cls._get_datapath(targetdir, datastd)
shutil.move(tmppath, outfi)
res = True
except Exception:
if not tmppath is None and tmppath != "" and os.path.isfile(
tmppath):
os.remove(tmppath)
if not outfi is None and outfi != "" and os.path.isfile(outfi):
os.remove(outfi)
cls._logger.error("Output data segments sub error: {}".format(
traceback.format_exc()))
return res
