def _get_cookie_kvp(self, cookie: str) -> (str, str):
"""parse cookie str to key-value pair"""
k, v = helper_str.get_kvp(cookie, '=')
if k is None or k == "" or v is None or v == "":
return (k, v)
v = v.rstrip().rstrip(';')
return (k, v)
def _parse_req_headers(self, headers: str):
"""parse str headers like:
aaa:aaa
bbb:bbb
...
to
{
"aaa":"aaa",
"bbb":"bbb"
}
"""
res = {}
try:
if headers is None or headers == "":
return res
allheaders = headers.split('\n')
for oneheader in allheaders:
k, v = helper_str.get_kvp(oneheader, ':')
if k is None or k == "" or v is None or v == "":
continue
# do not check duplicate keys here
res[k.strip()] = v.strip()
except Exception as ex:
raise ex
return res
def _parse_data(self, data: str):
"""parse str data like:
aaa=aaa&bbb=bbb
...
to
{
"aaa":"aaa",
"bbb":"bbb"
}
"""
res = {}
try:
if data is None or data == "":
return res
alldatas = data.split('&')
for onedata in alldatas:
k, v = helper_str.get_kvp(onedata, '=')
if k is None or k == "" or v is None or v == "":
continue
# do not check duplicate keys here
res[k] = v
except Exception as ex:
raise ex
return res
def __get_segs_from_stream(self, stm: io.IOBase,
targettype: type = None) -> iter:
"""从流中读取字段，返回 DataSeg 对象迭代器\n
targettype: 目标类型，可以为任何 DataSeg 类型的子类，若为空，则返回 DataSeg 类型的对象"""
try:
if stm is None or stm.closed:
raise Exception("Data stream is unreadable")
segidx = 1
segline = 0
seg: DataSeg = DataSeg()
prevline: str = ''
for line in stm:
if line is None or line == "":
break
segline += 1
if (line == '\n' or line == '\r\n') and \
(prevline.endswith('\n') or prevline.endswith('\r\n')):
seg.segindex = segidx
segidx += 1
seg.segline = segline
yield seg
seg = DataSeg()
else:
k, v = helper_str.get_kvp(line, ':')
if not k is None and not k == "":
v = helper_str.base64_decode_format(v)
# 这里若遇到键相同，容错处理只保留第一次读到的键值对
if not seg._fields.__contains__(k):
seg._fields[k] = v.strip()
prevline = line
if not seg is None and len(seg._fields) > 0:
seg.segindex = segidx
seg.segline = segline
yield seg
except Exception as ex:
raise ex
def _get_segments(self, data: InputData) -> iter:
"""读取数据流，返回数据段枚举"""
try:
if data.stream is None or not data.stream.readable():
self._logger.error(
"Data stream is None when trying to convert to standard Task: %s"
% data._source)
return
segidx = 1
seg: DataSeg = DataSeg()
prevline: str = None
for line in data.stream:
if line is None or line == "":
break
if (line == '\n' or line == '\r\n') and (prevline == '\n' or
prevline == '\r\n'):
seg.segindex = segidx
segidx += 1
yield seg
seg = DataSeg()
k, v = helper_str.get_kvp(line, ':')
if not k is None and not k == "":
# 这里若遇到键相同，容错处理只保留第一次读到的键值对
v = helper_str.base64_decode_format(v)
if not seg._fields.__contains__(k):
seg._fields[k] = v.strip()
prevline = line
if not seg is None and len(seg._fields) > 0:
yield seg
except Exception:
self._logger.error("Read data segments error:\ndata:%s\nex:%s" %
(data._source, traceback.format_exc()))
if not data is None:
data.on_complete(False)