def extract_contents(contents, extraction_types):
result = contents
for t in extraction_types:
if t == 'application/x-gzip':
result = util.decomp_gzip(result, quiet=False, decode=False)
elif t == 'application/base64':
result = base64.b64decode(result)
elif t == UNKNOWN_ENC:
pass
return result
def extract_contents(contents, extraction_types):
result = str(contents)
for t in extraction_types:
if t == 'application/x-gzip':
result = util.decomp_gzip(result, quiet=False)
elif t == 'application/base64':
result = base64.b64decode(result)
elif t == UNKNOWN_ENC:
pass
return result
def _decode(data, encoding=None):
if not data:
return b''
if not encoding or encoding.lower() in ['raw']:
return util.encode_text(data)
elif encoding.lower() in ['base64', 'b64']:
return base64.b64decode(data)
elif encoding.lower() in ['gzip', 'gz']:
return util.decomp_gzip(data, quiet=False, decode=None)
else:
raise IOError("Unknown random_seed encoding: %s" % (encoding))
def _decode(data, encoding=None):
if not data:
return ""
if not encoding or encoding.lower() in ["raw"]:
return data
elif encoding.lower() in ["base64", "b64"]:
return base64.b64decode(data)
elif encoding.lower() in ["gzip", "gz"]:
return util.decomp_gzip(data, quiet=False)
else:
raise IOError("Unknown random_seed encoding: %s" % (encoding))
def _decode(data, encoding=None):
if not data:
return b""
if not encoding or encoding.lower() in ["raw"]:
return util.encode_text(data)
elif encoding.lower() in ["base64", "b64"]:
return base64.b64decode(data)
elif encoding.lower() in ["gzip", "gz"]:
return util.decomp_gzip(data, quiet=False, decode=None)
else:
raise IOError("Unknown random_seed encoding: %s" % (encoding))
def load_userdata(ud_file_path):
"""Attempt to return a string of user-data from ud_file_path
Attempt to decode or decompress if needed.
If unable to decode the content, raw bytes will be returned.
@returns: String of uncompressed userdata if possible, otherwise bytes.
"""
bdata = util.load_file(ud_file_path, decode=False)
try:
return bdata.decode('utf-8')
except UnicodeDecodeError:
return util.decomp_gzip(bdata, quiet=False, decode=True)
def decode(key, enc_type, data):
LOG.debug("Getting encoded data for key=%s, enc=%s", key, enc_type)
raw_data = None
if enc_type in ["gzip+base64", "gz+b64"]:
LOG.debug("Decoding %s type of %s", enc_type, key)
raw_data = util.decomp_gzip(util.b64d(data))
elif enc_type in ["base64", "b64"]:
LOG.debug("Decoding %s type of %s", enc_type, key)
raw_data = util.b64d(data)
else:
LOG.debug("Plain-text data %s", key)
raw_data = data
return util.decode_binary(raw_data)
def convert_string(raw_data, headers=None):
if not raw_data:
raw_data = ''
if not headers:
headers = {}
data = util.decode_binary(util.decomp_gzip(raw_data))
if "mime-version:" in data[0:4096].lower():
msg = util.message_from_string(data)
for (key, val) in headers.items():
_replace_header(msg, key, val)
else:
mtype = headers.get(CONTENT_TYPE, NOT_MULTIPART_TYPE)
maintype, subtype = mtype.split("/", 1)
msg = MIMEBase(maintype, subtype, *headers)
msg.set_payload(data)
return msg
def convert_string(raw_data, headers=None):
if not raw_data:
raw_data = ''
if not headers:
headers = {}
data = util.decode_binary(util.decomp_gzip(raw_data))
if "mime-version:" in data[0:4096].lower():
msg = util.message_from_string(data)
for (key, val) in headers.items():
_replace_header(msg, key, val)
else:
mtype = headers.get(CONTENT_TYPE, NOT_MULTIPART_TYPE)
maintype, subtype = mtype.split("/", 1)
msg = MIMEBase(maintype, subtype, *headers)
msg.set_payload(data)
return msg
def decode(key, enc_type, data):
"""
decode returns the decoded string value of data
key is a string used to identify the data being decoded in log messages
"""
LOG.debug("Getting encoded data for key=%s, enc=%s", key, enc_type)
raw_data = None
if enc_type in ["gzip+base64", "gz+b64"]:
LOG.debug("Decoding %s format %s", enc_type, key)
raw_data = util.decomp_gzip(util.b64d(data))
elif enc_type in ["base64", "b64"]:
LOG.debug("Decoding %s format %s", enc_type, key)
raw_data = util.b64d(data)
else:
LOG.debug("Plain-text data %s", key)
raw_data = data
return util.decode_binary(raw_data)
def convert_string(raw_data, content_type=NOT_MULTIPART_TYPE):
if not raw_data:
raw_data = ''
def create_binmsg(data, content_type):
maintype, subtype = content_type.split("/", 1)
msg = MIMEBase(maintype, subtype)
msg.set_payload(data)
return msg
try:
data = util.decode_binary(util.decomp_gzip(raw_data))
if "mime-version:" in data[0:4096].lower():
msg = util.message_from_string(data)
else:
msg = create_binmsg(data, content_type)
except UnicodeDecodeError:
msg = create_binmsg(raw_data, content_type)
return msg
def convert_string(raw_data, content_type=NOT_MULTIPART_TYPE):
if not raw_data:
raw_data = ""
def create_binmsg(data, content_type):
maintype, subtype = content_type.split("/", 1)
msg = MIMEBase(maintype, subtype)
msg.set_payload(data)
return msg
try:
data = util.decode_binary(util.decomp_gzip(raw_data))
if "mime-version:" in data[0:4096].lower():
msg = util.message_from_string(data)
else:
msg = create_binmsg(data, content_type)
except UnicodeDecodeError:
msg = create_binmsg(raw_data, content_type)
return msg
def convert_string(raw_data, content_type=NOT_MULTIPART_TYPE):
"""convert a string (more likely bytes) or a message into
a mime message."""
if not raw_data:
raw_data = b''
def create_binmsg(data, content_type):
maintype, subtype = content_type.split("/", 1)
msg = MIMEBase(maintype, subtype)
msg.set_payload(data)
return msg
if isinstance(raw_data, str):
bdata = raw_data.encode('utf-8')
else:
bdata = raw_data
bdata = util.decomp_gzip(bdata, decode=False)
if b"mime-version:" in bdata[0:4096].lower():
msg = util.message_from_string(bdata.decode('utf-8'))
else:
msg = create_binmsg(bdata, content_type)
return msg
def _process_msg(self, base_msg, append_msg):
def find_ctype(payload):
return handlers.type_from_starts_with(payload)
for part in base_msg.walk():
if is_skippable(part):
continue
ctype = None
ctype_orig = part.get_content_type()
payload = util.fully_decoded_payload(part)
was_compressed = False
# When the message states it is of a gzipped content type ensure
# that we attempt to decode said payload so that the decompressed
# data can be examined (instead of the compressed data).
if ctype_orig in DECOMP_TYPES:
try:
payload = util.decomp_gzip(payload, quiet=False)
# At this point we don't know what the content-type is
# since we just decompressed it.
ctype_orig = None
was_compressed = True
except util.DecompressionError as e:
LOG.warning(
"Failed decompressing payload from %s of"
" length %s due to: %s", ctype_orig, len(payload), e)
continue
# Attempt to figure out the payloads content-type
if not ctype_orig:
ctype_orig = UNDEF_TYPE
if ctype_orig in TYPE_NEEDED:
ctype = find_ctype(payload)
if ctype is None:
ctype = ctype_orig
# In the case where the data was compressed, we want to make sure
# that we create a new message that contains the found content
# type with the uncompressed content since later traversals of the
# messages will expect a part not compressed.
if was_compressed:
maintype, subtype = ctype.split("/", 1)
n_part = MIMENonMultipart(maintype, subtype)
n_part.set_payload(payload)
# Copy various headers from the old part to the new one,
# but don't include all the headers since some are not useful
# after decoding and decompression.
if part.get_filename():
_set_filename(n_part, part.get_filename())
for h in ('Launch-Index', ):
if h in part:
_replace_header(n_part, h, str(part[h]))
part = n_part
if ctype != ctype_orig:
_replace_header(part, CONTENT_TYPE, ctype)
if ctype in INCLUDE_TYPES:
self._do_include(payload, append_msg)
continue
if ctype in ARCHIVE_TYPES:
self._explode_archive(payload, append_msg)
continue
# TODO(harlowja): Should this be happening, shouldn't
# the part header be modified and not the base?
_replace_header(base_msg, CONTENT_TYPE, ctype)
self._attach_part(append_msg, part)
def _process_msg(self, base_msg, append_msg):
def find_ctype(payload):
return handlers.type_from_starts_with(payload)
for part in base_msg.walk():
if is_skippable(part):
continue
ctype = None
ctype_orig = part.get_content_type()
payload = util.fully_decoded_payload(part)
was_compressed = False
# When the message states it is of a gzipped content type ensure
# that we attempt to decode said payload so that the decompressed
# data can be examined (instead of the compressed data).
if ctype_orig in DECOMP_TYPES:
try:
payload = util.decomp_gzip(payload, quiet=False)
# At this point we don't know what the content-type is
# since we just decompressed it.
ctype_orig = None
was_compressed = True
except util.DecompressionError as e:
LOG.warn(
"Failed decompressing payload from %s of length" " %s due to: %s", ctype_orig, len(payload), e
)
continue
# Attempt to figure out the payloads content-type
if not ctype_orig:
ctype_orig = UNDEF_TYPE
if ctype_orig in TYPE_NEEDED:
ctype = find_ctype(payload)
if ctype is None:
ctype = ctype_orig
# In the case where the data was compressed, we want to make sure
# that we create a new message that contains the found content
# type with the uncompressed content since later traversals of the
# messages will expect a part not compressed.
if was_compressed:
maintype, subtype = ctype.split("/", 1)
n_part = MIMENonMultipart(maintype, subtype)
n_part.set_payload(payload)
# Copy various headers from the old part to the new one,
# but don't include all the headers since some are not useful
# after decoding and decompression.
if part.get_filename():
_set_filename(n_part, part.get_filename())
for h in ("Launch-Index",):
if h in part:
_replace_header(n_part, h, str(part[h]))
part = n_part
if ctype != ctype_orig:
_replace_header(part, CONTENT_TYPE, ctype)
if ctype in INCLUDE_TYPES:
self._do_include(payload, append_msg)
continue
if ctype in ARCHIVE_TYPES:
self._explode_archive(payload, append_msg)
continue
# TODO(harlowja): Should this be happening, shouldn't
# the part header be modified and not the base?
_replace_header(base_msg, CONTENT_TYPE, ctype)
self._attach_part(append_msg, part)