def make_result(self):
#encoding = chardet.detect(s_to_b(self.stdoutdata))["encoding"] or "utf-8"
self.tend = time.time()
result = b_to_s(self.stdoutdata)
result_json = None
try:
# Let's see if the output can be decoded as JSON
result_json = json.loads(result)
except:
pass
output = b_to_s(self.stderrdata)
output_json = None
try:
# Let's see if the output can be decoded as JSON
output_json = json.loads(output)
except:
pass
results = {
"commandline": self.commandline,
"start": int(self.tstart * 1000.0),
"end": int(self.tend * 1000.0),
"elapsed": int((self.tend - self.tstart) * 1000.0),
"exitcode": self.retcode, # Nonzero exit code indicates error
"stdout": result,
"stderr": output,
"stdout_json": result_json, # May be null
"stderr_json": output_json # May be null
}
return results
def convert_base64_encoding(payload):
""" look for base64 mime type and convert the data that follows.
return: payload with the base64 encoded data substituted
"""
if isinstance(payload, list):
return [convert_base64_encoding(item) for item in payload]
result = payload
# determine if we have embedded base64
match = RE_CONTENT_TYPE.search(payload)
if match:
LOG.INFO("Found bas64 encoded content")
# find the start of the data which is demarked by an empty line
match_base64 = RE_START_BASE64.search(payload[match.end():])
if match_base64:
base64_data = payload[match.end()+match_base64.end():]
else:
base64_data = payload[match.end():] # just start where we found the mime information
# ensure we are really dealing with base64 data
if RE_BASE64.search(base64_data):
LOG.debug(base64_data)
decoded_data = b_to_s(decode_mail_body(base64_data))
# insert where we found it
result = "\n".join([payload[:match.end()], decoded_data])
return result
def test_success(self, circuits_app, base64content, expected_result):
""" Test calling with sample values for the parameters """
function_params = {
"base64content": b_to_s(base64content),
"incident_id": 1001
}
result = call_email_parse_function(circuits_app, function_params)
verify_subset(expected_result, result["content"])
def test_success(self, circuits_app, string_to_convert_to_attachment, attachment_name, incident_id, expected_results):
""" Test calling with sample values for the parameters """
function_params = {
"string_to_convert_to_attachment": b_to_s(string_to_convert_to_attachment),
"attachment_name": attachment_name,
"incident_id": incident_id
}
results = call_utilities_string_to_attachment_function(circuits_app, function_params)
verify_subset(expected_results, results)
def _attachment_to_base64_function(self, event, *args, **kwargs):
"""Function: Produce base64 content of a file attachment."""
try:
log = logging.getLogger(__name__)
# Get the function parameters:
incident_id = kwargs.get("incident_id") # number
task_id = kwargs.get("task_id") # number
attachment_id = kwargs.get("attachment_id") # number
artifact_id = kwargs.get("artifact_id") # number
log.info("incident_id: %s", incident_id)
log.info("task_id: %s", task_id)
log.info("attachment_id: %s", attachment_id)
log.info("artifact_id: %s", artifact_id)
if incident_id is None:
raise FunctionError("Error: incident_id must be specified.")
elif attachment_id is None and artifact_id is None:
raise FunctionError(
"Error: attachment_id or artifact_id must be specified.")
else:
yield StatusMessage("> Function inputs OK")
yield StatusMessage("> Reading attachment...")
client = self.rest_client()
data = get_file_attachment(client,
incident_id,
artifact_id=artifact_id,
task_id=task_id,
attachment_id=attachment_id)
metadata = get_file_attachment_metadata(
client,
incident_id,
artifact_id=artifact_id,
task_id=task_id,
attachment_id=attachment_id)
results = {
"filename": metadata["name"],
"content_type": metadata["content_type"],
"size": metadata["size"],
"created": metadata["created"],
"content": b_to_s(base64.b64encode(data)),
}
yield StatusMessage("> Complete...")
# Produce a FunctionResult with the return value
log.debug(json.dumps(results, indent=2))
yield FunctionResult(results)
except Exception:
yield FunctionError()
def test_success(self, circuits_app, base64content, incident_id,
artifact_file_type, file_name, content_type, description,
expected_result):
""" Test calling with sample values for the parameters """
function_params = {
"base64content": b_to_s(base64content),
"incident_id": incident_id,
"artifact_file_type": artifact_file_type,
"file_name": file_name,
"content_type": content_type,
"description": description
}
result = call_base64_to_artifact_function(circuits_app,
function_params)
verify_subset(expected_result, result)
def test_success(self, circuits_app, xml_source, xml_stylesheet,
expected_results):
curr_dir = os.path.dirname(os.path.realpath(__file__))
xml_data = open(os.path.join(curr_dir,
TestUtilitiesXmlTransformation.DATA_DIR,
xml_source),
mode="rb").read()
expected_results = open(os.path.join(
curr_dir, TestUtilitiesXmlTransformation.DATA_DIR,
expected_results),
mode="r").read()
""" Test calling with sample values for the parameters """
function_params = {
"xml_source": b_to_s(xml_data),
"xml_stylesheet": xml_stylesheet
}
results = call_utilities_xml_transformation_function(
circuits_app, function_params)
assert expected_results == results['content']
def _email_parse_function(self, event, *args, **kwargs):
"""Function: Extract message headers and body parts from an email message (.eml or .msg).
Any attachments found are added to the Incident as Artifacts if 'utilities_parse_email_attachments' is set to True"""
try:
log = logging.getLogger(__name__)
# Set variables
parsed_email = path_tmp_file = path_tmp_dir = reason = results = None
# Get the function inputs:
fn_inputs = validate_fields(["incident_id"], kwargs)
# Instansiate ResultPayload
rp = ResultPayload(CONFIG_DATA_SECTION, **kwargs)
# If its just base64content as input, use parse_from_string
if fn_inputs.get("base64content"):
yield StatusMessage("Processing provided base64content")
parsed_email = mailparser.parse_from_string(
b_to_s(base64.b64decode(fn_inputs.get("base64content"))))
yield StatusMessage("Provided base64content processed")
else:
# Validate that either: (incident_id AND attachment_id OR artifact_id) OR (task_id AND attachment_id) is defined
if not (fn_inputs.get("incident_id") and (fn_inputs.get("attachment_id") or fn_inputs.get("artifact_id"))) and \
not (fn_inputs.get("task_id") and fn_inputs.get("attachment_id")):
raise FunctionError(
"You must define either: (incident_id AND attachment_id OR artifact_id) OR (task_id AND attachment_id)"
)
# Instansiate new Resilient API object
res_client = self.rest_client()
# Get attachment metadata
attachment_metadata = get_file_attachment_metadata(
res_client=res_client,
incident_id=fn_inputs.get("incident_id"),
artifact_id=fn_inputs.get("artifact_id"),
task_id=fn_inputs.get("task_id"),
attachment_id=fn_inputs.get("attachment_id"))
# Get attachment content
attachment_contents = get_file_attachment(
res_client=res_client,
incident_id=fn_inputs.get("incident_id"),
artifact_id=fn_inputs.get("artifact_id"),
task_id=fn_inputs.get("task_id"),
attachment_id=fn_inputs.get("attachment_id"))
# Write the attachment_contents to a temp file
path_tmp_file, path_tmp_dir = write_to_tmp_file(
attachment_contents,
tmp_file_name=attachment_metadata.get("name"))
# Get the file_extension
file_extension = os.path.splitext(path_tmp_file)[1]
if file_extension == ".msg":
yield StatusMessage("Processing MSG File")
try:
parsed_email = mailparser.parse_from_file_msg(
path_tmp_file)
yield StatusMessage("MSG File processed")
except Exception as err:
reason = u"Could not parse {0} MSG File".format(
attachment_metadata.get("name"))
yield StatusMessage(reason)
results = rp.done(success=False,
content=None,
reason=reason)
log.error(err)
else:
yield StatusMessage("Processing Raw Email File")
try:
parsed_email = mailparser.parse_from_file(
path_tmp_file)
yield StatusMessage("Raw Email File processed")
except Exception as err:
reason = u"Could not parse {0} Email File".format(
attachment_metadata.get("name"))
yield StatusMessage(reason)
results = rp.done(success=False,
content=None,
reason=reason)
log.error(err)
if parsed_email is not None:
if not parsed_email.mail:
reason = u"Raw email in unsupported format. Failed to parse {0}".format(
u"provided base64content" if fn_inputs.
get("base64content"
) else attachment_metadata.get("name"))
yield StatusMessage(reason)
results = rp.done(success=False,
content=None,
reason=reason)
else:
# Load all parsed email attributes into a Python Dict
parsed_email_dict = json.loads(parsed_email.mail_json,
encoding="utf-8")
parsed_email_dict[
"plain_body"] = parsed_email.text_plain_json
parsed_email_dict[
"html_body"] = parsed_email.text_html_json
yield StatusMessage("Email parsed")
# If the input 'utilities_parse_email_attachments' is true and some attachments were found
if fn_inputs.get("utilities_parse_email_attachments"
) and parsed_email_dict.get(
"attachments"):
yield StatusMessage(
"Attachments found in email message")
attachments_found = parsed_email_dict.get(
"attachments")
# Loop attachments found
for attachment in attachments_found:
yield StatusMessage(
u"Attempting to add {0} to Incident: {1}".
format(attachment.get("filename"),
fn_inputs.get("incident_id")))
# Write the attachment.payload to a temp file
path_tmp_file, path_tmp_dir = write_to_tmp_file(
data=s_to_b(attachment.get("payload")),
tmp_file_name=attachment.get("filename"),
path_tmp_dir=path_tmp_dir)
artifact_description = u"This email attachment was found in the parsed email message from: '{0}'".format(
u"provided base64content" if fn_inputs.
get("base64content"
) else attachment_metadata.get("name"))
# POST the artifact to Resilient as an 'Email Attachment' Artifact
res_client.post_artifact_file(
uri=ARTIFACT_URI.format(
fn_inputs.get("incident_id")),
artifact_type=EMAIL_ATTACHMENT_ARTIFACT_ID,
artifact_filepath=path_tmp_file,
description=artifact_description,
value=attachment.get("filename"),
mimetype=attachment.get("mail_content_type"))
results = rp.done(True, parsed_email_dict)
else:
reason = u"Raw email in unsupported format. Failed to parse {0}".format(
u"provided base64content" if fn_inputs.
get("base64content") else attachment_metadata.get("name"))
yield StatusMessage(reason)
results = rp.done(success=False, content=None, reason=reason)
log.info("Done")
yield FunctionResult(results)
except Exception:
yield FunctionError()
finally:
# Remove the tmp directory
if path_tmp_dir and os.path.isdir(path_tmp_dir):
shutil.rmtree(path_tmp_dir)
def test_data_b64_by_id(id):
"""Read a test data file, return its contents as base64"""
return b_to_s(base64.b64encode(AttachmentMock.test_data_by_id(id)))
def test_data_b64(filename):
"""Read a test data file, return its contents as base64"""
return b_to_s(base64.b64encode(AttachmentMock.test_data(filename)))
def _attachment_zip_list_function(self, event, *args, **kwargs):
"""Function: For a zipfile attachment, return a list of its contents."""
try:
log = logging.getLogger(__name__)
# Get the function parameters:
incident_id = kwargs.get("incident_id") # number
task_id = kwargs.get("task_id") # number
attachment_id = kwargs.get("attachment_id") # number
log.info("incident_id: %s", incident_id)
log.info("task_id: %s", task_id)
log.info("attachment_id: %s", attachment_id)
if incident_id is None and task_id is None:
raise FunctionError(
"Error: incident_id or task_id must be specified.")
if attachment_id is None:
raise FunctionError("Error: attachment_id must be specified.")
yield StatusMessage("Reading attachment...")
client = self.rest_client()
data = get_file_attachment(client,
incident_id,
task_id=task_id,
attachment_id=attachment_id)
results = {}
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
try:
temp_file.write(data)
temp_file.close()
# Examine with zip
zfile = zipfile.ZipFile(temp_file.name, "r")
results["namelist"] = zfile.namelist()
# Don't include zinfo.extra since it's not a string
results["infolist"] = [{
"filename":
zinfo.filename,
"date_time":
epoch_millis(zinfo.date_time),
"compress_type":
zinfo.compress_type,
"comment":
b_to_s(zinfo.comment),
"create_system":
zinfo.create_system,
"create_version":
zinfo.create_version,
"extract_version":
zinfo.extract_version,
"flag_bits":
zinfo.flag_bits,
"volume":
zinfo.volume,
"internal_attr":
zinfo.internal_attr,
"external_attr":
zinfo.external_attr,
"header_offset":
zinfo.header_offset,
"CRC":
zinfo.CRC,
"compress_size":
zinfo.compress_size,
"file_size":
zinfo.file_size
} for zinfo in zfile.infolist()]
except (zipfile.LargeZipFile, zipfile.BadZipfile) as exc:
# results["error"] = str(exc)
raise
finally:
os.unlink(temp_file.name)
# Produce a FunctionResult with the return value
yield FunctionResult(results)
except Exception:
yield FunctionError()
def _attachment_zip_extract_function(self, event, *args, **kwargs):
"""Function: Extract a file from a zipfile attachment, producing a base64 string."""
try:
log = logging.getLogger(__name__)
# Get the function parameters:
incident_id = kwargs.get("incident_id") # number
task_id = kwargs.get("task_id") # number
attachment_id = kwargs.get("attachment_id") # number
file_path = kwargs.get("file_path") # text
zipfile_password = kwargs.get("zipfile_password") # text
if incident_id is None and task_id is None:
raise FunctionError(
"Error: incident_id or task_id must be specified.")
if attachment_id is None:
raise FunctionError("Error: attachment_id must be specified.")
if file_path is None:
raise FunctionError("Error: file_path must be specified.")
log.info("incident_id: %s", incident_id)
log.info("task_id: %s", task_id)
log.info("attachment_id: %s", attachment_id)
log.info("file_path: %s", file_path)
yield StatusMessage("Reading attachment...")
client = self.rest_client()
data = get_file_attachment(client,
incident_id,
task_id=task_id,
attachment_id=attachment_id)
results = {}
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
try:
temp_file.write(data)
temp_file.close()
# Examine with zip
zfile = zipfile.ZipFile(temp_file.name, "r")
# Read the metadata, since it may be useful
zinfo = zfile.getinfo(file_path)
# Don't include zinfo.extra since it's not a string
results["info"] = {
"filename": zinfo.filename,
"date_time": epoch_millis(zinfo.date_time),
"compress_type": zinfo.compress_type,
"comment": b_to_s(zinfo.comment),
"create_system": zinfo.create_system,
"create_version": zinfo.create_version,
"extract_version": zinfo.extract_version,
"flag_bits": zinfo.flag_bits,
"volume": zinfo.volume,
"internal_attr": zinfo.internal_attr,
"external_attr": zinfo.external_attr,
"header_offset": zinfo.header_offset,
"CRC": zinfo.CRC,
"compress_size": zinfo.compress_size,
"file_size": zinfo.file_size
}
# Extract the file we want
b64data = base64.b64encode(
zfile.read(file_path, s_to_b(zipfile_password)))
results["content"] = b_to_s(b64data)
except (KeyError, zipfile.LargeZipFile,
zipfile.BadZipfile) as exc:
# results["error"] = str(exc)
# To help debug, list the contents
log.info(zfile.namelist())
raise
finally:
os.unlink(temp_file.name)
# Produce a FunctionResult with the return value
yield FunctionResult(results)
except Exception:
yield FunctionError()