def send_data(type_info, inc_id, rest_client_helper, payload,\
feed_outputs, is_deleted, incl_attachment_data):
"""
perform the sync to the different datastores
:param type_info:
:param inc_id:
:param rest_client:
:param payload:
:param feed_outputs:
:param is_deleted: true/false
:param incl_attachment_data: true/false
:return: None
"""
context = FeedContext(type_info, inc_id,
rest_client_helper.inst_rest_client, is_deleted)
type_name = type_info.get_pretty_type_name()
# make sure the incident has a org_name
if type_name == 'incident':
payload['org_name'] = type_info.get_org_name(payload['org_id'])
# collect attachment data to pass on
elif not is_deleted and incl_attachment_data \
and type_name == 'attachment':
# this will return a byte string
payload['content'] = get_file_attachment(
rest_client_helper.inst_rest_client,
inc_id,
task_id=payload.get('task_id'),
attachment_id=payload['id'])
elif not is_deleted and incl_attachment_data \
and type_name == 'artifact' \
and payload.get('attachment'):
# this will return a byte string
payload['content'] = get_file_attachment(
rest_client_helper.inst_rest_client,
inc_id,
artifact_id=payload['id'])
for feed_output in feed_outputs:
# don't let a failure in one feed break all the rest
try:
LOG.debug("Calling feed %s", feed_output.__class__.__name__)
feed_output.send_data(context, payload)
except Exception as err:
LOG.error("Failure in update to %s %s",
feed_output.__class__.__name__, err)
error_trace = traceback.format_exc()
LOG.error("Traceback %s", error_trace)
def _utility_email_file_parser_function(self, event, *args, **kwargs):
"""Function: Parses .eml files for email forensics. Useful for reported phishes."""
try:
# Get the function parameters:
incident_id = kwargs.get("incident_id") # number
attachment_id = kwargs.get("attachment_id") # number
eml_filename = kwargs.get("attachment_name") # text
# Get the eml file attachment by its incident and attachment IDs
eml_file = get_file_attachment(self.rest_client(), incident_id, artifact_id=None, task_id=None, attachment_id=attachment_id)
yield StatusMessage('Reading and decoding email message (' + eml_filename + ')...')
# Parse the email content
mail = email.message_from_string(eml_file.decode("utf-8")) # Get the email object from the raw contents
email_body, attachments, urls = get_decoded_email_body(self, incident_id, eml_filename, mail) # Get the UTF-8 encoded body from the raw email string
email_header = get_decoded_email_header(mail.items())
results = {}
results['body'] = str(email_body) # The full email, HTML formatted
results['header'] = email_header # List of 2-tuples containing all the message’s field headers and values
# results['mail_items'] = mail.items() # List of 2-tuples containing all the decoded message’s field headers and values (3/12/2019: deprecated, use 'header')
results['attachments'] = attachments # List of attachment names from EML file
results['urls'] = list(set(urls)) # URLs from body. Set inside of the list used to ensures no duplicates.
# Produce a FunctionResult with the results
yield FunctionResult(results)
except Exception:
yield FunctionError()
def _attachment_to_base64_function(self, event, *args, **kwargs):
"""Function: Produce base64 content of a file attachment."""
try:
log = logging.getLogger(__name__)
# Get the function parameters:
incident_id = kwargs.get("incident_id") # number
task_id = kwargs.get("task_id") # number
attachment_id = kwargs.get("attachment_id") # number
artifact_id = kwargs.get("artifact_id") # number
log.info("incident_id: %s", incident_id)
log.info("task_id: %s", task_id)
log.info("attachment_id: %s", attachment_id)
log.info("artifact_id: %s", artifact_id)
if incident_id is None:
raise FunctionError("Error: incident_id must be specified.")
elif attachment_id is None and artifact_id is None:
raise FunctionError(
"Error: attachment_id or artifact_id must be specified.")
else:
yield StatusMessage("> Function inputs OK")
yield StatusMessage("> Reading attachment...")
client = self.rest_client()
data = get_file_attachment(client,
incident_id,
artifact_id=artifact_id,
task_id=task_id,
attachment_id=attachment_id)
metadata = get_file_attachment_metadata(
client,
incident_id,
artifact_id=artifact_id,
task_id=task_id,
attachment_id=attachment_id)
results = {
"filename": metadata["name"],
"content_type": metadata["content_type"],
"size": metadata["size"],
"created": metadata["created"],
"content": b_to_s(base64.b64encode(data)),
}
yield StatusMessage("> Complete...")
# Produce a FunctionResult with the return value
log.debug(json.dumps(results, indent=2))
yield FunctionResult(results)
except Exception:
yield FunctionError()
def download_attachment_if_available(cls, artifact_id, attachment_id, gcp_artifact_input, incident_id,
task_id):
"""
Attempts to download an attachment if an artifact input was not specified.
:param artifact_id:
:param attachment_id:
:param gcp_artifact_input:
:param incident_id:
:param task_id:
:return:
"""
attachment_input = None
attachment_name = None
# Check whether we are dealing with an attachment or artifact
if (artifact_id or attachment_id or task_id) and gcp_artifact_input is None:
LOG.info("Input appears to be an attachment, downloading from REST API")
# Get the files data
attachment_input = resilient_lib.get_file_attachment(
incident_id=incident_id, artifact_id=artifact_id,
attachment_id=attachment_id, task_id=task_id, res_client=cls.res_client)
# Get the files name
attachment_name = resilient_lib.get_file_attachment_name(
incident_id=incident_id, artifact_id=artifact_id,
attachment_id=attachment_id, task_id=task_id, res_client=cls.res_client)
# Perform some special handling to get the text out of a PDF
if '.pdf' in attachment_name:
LOG.debug("Dealing with a PDF")
attachment_input = cls.extract_text_from_pdf(attachment_input)
elif '.odt' in attachment_name:
LOG.debug("Dealing with a ODT")
attachment_input = cls.extract_text_from_odt(attachment_input)
elif '.docx' in attachment_name:
LOG.debug("Dealing with a docx")
attachment_input = cls.extract_text_from_docx(attachment_input)
else:
# We are not dealing with an attachment
LOG.debug("Working with an artifact")
return cls.attempt_to_parse_as_utf8(attachment_input), attachment_name
def _utility_txt_to_json_structure_function(self, event, *args, **kwargs):
results = {}
results["was_successful"] = False
try:
# Get the function parameters:
incident_id = kwargs.get("incident_id") # number
attachment_id = kwargs.get("attachment_id") # number
attachment_name = kwargs.get("attachment_name") # text (not required at this time)
split_rows_on_new_lines = kwargs.get("split_rows_on_new_lines") # bool
row_limit = kwargs.get("row_limit") # number (optional)
log = logging.getLogger(__name__) # Establish logging
yield StatusMessage('Converting {} data to JSON...'.format(attachment_name))
# Get the TXT file attachment by its incident and attachment IDs
txt_file_data = get_file_attachment(self.rest_client(), incident_id, artifact_id=None, task_id=None, attachment_id=attachment_id)
txt_file = StringIO(unicodedata.normalize("NFKD", txt_file_data.decode('utf-8', 'ignore')))
txt_data = []
if split_rows_on_new_lines is True: # Each line will be split and added to the txt_data list
lines = txt_file.readlines()
row_index = 0
for line in lines:
row_index += 1
txt_data.append({'content': line})
if row_limit and row_index >= int(row_limit): break
else: # All content will be in a single item in the txt_data list
txt_data = {'content': txt_file.read()}
results["json_data"] = [txt_data]
results["was_successful"] = True
# Produce a FunctionResult with the results
yield FunctionResult(results)
except Exception:
yield FunctionError()
def get_incident_file_attachment(rest_client, incident_id, attachment_name):
"""
Get incident file attachment data from Resilient.
Gracefully fail by returning null if the attachment is not found.
:param rest_client: Resilient REST API client
:param incident_id: Resilient incident ID
:param attachment_name: name of the attachment to get
:return: base64 encoded attachment data, attachment ID
"""
# Get the attachment ID
attachment_id = get_attachment_id(rest_client, incident_id,
attachment_name)
if not attachment_id:
return None, None
# Get the attachment data content, base64 encode it, and decode bytes to string
content = get_file_attachment(rest_client,
incident_id,
attachment_id=attachment_id)
encodedContent = base64.b64encode(content).decode("utf-8")
return encodedContent, attachment_id
def _email_parse_function(self, event, *args, **kwargs):
"""Function: Extract message headers and body parts from an email message (.eml or .msg).
Any attachments found are added to the Incident as Artifacts if 'utilities_parse_email_attachments' is set to True"""
try:
log = logging.getLogger(__name__)
# Set variables
parsed_email = path_tmp_file = path_tmp_dir = reason = results = None
# Get the function inputs:
fn_inputs = validate_fields(["incident_id"], kwargs)
# Instansiate ResultPayload
rp = ResultPayload(CONFIG_DATA_SECTION, **kwargs)
# If its just base64content as input, use parse_from_string
if fn_inputs.get("base64content"):
yield StatusMessage("Processing provided base64content")
parsed_email = mailparser.parse_from_string(
b_to_s(base64.b64decode(fn_inputs.get("base64content"))))
yield StatusMessage("Provided base64content processed")
else:
# Validate that either: (incident_id AND attachment_id OR artifact_id) OR (task_id AND attachment_id) is defined
if not (fn_inputs.get("incident_id") and (fn_inputs.get("attachment_id") or fn_inputs.get("artifact_id"))) and \
not (fn_inputs.get("task_id") and fn_inputs.get("attachment_id")):
raise FunctionError(
"You must define either: (incident_id AND attachment_id OR artifact_id) OR (task_id AND attachment_id)"
)
# Instansiate new Resilient API object
res_client = self.rest_client()
# Get attachment metadata
attachment_metadata = get_file_attachment_metadata(
res_client=res_client,
incident_id=fn_inputs.get("incident_id"),
artifact_id=fn_inputs.get("artifact_id"),
task_id=fn_inputs.get("task_id"),
attachment_id=fn_inputs.get("attachment_id"))
# Get attachment content
attachment_contents = get_file_attachment(
res_client=res_client,
incident_id=fn_inputs.get("incident_id"),
artifact_id=fn_inputs.get("artifact_id"),
task_id=fn_inputs.get("task_id"),
attachment_id=fn_inputs.get("attachment_id"))
# Write the attachment_contents to a temp file
path_tmp_file, path_tmp_dir = write_to_tmp_file(
attachment_contents,
tmp_file_name=attachment_metadata.get("name"))
# Get the file_extension
file_extension = os.path.splitext(path_tmp_file)[1]
if file_extension == ".msg":
yield StatusMessage("Processing MSG File")
try:
parsed_email = mailparser.parse_from_file_msg(
path_tmp_file)
yield StatusMessage("MSG File processed")
except Exception as err:
reason = u"Could not parse {0} MSG File".format(
attachment_metadata.get("name"))
yield StatusMessage(reason)
results = rp.done(success=False,
content=None,
reason=reason)
log.error(err)
else:
yield StatusMessage("Processing Raw Email File")
try:
parsed_email = mailparser.parse_from_file(
path_tmp_file)
yield StatusMessage("Raw Email File processed")
except Exception as err:
reason = u"Could not parse {0} Email File".format(
attachment_metadata.get("name"))
yield StatusMessage(reason)
results = rp.done(success=False,
content=None,
reason=reason)
log.error(err)
if parsed_email is not None:
if not parsed_email.mail:
reason = u"Raw email in unsupported format. Failed to parse {0}".format(
u"provided base64content" if fn_inputs.
get("base64content"
) else attachment_metadata.get("name"))
yield StatusMessage(reason)
results = rp.done(success=False,
content=None,
reason=reason)
else:
# Load all parsed email attributes into a Python Dict
parsed_email_dict = json.loads(parsed_email.mail_json,
encoding="utf-8")
parsed_email_dict[
"plain_body"] = parsed_email.text_plain_json
parsed_email_dict[
"html_body"] = parsed_email.text_html_json
yield StatusMessage("Email parsed")
# If the input 'utilities_parse_email_attachments' is true and some attachments were found
if fn_inputs.get("utilities_parse_email_attachments"
) and parsed_email_dict.get(
"attachments"):
yield StatusMessage(
"Attachments found in email message")
attachments_found = parsed_email_dict.get(
"attachments")
# Loop attachments found
for attachment in attachments_found:
yield StatusMessage(
u"Attempting to add {0} to Incident: {1}".
format(attachment.get("filename"),
fn_inputs.get("incident_id")))
# Write the attachment.payload to a temp file
path_tmp_file, path_tmp_dir = write_to_tmp_file(
data=s_to_b(attachment.get("payload")),
tmp_file_name=attachment.get("filename"),
path_tmp_dir=path_tmp_dir)
artifact_description = u"This email attachment was found in the parsed email message from: '{0}'".format(
u"provided base64content" if fn_inputs.
get("base64content"
) else attachment_metadata.get("name"))
# POST the artifact to Resilient as an 'Email Attachment' Artifact
res_client.post_artifact_file(
uri=ARTIFACT_URI.format(
fn_inputs.get("incident_id")),
artifact_type=EMAIL_ATTACHMENT_ARTIFACT_ID,
artifact_filepath=path_tmp_file,
description=artifact_description,
value=attachment.get("filename"),
mimetype=attachment.get("mail_content_type"))
results = rp.done(True, parsed_email_dict)
else:
reason = u"Raw email in unsupported format. Failed to parse {0}".format(
u"provided base64content" if fn_inputs.
get("base64content") else attachment_metadata.get("name"))
yield StatusMessage(reason)
results = rp.done(success=False, content=None, reason=reason)
log.info("Done")
yield FunctionResult(results)
except Exception:
yield FunctionError()
finally:
# Remove the tmp directory
if path_tmp_dir and os.path.isdir(path_tmp_dir):
shutil.rmtree(path_tmp_dir)
def _attachment_zip_list_function(self, event, *args, **kwargs):
"""Function: For a zipfile attachment, return a list of its contents."""
try:
log = logging.getLogger(__name__)
# Get the function parameters:
incident_id = kwargs.get("incident_id") # number
task_id = kwargs.get("task_id") # number
attachment_id = kwargs.get("attachment_id") # number
log.info("incident_id: %s", incident_id)
log.info("task_id: %s", task_id)
log.info("attachment_id: %s", attachment_id)
if incident_id is None and task_id is None:
raise FunctionError(
"Error: incident_id or task_id must be specified.")
if attachment_id is None:
raise FunctionError("Error: attachment_id must be specified.")
yield StatusMessage("Reading attachment...")
client = self.rest_client()
data = get_file_attachment(client,
incident_id,
task_id=task_id,
attachment_id=attachment_id)
results = {}
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
try:
temp_file.write(data)
temp_file.close()
# Examine with zip
zfile = zipfile.ZipFile(temp_file.name, "r")
results["namelist"] = zfile.namelist()
# Don't include zinfo.extra since it's not a string
results["infolist"] = [{
"filename":
zinfo.filename,
"date_time":
epoch_millis(zinfo.date_time),
"compress_type":
zinfo.compress_type,
"comment":
b_to_s(zinfo.comment),
"create_system":
zinfo.create_system,
"create_version":
zinfo.create_version,
"extract_version":
zinfo.extract_version,
"flag_bits":
zinfo.flag_bits,
"volume":
zinfo.volume,
"internal_attr":
zinfo.internal_attr,
"external_attr":
zinfo.external_attr,
"header_offset":
zinfo.header_offset,
"CRC":
zinfo.CRC,
"compress_size":
zinfo.compress_size,
"file_size":
zinfo.file_size
} for zinfo in zfile.infolist()]
except (zipfile.LargeZipFile, zipfile.BadZipfile) as exc:
# results["error"] = str(exc)
raise
finally:
os.unlink(temp_file.name)
# Produce a FunctionResult with the return value
yield FunctionResult(results)
except Exception:
yield FunctionError()
def _attachment_zip_extract_function(self, event, *args, **kwargs):
"""Function: Extract a file from a zipfile attachment, producing a base64 string."""
try:
log = logging.getLogger(__name__)
# Get the function parameters:
incident_id = kwargs.get("incident_id") # number
task_id = kwargs.get("task_id") # number
attachment_id = kwargs.get("attachment_id") # number
file_path = kwargs.get("file_path") # text
zipfile_password = kwargs.get("zipfile_password") # text
if incident_id is None and task_id is None:
raise FunctionError(
"Error: incident_id or task_id must be specified.")
if attachment_id is None:
raise FunctionError("Error: attachment_id must be specified.")
if file_path is None:
raise FunctionError("Error: file_path must be specified.")
log.info("incident_id: %s", incident_id)
log.info("task_id: %s", task_id)
log.info("attachment_id: %s", attachment_id)
log.info("file_path: %s", file_path)
yield StatusMessage("Reading attachment...")
client = self.rest_client()
data = get_file_attachment(client,
incident_id,
task_id=task_id,
attachment_id=attachment_id)
results = {}
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
try:
temp_file.write(data)
temp_file.close()
# Examine with zip
zfile = zipfile.ZipFile(temp_file.name, "r")
# Read the metadata, since it may be useful
zinfo = zfile.getinfo(file_path)
# Don't include zinfo.extra since it's not a string
results["info"] = {
"filename": zinfo.filename,
"date_time": epoch_millis(zinfo.date_time),
"compress_type": zinfo.compress_type,
"comment": b_to_s(zinfo.comment),
"create_system": zinfo.create_system,
"create_version": zinfo.create_version,
"extract_version": zinfo.extract_version,
"flag_bits": zinfo.flag_bits,
"volume": zinfo.volume,
"internal_attr": zinfo.internal_attr,
"external_attr": zinfo.external_attr,
"header_offset": zinfo.header_offset,
"CRC": zinfo.CRC,
"compress_size": zinfo.compress_size,
"file_size": zinfo.file_size
}
# Extract the file we want
b64data = base64.b64encode(
zfile.read(file_path, s_to_b(zipfile_password)))
results["content"] = b_to_s(b64data)
except (KeyError, zipfile.LargeZipFile,
zipfile.BadZipfile) as exc:
# results["error"] = str(exc)
# To help debug, list the contents
log.info(zfile.namelist())
raise
finally:
os.unlink(temp_file.name)
# Produce a FunctionResult with the return value
yield FunctionResult(results)
except Exception:
yield FunctionError()
def upload_attachment(self, src_rest_client, orig_org_id, orig_inc_id,
type_name, payload, orig_type_id):
"""
attachments may be incident level, associated with tasks, or part of an artifact
see write_artifact_attachment for artifact file uploads
:param src_rest_client:
:param orig_org_id:
:param orig_inc_id:
:param type_name:
:param payload:
:param orig_type_id:
:return: None
"""
src_artifact_id = src_attachment_id = src_task_id = dst_task_id = None
if payload.get('attachment', {}).get('content_type', None):
src_artifact_id = orig_type_id
else:
src_attachment_id = orig_type_id
# find the incident for this attachment
sync_inc_id, sync_state = self.dbsync.find_incident(
orig_org_id, orig_inc_id)
# do nothing if incident already deleted, bypassed or filtered
if sync_state in ['deleted', 'bypassed', 'filtered']:
LOG.debug("No action on %s: %s:%s->%s", sync_state, orig_inc_id,
type_name, orig_type_id)
return
# is this a task based attachment?
if payload.get("task_id", None):
src_task_id = payload.get("task_id")
_, dst_task_id, sync_state = self.dbsync.find_sync_row(
orig_org_id, orig_inc_id, "task", src_task_id)
# if the task doesn't exist, the attachment will be requeued
if not dst_task_id:
LOG.warning(
"task:%s->%s for attachment id '%s' does not exist, queuing",
orig_inc_id, src_task_id, orig_type_id)
self.dbsync.create_retry_row(orig_org_id, orig_inc_id, "task",
src_task_id, type_name,
orig_type_id, sync_inc_id,
payload, 1)
return
else:
# get the target attachment, if it exists
_, sync_type_id, _ = self.dbsync.find_sync_row(
orig_org_id, orig_inc_id, type_name, orig_type_id)
# attachments cannot be updated
if sync_type_id:
return
# incident missing?
if not sync_inc_id:
self.dbsync.create_retry_row(
orig_org_id, orig_inc_id,
"task" if src_task_id else "incident",
src_task_id if src_task_id else orig_inc_id, type_name,
orig_type_id, None, payload, 1)
return
# read the attachment from the source Resilient
attachment_contents = get_file_attachment(
src_rest_client,
orig_inc_id,
attachment_id=src_attachment_id,
task_id=src_task_id,
artifact_id=src_artifact_id)
file_handle = io.BytesIO(attachment_contents)
LOG.debug('adding %s:%s->%s to %s:%s', type_name, orig_inc_id,
orig_type_id, self.rest_client.org_id, sync_inc_id)
# artifact as file attachment?
if src_artifact_id:
response = self.write_artifact_file(payload['attachment']['name'],
file_handle, sync_inc_id,
payload)
else:
try:
response = write_file_attachment(
self.rest_client,
payload['name'],
file_handle,
sync_inc_id,
task_id=dst_task_id,
content_type=payload['content_type'])
except Exception as err:
LOG.error("Unable to create attachment for file: %s",
payload['name'])
LOG.error(payload)
LOG.exception(err)
response = None
# create sync row
if response:
new_type_id = response.get('id', None)
self.dbsync.create_sync_row(orig_org_id, orig_inc_id, type_name,
orig_type_id, sync_inc_id, new_type_id,
'active')
LOG.info('added %s:%s->%s to %s:%s->%s', type_name, orig_inc_id,
orig_type_id, self.rest_client.org_id, sync_inc_id,
new_type_id)
else:
LOG.error('error adding %s:%s->%s to %s:%s', type_name,
orig_inc_id, orig_type_id, self.rest_client.org_id,
sync_inc_id)
def _isitphishing_html_document_function(self, event, *args, **kwargs):
"""Function: isitphishing_html_document
This function takes an incident id as a required parameter and
task_id, attachment_id, and artifact_id as optional input which
specifies an HTML document to be base64 encoded and sent to the
Vade Secure API endpoint:
https://ws.isitphishing.org/api/v2/document
for analysis to detemine if the document contains phishing.
The "results" dictionary contains the result of the API query in
"contents" and the "inputs" parameters to the function.
"""
try:
rp = ResultPayload(CONFIG_DATA_SECTION, **kwargs)
# Get the function parameters:
incident_id = kwargs.get("incident_id") # number
task_id = kwargs.get("task_id") # number
attachment_id = kwargs.get("attachment_id") # number
artifact_id = kwargs.get("artifact_id") # number
log = logging.getLogger(__name__)
log.info("incident_id: %s", incident_id)
log.info("task_id: %s", task_id)
log.info("attachment_id: %s", attachment_id)
log.info("artifact_id: %s", artifact_id)
# Form the URL for API request.
API_URL = u"{0}/document".format(self.options["isitphishing_api_url"])
# Get the license key to access the API endpoint.
auth_token = get_license_key(self.options["isitphishing_name"], self.options["isitphishing_license"])
# Build the header and the data payload.
headers = {
"Authorization": u'Bearer {}'.format(auth_token),
"Content-type": "application/json",
"Accept": "application/json"
}
# Build the document payload which is a base64-encoded string.
client = self.rest_client()
# Get the attachment data
data = get_file_attachment(client, incident_id, artifact_id, task_id, attachment_id)
filename = get_file_attachment_name(client, incident_id, artifact_id, task_id, attachment_id)
# Base64 encode the document string and build payload.
base64encoded_doc = base64.b64encode(data).decode("ascii")
payload = {"document": base64encoded_doc}
yield StatusMessage("Query isitPhishing endpoint for status of document.")
# Make API URL request
rc = RequestsCommon(self.opts, self.options)
results_analysis = rc.execute_call("post", API_URL, payload, log=log, headers=headers)
results = rp.done(True, results_analysis)
# add back in the filename
results["inputs"]["filename"] = filename
# Produce a FunctionResult with the results
yield FunctionResult(results)
except Exception as err:
yield FunctionError()
def _fn_vmray_sandbox_analyzer_function(self, event, *args, **kwargs):
"""Function: for VMRay Cloud Analyzer integration"""
def write_temp_file(data, name=None):
if name:
path = os.path.join(tempfile.gettempdir(), name)
else:
tf = tempfile.mkstemp()
path = tf[1]
fo = open(path, 'wb')
fo.write(data)
fo.close()
return path
try:
# Get VMRay Sandbox options from app.config file
VMRAY_API_KEY = self.options.get("vmray_api_key")
VMRAY_ANALYZER_URL = self.options.get("vmray_analyzer_url")
VMRAY_ANALYSIS_REPORT_REQUEST_TIMEOUT = float(
self.options.get("vmray_analyzer_report_request_timeout"))
# Get the function parameters:
incident_id = kwargs.get("incident_id") # number
artifact_id = kwargs.get("artifact_id") # number
attachment_id = kwargs.get("attachment_id") # number
analysis_report_status = kwargs.get(
"analysis_report_status") # Boolean
sample_ids = kwargs.get("sample_ids") or [] # List
if not incident_id:
raise ValueError("incident_id is required")
if (not attachment_id) and (not artifact_id):
raise ValueError("attachment_id or artifact_id is required")
log = logging.getLogger(__name__)
log.info("incident_id: %s", incident_id)
log.info("artifact_id: %s", artifact_id)
log.info("attachment_id: %s", attachment_id)
log.info("analysis_report_status: %s", analysis_report_status)
log.info("sample_ids: %s", sample_ids)
sample_final_result = []
if not analysis_report_status:
# VMRay client and Resilient client
vmray = VMRayAPI(VMRAY_API_KEY,
url=VMRAY_ANALYZER_URL,
proxies=RequestsCommon(
self.opts, self.options).get_proxies())
resilient = self.rest_client()
# Get attachment entity we are dealing with (either attachment or artifact)
# then submit it to VMRay Analyzer
sample_file = get_file_attachment(res_client=resilient,
incident_id=incident_id,
artifact_id=artifact_id,
attachment_id=attachment_id)
sample_name = get_file_attachment_name(
res_client=resilient,
incident_id=incident_id,
artifact_id=artifact_id,
attachment_id=attachment_id)
# with tempfile.NamedTemporaryFile('w+b', bufsize=0, delete=True) as temp_file_binary:
# temp_file_binary.write(sample_file)
# sample_ids = [sample["sample_id"] for sample in vmray.submit_samples(temp_file_binary.name, sample_name)]
with open(write_temp_file(sample_file, sample_name),
"rb") as handle:
sample_ids = [
sample["sample_id"] for sample in vmray.submit_samples(
handle, sample_name)
]
log.info("sample_ids: " + str(sample_ids))
# New samples submission might need take as long as hours to finished,
# need to check the if the analysis have been done.
time_of_begin_check_report = time.time()
is_samples_analysis_finished = all(
vmray.check(sample_id) for sample_id in sample_ids)
while not is_samples_analysis_finished:
if time.time(
) - time_of_begin_check_report > VMRAY_ANALYSIS_REPORT_REQUEST_TIMEOUT:
yield StatusMessage(
"Analysis processing still running at Cloud VMRay Analyzer, please check it later. "
)
break
yield StatusMessage(
"Analysis Report not done yet, retrieve every {} seconds"
.format(CHECK_REPORTS_SLEEP_TIME))
time.sleep(CHECK_REPORTS_SLEEP_TIME)
is_samples_analysis_finished = all(
vmray.check(sample_id) for sample_id in sample_ids)
if is_samples_analysis_finished:
for sample_id in sample_ids:
sample_final_result.append({
"sample_id":
sample_id,
"sample_report":
vmray.get_sample_report(sample_id)["data"],
"sample_reputation_report":
vmray.get_sample_reputation_report(sample_id)
["data"],
"sample_analysis_report":
vmray.get_sample_anlysis_report(sample_id)["data"]
})
analysis_report_status = True
results = {
"analysis_report_status": analysis_report_status,
"incident_id": incident_id,
"artifact_id": artifact_id,
"attachment_id": attachment_id,
"sample_final_result": sample_final_result
}
log.info("results: " + str(results))
# Produce a FunctionResult with the results
yield FunctionResult(results)
except Exception as err:
yield FunctionError(err)
def _attachment_hash_function(self, event, *args, **kwargs):
"""Function: Calculate hashes for a file attachment."""
try:
log = logging.getLogger(__name__)
# Get the function parameters:
incident_id = kwargs.get("incident_id") # number
task_id = kwargs.get("task_id") # number
attachment_id = kwargs.get("attachment_id") # number
log.info("incident_id: %s", incident_id)
log.info("task_id: %s", task_id)
log.info("attachment_id: %s", attachment_id)
if incident_id is None and task_id is None:
raise FunctionError(
"Error: incident_id or task_id must be specified.")
if attachment_id is None:
raise FunctionError("Error: attachment_id must be specified.")
yield StatusMessage("Reading attachment...")
client = self.rest_client()
data = get_file_attachment(client,
incident_id,
task_id=task_id,
attachment_id=attachment_id)
metadata = get_file_attachment_metadata(
client,
incident_id,
task_id=task_id,
attachment_id=attachment_id)
results = {
"filename": metadata["name"],
"content_type": metadata["content_type"],
"size": metadata["size"],
"created": metadata["created"]
}
# Hashlib provides a list of all "algorithms_available", but there's duplication, so
# use the standard list: ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')
# Hashlib in 3.2 and above does not supports hashlib.algorithms_guaranteed this contains a longer
# list but will be used instead of hashlib.algorithms
if sys.version_info.major >= 3:
algorithms = hashlib.algorithms_guaranteed
else:
algorithms = hashlib.algorithms
for algo in algorithms:
impl = hashlib.new(algo)
impl.update(data)
# shake algorithms require a 'length' parameter
if algo.startswith("shake_"):
length_list = algo.split('_')
results[algo] = impl.hexdigest(int(length_list[-1]))
else:
results[algo] = impl.hexdigest()
log.info(u"{} sha1={}".format(metadata["name"], results["sha1"]))
# Produce a FunctionResult with the return value
log.debug(json.dumps(results))
yield FunctionResult(results)
except Exception:
yield FunctionError()
def _docker_run_docker_container_function(self, event, *args, **kwargs):
"""Function: A function intended to be used to create a Docker Container from an image, feed an input to the container and then return the results."""
try:
# Get the function parameters:
artifact_id = kwargs.get("artifact_id") # number
attachment_id = kwargs.get("attachment_id") # number
incident_id = kwargs.get("incident_id") # number
task_id = kwargs.get("task_id") # number
docker_image = self.get_select_param(
kwargs.get("docker_image")
) # select, values: "volatility", "nsrl", "plaso", "bloodhound"
docker_input = kwargs.get("docker_input") # text
docker_operation = kwargs.get("docker_operation") # text
docker_artifact_type = kwargs.get("docker_artifact_type")
attachment_name = None # Initialise attachment name as none
payload = ResultPayload("fn_docker", **kwargs)
log = logging.getLogger(__name__)
log.info("artifact_id: %s", artifact_id)
log.info("attachment_id: %s", attachment_id)
log.info("incident_id: %s", incident_id)
log.info("task_id: %s", task_id)
log.info("docker_image: %s", docker_image)
log.info("docker_input: %s", docker_input)
log.info("docker_artifact_type: %s", docker_artifact_type)
log.info("docker_operation: %s", docker_operation)
helper = ResDockerHelper(self.options)
image_to_use = helper.get_config_option(
self.options, "docker_image", True) or docker_image
# Prepare the args which will be rendered into the app.config cmd
escaped_args = {
"docker_input": render(u"{{docker_input|%s}}" % "sh", kwargs),
}
attachment_file_name = None # Initialise filename as None to avoid reference errors
# Check whether we are dealing with an attachment or artifact
if (artifact_id or attachment_id
or task_id) and docker_input is None:
log.debug(
"Input appears to be an attachment, attempting to parse.")
yield StatusMessage("Downloading Attachment from REST API")
# Get the files data
attachment_input = resilient_lib.get_file_attachment(
incident_id=incident_id,
artifact_id=artifact_id,
attachment_id=attachment_id,
task_id=task_id,
res_client=self.rest_client())
# Get the files name
attachment_name = resilient_lib.get_file_attachment_name(
incident_id=incident_id,
artifact_id=artifact_id,
attachment_id=attachment_id,
task_id=task_id,
res_client=self.rest_client())
# Get the external directory in which to save the file
output_vol = helper.get_config_option(
options=self.all_options.get('{}{}'.format(
CONFIGSECTIONPREFIX, image_to_use)),
option_name="primary_source_dir",
optional=True)
log.debug("Writing attachment to bind folder")
if os.path.isdir(output_vol):
# Convert to named temp file
with tempfile.NamedTemporaryFile(
delete=False,
prefix=DOCKERATTACHMENTPREFIX,
dir=output_vol) as temp_file:
try:
temp_file.write(attachment_input)
os.chmod(temp_file.name, 0o666)
temp_file.close()
finally:
attachment_file_name = os.path.split(
temp_file.name)[1]
log.debug("Saving file to %s", temp_file.name)
# Add a attachment_input arg to be rendered into the cmd command
escaped_args.update({
"attachment_input":
render("{{attachment_input|%s}}" % "sh", {
u"attachment_input":
attachment_file_name
}),
})
yield StatusMessage(
u"Added this as an Attachment Input: {}".
format(attachment_name))
else:
errMsg = u"""Could not write file to directory, does the directory {0} exist? If not create it with mkdir {0}""".format(
output_vol)
raise FunctionError(errMsg)
else:
# We are not dealing with an attachment
log.debug("Working with an artifact")
docker_interface = DockerUtils()
# Decide whether to use local connection or remote
docker_interface.setup_docker_connection(options=self.options)
# Ensure the specified image is an approved one
if image_to_use not in helper.get_config_option(
self.options, "docker_approved_images").split(","):
raise ValueError(
"Image is not in list of approved images. Review your app.config"
)
# Gather the command to send to the image and format docker_extra_kwargs for any image specific volumes
command, docker_extra_kwargs, image_fullname, docker_operation = docker_interface.gather_image_args_and_volumes(
helper, image_to_use, self.all_options, escaped_args,
docker_operation)
log.info(
"Command: %s \n Volume Bind: %s",
(command, docker_extra_kwargs.get('volumes', "No Volumes")))
# Now Get the Image
docker_interface.get_image(image_fullname)
# Get the Client
docker_client = docker_interface.get_client()
yield StatusMessage("Now starting container with input")
try:
# Run container using client
container = docker_client.containers.run(
image=image_fullname,
command=render(command, escaped_args),
detach=True, # Detach from container
remove=
False, # Remove set to false as will be removed manually after gathering info
**docker_extra_kwargs)
container_stats = docker_interface.gather_container_stats(
container_id=container.id)
container_id = container.id
# Gather the logs as they happen, until the container finishes.
container_logs = container.logs(follow=True)
yield StatusMessage("Container has finished and logs gathered")
"""
Attempt to remove the container now we have finished.
Will throw an exception if the container has already been removed"""
container_status = container.wait()
container.remove()
except requests.exceptions.HTTPError as request_exception:
yield StatusMessage(
u"""Encountered issue when trying to remove container: {} \n {}"""
.format(
request_exception,
u"""If you supplied an extra app.config value to remove the container this is expected."""
))
timestamp_epoch = int(time.time() * 1000)
# Setup tempfile to write back the attachment
with tempfile.NamedTemporaryFile(mode="w+t",
delete=False) as temp_upload_file:
try:
new_attachment_name = helper.format_result_attachment_name(
image_to_use, container_id)
# Write and close tempfile
temp_upload_file.write(
helper.format_output_attachment_body(
container_id, docker_operation,
attachment_file_name,
docker_artifact_type, docker_input,
container_logs.decode('utf-8'), timestamp_epoch))
temp_upload_file.close()
# Access Resilient API
client = self.rest_client()
# Create POST uri
# ..for a task, if task_id is defined
if task_id:
attachment_uri = '/tasks/{}/attachments'.format(
task_id)
# ...else for an attachment
else:
attachment_uri = '/incidents/{}/attachments'.format(
incident_id)
# POST the new attachment
new_attachment = client.post_attachment(
attachment_uri,
temp_upload_file.name,
filename=new_attachment_name,
mimetype='text/plain')
except Exception as upload_exception:
err_msg = u"""Encountered issue when saving results to a file and uploading via REST API. Exception: {0}""".format(
upload_exception)
raise FunctionError(err_msg)
finally:
os.unlink(temp_upload_file.name)
results = payload.done(
# If container had no errors, 0 will be returned. Use a falsey check to ensure we get 0 else False
success=True
if not container_status.get("StatusCode", 1) else False,
content={
"logs": container_logs.decode('utf-8'),
"container_exit_status": container_status,
"container_stats": container_stats,
"container_id": container_id,
"res_links": {
"res_object":
helper.prepare_res_link(host=self.host_config[0],
incident_id=incident_id,
task_id=task_id)
},
"attachment_name": attachment_name or None,
})
results["metrics"]["timestamp_epoch"] = timestamp_epoch
# Update the docker_operation input to ensure we have captured the operation done, if any
results["inputs"]["docker_operation"] = docker_operation
# Produce a FunctionResult with the results using the FunctionPayload
yield FunctionResult(results)
log.debug("RESULTS: %s", results)
log.info("Complete")
except Exception:
yield FunctionError()
finally:
try:
os.unlink(temp_file.name)
except NameError:
log.debug(
"Error when trying to unlink file, appears file does not exist."
)
else:
log.debug("Successfully cleaned up file")
def _utility_csv_to_json_structure_function(self, event, *args, **kwargs):
results = {}
results["was_successful"] = False
try:
# Get the function parameters:
incident_id = kwargs.get("incident_id") # number
attachment_id = kwargs.get("attachment_id") # number
csv_filename = kwargs.get(
"attachment_name") # text (not required at this time)
csv_fields = kwargs.get(
"csv_fields"
) # text list (optional) (ie: inputs.csv_fields = "Name", "LastAccessTime", "Field To Skip" "CreationTime")
table_fields = kwargs.get(
"table_fields"
) # text list (optional) (ie: inputs.table_fields = "Name", "LastAccessTime", "CreationTime")
row_limit = kwargs.get("row_limit") # number (optional)
column_limit = kwargs.get("column_limit") # number (optional)
log = logging.getLogger(__name__) # Establish logging
# Get the CSV file attachment by its incident and attachment IDs
csv_file_data = get_file_attachment(self.rest_client(),
incident_id,
artifact_id=None,
task_id=None,
attachment_id=attachment_id)
csv_file = StringIO(
unicodedata.normalize("NFKD",
csv_file_data.decode('utf-8', 'ignore')))
try:
csv_dialect = csv.Sniffer().sniff(csv_file.readline())
csv_delimiter = csv_dialect.delimiter
except:
csv_dialect = None
csv_delimiter = ','
csv_file.seek(0)
# Clean the pre-processor provided csv_fields to ensure a clean tuple
if csv_fields:
if ',' in csv_fields and '[' not in csv_fields:
if '(' not in csv_fields:
csv_fields = str('(' + csv_fields + ')')
csv_fields = tuple(
item.strip()
for item in csv_fields.replace('(', '').replace(
')', '').replace("'", ',').split(','))
elif '[' in csv_fields:
csv_fields = tuple([
row.strip(' ')
for row in csv_fields.strip('][').split(',')
])
else:
csv_fields = tuple(csv_fields)
# Clean the pre-processor provided table_fields to ensure a clean tuple
if table_fields:
if ',' in table_fields and '[' not in table_fields:
if '(' not in table_fields:
table_fields = str('(' + table_fields + ')')
table_fields = tuple(
item.strip()
for item in table_fields.replace('(', '').replace(
')', '').replace("'", ',').split(','))
elif '[' in table_fields:
table_fields = tuple([
row.strip(' ')
for row in table_fields.strip('][').split(',')
])
else:
table_fields = tuple(table_fields)
# If csv_fields is not provided
if not csv_fields:
csv_fields = None # Use first row as keys
# If csv_fields is the same as the first row of the data
if csv_fields == (csv_file_data.partition('\n')[0]):
next(csv_file) # Pass over first row
yield StatusMessage(
'Converting {} data to JSON...'.format(csv_filename))
csv_data = csv.DictReader(
(line.replace('\0', '') for line in csv_file),
fieldnames=csv_fields,
dialect='excel',
delimiter=csv_delimiter)
if not table_fields:
table_fields = csv_data.fieldnames
# Python 2 returns an unordered dictionary from csv.DictReader(), but using the order of fieldnames, we can reorder using OrderedDict and lambda
order_maintained_rows = []
row_index = 0
for row in csv_data:
if not any([str(value).strip() for value in row.values()]):
continue # Skip empty rows
row_index += 1
row_dict = dict(
filter(lambda i: i[0] in table_fields, row.items())
) # Remove row items not included in table_fields
fieldnames_tuple = tuple(
filter(lambda i: i in row_dict.keys(), table_fields)
) # Remove table_fields not found in row items
if column_limit and int(column_limit) < row.items():
order_maintained_rows.append(
OrderedDict(
sorted(row_dict.items(),
key=lambda item: fieldnames_tuple.index(
item[0]))[:int(column_limit)]))
else:
order_maintained_rows.append(
OrderedDict(
sorted(row_dict.items(),
key=lambda item: fieldnames_tuple.index(
item[0]))))
if row_limit and row_index >= int(row_limit): break
results["json_data"] = order_maintained_rows
results["fieldnames"] = table_fields
results["was_successful"] = True
# Produce a FunctionResult with the results
yield FunctionResult(results)
except Exception:
yield FunctionError()
def _dt_utils_create_csv_table_function(self, event, *args, **kwargs):
"""Function: Create a utility function to take csv data and add the results to a named datatable."""
try:
# Instantiate new Resilient API object
res_client = self.rest_client()
inputs = {
"incident_id":
get_function_input(kwargs, "incident_id",
optional=False), # number (required)
"attachment_id":
get_function_input(kwargs, "attachment_id",
optional=True), # number (optional)
"has_headers":
get_function_input(kwargs, "dt_has_headers",
optional=False), # boolean (optional)
"csv_data":
get_function_input(kwargs, "dt_csv_data",
optional=True), # text (optional)
"datable_name":
get_function_input(kwargs, "dt_datable_name",
optional=False), # text (required)
"mapping_table":
get_function_input(kwargs, "dt_mapping_table",
optional=False), # text (optional)
"date_time_format":
get_function_input(kwargs,
"dt_date_time_format",
optional=True), # text (optional)
"start_row":
get_function_input(kwargs, "dt_start_row",
optional=True), # number (optional)
"max_rows":
get_function_input(kwargs, "dt_max_rows",
optional=True), # number (optional)
}
LOG.info(inputs)
yield StatusMessage("Starting ...")
mapping_table = convert_json(inputs['mapping_table'])
if not mapping_table:
raise ValueError(
u"Unable to convert mapping_table to json: %s",
inputs['mapping_table'])
# Create payload dict with inputs
rp = ResultPayload(PACKAGE_NAME, **kwargs)
if (inputs["attachment_id"] and inputs["csv_data"]) or \
not (inputs["attachment_id"] or inputs["csv_data"]):
raise ValueError("Specify either attachment_id or csv_data")
# Either an attachment ID or CSV Data is needed to be able to add rows
if inputs["attachment_id"]:
attachment_name = get_file_attachment_name(
res_client,
inputs['incident_id'],
attachment_id=inputs["attachment_id"])
b_csv_data = get_file_attachment(
res_client,
inputs['incident_id'],
attachment_id=inputs["attachment_id"])
csv_data = b_csv_data.decode("utf-8")
if sys.version_info.major < 3:
inline_data = BytesIO(b_csv_data)
else:
inline_data = StringIO(csv_data)
else:
attachment_name = None
csv_data = inputs["csv_data"]
if sys.version_info.major < 3:
inline_data = StringIO(csv_data.encode("utf-8"))
else:
inline_data = StringIO(csv_data)
datatable = RESDatatable(res_client, inputs["incident_id"],
inputs["datable_name"])
# Retrieve the column names for the datatable, and their data_types,
# to compare against what the user provides, and attempt data conversion, if necessary
fields = datatable.get_dt_headers()
dt_ordered_columns = {
fields[field]['order']:
(fields[field]['name'], fields[field]['input_type'])
for field in fields
}
# ordered column names if we need to assign the headers to the columns in column order
dt_column_names = OrderedDict([
dt_ordered_columns[field]
for field in sorted(dt_ordered_columns.keys())
])
# different readers if we have headers or not
dialect = csv.Sniffer().sniff(
csv_data[0:csv_data.find('\n')]) # limit analysis to first row
# py2 needs changes to dialect to avoid unicode attributes
if sys.version_info.major < 3:
for attr in dir(dialect):
a = getattr(dialect, attr)
if type(a) == unicode:
setattr(dialect, attr, bytes(a))
LOG.debug(dialect.__dict__)
if inputs["has_headers"]:
reader = csv.DictReader(
inline_data, dialect=dialect
) # each row is a dictionary keyed by the column name
csv_headers = reader.fieldnames # just the headers
else:
reader = csv.reader(
inline_data,
dialect=dialect) # each row is a list of values
csv_headers = []
mapping_table = build_mapping_table(mapping_table, csv_headers,
dt_column_names)
LOG.debug("csv headers to datatable columns: %s", mapping_table)
# perform the api calls to the datatable
number_of_added_rows, number_of_rows_with_errors = self.add_to_datatable(
reader, datatable, mapping_table, dt_column_names,
inputs['date_time_format'], inputs['start_row'],
inputs['max_rows'])
LOG.info("Number of rows added: %s ", number_of_added_rows)
LOG.info("Number of rows that could not be added: %s",
number_of_rows_with_errors)
row_data = {
"data_source":
attachment_name if attachment_name else "CSV data",
"rows_added": number_of_added_rows,
"rows_with_errors": number_of_rows_with_errors
}
results = rp.done(True, row_data)
yield StatusMessage("Ending ...")
# Produce a FunctionResult with the results
yield FunctionResult(results)
except Exception as err:
yield FunctionError(err)
