def memory_urls(self, memory_urls):
self._memory_urls = []
# If we were given a single string, add that.
if RegexHelpers.is_url(str(memory_urls)):
if not self.whitelister.is_url_whitelisted(str(memory_urls)):
self._memory_urls.append(str(memory_urls))
# Otherwise, try and process it like a list or set.
else:
try:
for url in memory_urls:
if RegexHelpers.is_url(str(url)):
if not self.whitelister.is_url_whitelisted(str(url)):
self._memory_urls.append(str(url))
except TypeError:
pass
def get_all_urls(self):
all_urls = []
all_urls += list(self.process_tree_urls)
all_urls += list(self.memory_urls)
all_urls += list(self.strings_urls)
for request in self.http_requests:
url = "http://" + request.host + request.uri
if RegexHelpers.is_url(url):
all_urls.append(url)
return sorted(list(set(all_urls)))
def generate_url_indicators(url_list, whitelister=None):
indicators = []
# In case we were given a string (a single URL), add it
# to a list for consistent processing.
if isinstance(url_list, str):
url_list = [url_list]
# Parse the URLs so that we can create Indicators and also prevent
# "duplicate" URLs like http://blah.com/ and http://blah.com
for url in url_list:
if RegexHelpers.is_url(url):
# Strip off the ending slash if it's there.
if url.endswith("/"):
url = url[:-1]
parsed_url = urlsplit(url)
# Is the netloc an IP address?
if RegexHelpers.is_ip(parsed_url.netloc):
netloc_type = "Address - ipv4-addr"
# If the netloc is not an IP, it must be a domain.
else:
netloc_type = "URI - Domain Name"
# Make an Indicator for the URI host.
try:
ind = Indicator(parsed_url.netloc, netloc_type)
ind.add_tags("uri_host")
ind.add_relationships(url)
indicators.append(ind)
except ValueError:
pass
# Make an Indicator for the full URL.
try:
ind = Indicator(url, "URI - URL")
ind.add_relationships(parsed_url.netloc)
indicators.append(ind)
except ValueError:
pass
# Make an Indicator for the path (if there is one).
if parsed_url.path and parsed_url.path != "/":
try:
ind = Indicator(parsed_url.path, "URI - Path")
ind.add_tags(["uri_path", parsed_url.netloc])
ind.add_relationships([url, parsed_url.netloc])
indicators.append(ind)
except ValueError:
pass
# Make an Indicator for the path including the query items.
if parsed_url.path and parsed_url.path != "/":
try:
# Check if there were any ? query items.
if parsed_url.query:
uri_path = parsed_url.path + "?" + parsed_url.query
ind = Indicator(uri_path, "URI - Path")
ind.add_tags(["uri_path", parsed_url.netloc])
ind.add_relationships([url, parsed_url.netloc])
indicators.append(ind)
except ValueError:
pass
return indicators
def update_phish_info(self, email_list):
self.logger.debug("Updating Phish Information section.")
# Create the parent div tag.
div = self.new_tag("div")
# Add the header tag.
header = self.new_tag("h2", parent=div)
header.string = "Phish E-mail Information"
# Create a new table tag.
table = self.new_tag("table", parent=div)
# Set up the table header row.
thead = self.new_tag("thead", parent=table)
tr = self.new_tag("tr", parent=thead)
titles = [
"URL", "Time", "From", "To", "Subject", "Attachments",
"MD5 Hashes", "CC", "BCC", "Reply-To", "Message ID"
]
for title in titles:
th = self.new_tag("th", parent=tr)
th.string = title
# Set up the table body rows.
tbody = self.new_tag("tbody", parent=table)
for email in email_list:
if isinstance(email, EmailParser.EmailParser):
tr = self.new_tag("tr", parent=tbody)
td = self.new_tag("td", parent=tr)
if RegexHelpers.is_url(email.reference):
link = self.new_tag("a", parent=td)
link["href"] = email.reference
link.string = "Alert"
td = self.new_tag("td", parent=tr)
td.string = email.received_time
td = self.new_tag("td", parent=tr)
td.string = email.from_address
td = self.new_tag("td", parent=tr)
td.string = email.to_string
td = self.new_tag("td", parent=tr)
if email.decoded_subject:
td.string = email.decoded_subject
else:
td.string = email.subject
td = self.new_tag("td", parent=tr)
td.string = email.attachments_string
td = self.new_tag("td", parent=tr)
td.string = email.md5_string
td = self.new_tag("td", parent=tr)
td.string = email.cc_string
td = self.new_tag("td", parent=tr)
td.string = email.bcc_string
td = self.new_tag("td", parent=tr)
td.string = email.replyto
td = self.new_tag("td", parent=tr)
td.string = email.message_id
self.update_section(div, old_section_id="phish_email_information")