def analyze(self, line):
if line[0] == 'Number':
return
# split the entry into elements
Number, Status, CC, Host, Port, Protocol, ASN, Last_Updated, First_Seen, Last_Seen, First_Active, Last_Active, SBL, Abuse_Contact, Details = line
url = Url(url="{}://{}:{}".format(Protocol, Host, Port))
url['tags'] = ['asprox']
evil = {}
evil['status'] = Status
evil['cc'] = CC
evil['status'] = Status
evil['date_added'] = datetime.datetime.strptime(First_Seen, "%Y-%m-%d %H:%M:%S")
evil['last_seen'] = datetime.datetime.strptime(Last_Seen, "%Y-%m-%d %H:%M:%S") if Last_Seen else datetime.datetime.utcnow()
evil['sbl'] = SBL
evil['abuse_contact'] = Abuse_Contact
evil['description'] = Details if Details else "N/A"
evil['id'] = md5.new(First_Seen+Host).hexdigest()
evil['source'] = self.name
url.seen(first=evil['date_added'], last=evil['last_seen'])
url.add_evil(evil)
self.commit_to_db(url)
def analyze(self, line):
if line[0] == 'Number':
return
Number,Status,CC,Host,Port,Protocol, ASN, Last_Updated, First_Seen, Last_Seen, First_Active, Last_Active, SBL, Abuse_Contact, Details = line # split the entry into elements
url = Url(url="{}://{}:{}".format(Protocol, Host, Port))
url['tags'] = ['asprox']
evil = {}
evil['status'] = Status
evil['cc'] = CC
evil['status'] = Status
print First_Seen
evil['date_added'] = datetime.datetime.strptime(First_Seen, "%Y-%m-%d %H:%M:%S")
print Last_Seen
evil['last_seen'] = datetime.datetime.strptime(Last_Seen, "%Y-%m-%d %H:%M:%S") if Last_Seen else "N/A"
evil['sbl'] = SBL
evil['abuse_contact'] = Abuse_Contact
evil['description'] = Details if Details else "N/A"
evil['id'] = md5.new(First_Seen+Host).hexdigest()
evil['source'] = self.name
url.add_evil(evil)
self.commit_to_db(url)
def analyze(self, dict):
# Create the new URL and store it in the DB
evil = dict
url = Url(url=re.search("Host: (?P<url>[^,]+),", dict['description']).group('url'))
evil['id'] = md5.new(dict['guid']).hexdigest()
url.add_evil(evil)
self.commit_to_db(url)
def analyze(self, dict):
g = re.match(r'^URL: (?P<url>.+), IP Address: (?P<ip>[\d.]+), Country: (?P<country>[A-Z]{2}), ASN: (?P<asn>\d+), MD5: (?P<md5>[a-f0-9]+)$', dict['description'])
evil = g.groupdict()
evil['description'] = "N/A"
evil['link'] = dict['link']
evil['id'] = md5.new(dict['description']).hexdigest()
evil['source'] = self.name
url = Url(url=evil['url'])
url.add_evil(evil)
self.commit_to_db(url)
def analyze(self, dict):
g = re.match(
r'^URL: (?P<url>.+), IP Address: (?P<ip>[\d.]+), Country: (?P<country>[A-Z]{2}), ASN: (?P<asn>\d+), MD5: (?P<md5>[a-f0-9]+)$',
dict['description'])
evil = g.groupdict()
evil['description'] = "N/A"
evil['link'] = dict['link']
evil['id'] = md5.new(dict['description']).hexdigest()
evil['source'] = self.name
url = Url(url=evil['url'])
url.add_evil(evil)
self.commit_to_db(url)
def analyze(self, dict):
g = re.match(r'^URL: (?P<url>.+), IP Address: (?P<ip>[\d.]+), Country: (?P<country>[A-Z]{2}), ASN: (?P<asn>\d+), MD5: (?P<md5>[a-f0-9]+)$', dict['description'])
if g:
evil = g.groupdict()
evil['description'] = "N/A"
evil['link'] = dict['link']
try:
d=dict['description'].encode('UTF-8')
evil['id'] = md5.new(d).hexdigest()
evil['source'] = self.name
url = Url(url=evil['url'])
url.add_evil(evil)
self.commit_to_db(url)
except UnicodeError:
sys.stderr.write('error Unicode : %s' % dict['description'])
class SiriUrzVX(Feed):
"""
This gets data from http://vxvault.siri-urz.net/URL_List.php
"""
def __init__(self, name):
super(SiriUrzVX, self).__init__(name, run_every="1h")
self.name = "SiriUrzVX"
def update(self):
feed = urllib2.urlopen(
"http://vxvault.siri-urz.net/URL_List.php").readlines()
self.status = "OK"
for line in feed:
self.analyze(line)
return True
def analyze(self, line):
if line.startswith('#') or line.startswith('\n'):
return
try:
url = toolbox.find_urls(line)[0]
except Exception, e:
# if find_ip raises an exception, it means no ip
# was found in the line, so we return
return
# Create the new ip and store it in the DB
url = Url(url=url, tags=['siri-urz'])
url, new = self.model.save(url, with_status=True)
if new:
self.elements_fetched += 1
class MalwarePatrolVX(Feed):
"""
This gets data from http://www.malwarepatrol.net/cgi/submit?action=list
"""
def __init__(self, name):
super(MalwarePatrolVX, self).__init__(name, run_every="1h")
self.name = "MalwarePatrolVX"
self.source = 'http://www.malwarepatrol.net/cgi/submit?action=list'
self.descriptin = ""
def update(self):
feed = urllib2.urlopen(self.source).readlines()
self.status = "OK"
for line in feed:
self.analyze(line)
return True
def analyze(self, line):
if line.startswith('#') or line.startswith('\n'):
return
try:
url = toolbox.find_urls(line)[0]
except Exception, e:
# if find_urls raises an exception, it means no ip was found in the line, so we return
return
# Create the new ip and store it in the DB
url = Url(url=url, tags=['malwarepatrol'])
url, new = self.model.save(url, with_status=True)
if new:
self.elements_fetched += 1
class CybercrimeTracker(Feed):
def __init__(self, name):
super(CybercrimeTracker, self).__init__(name, run_every="12h")
self.name = "CybercrimeTracker"
self.description = "CyberCrime Tracker - Latest 20 CnC URLS"
self.source = "http://cybercrime-tracker.net/rss.xml"
self.confidence = 90
def update(self):
for dict in self.update_xml('item', ["title", "link", "pubDate", "description"]):
self.analyze(dict)
def analyze(self, dict):
try:
url = toolbox.find_urls(dict['title'])[0]
except Exception, e:
return # if no URL is found, bail
url = Url(url=url, tags=[dict['description'].lower()])
evil = {}
evil['description'] = "%s CC" % (dict['description'].lower())
evil['date_added'] = datetime.datetime.strptime(dict['pubDate'], "%d-%m-%Y")
evil['id'] = md5.new(dict['title']+dict['pubDate']+dict['description']).hexdigest()
evil['source'] = self.name
url.seen(first=evil['date_added'])
url.add_evil(evil)
self.commit_to_db(url)
class CybercrimeTracker(Feed):
def __init__(self, name):
super(CybercrimeTracker, self).__init__(name, run_every="12h")
self.name = "CybercrimeTracker"
self.description = "CyberCrime Tracker - Latest 20 CnC URLS"
self.source = "http://cybercrime-tracker.net/rss.xml"
self.confidence = 90
def update(self, testing=False):
self.update_xml('item', ["title", "link", "pubDate", "description"])
def analyze(self, dict, testing=False):
try:
url = toolbox.find_urls(dict['title'])[0]
except Exception, e:
return # if no URL is found, bail
# Create the new url and store it in the DB
url = Url(
url=url,
tags=['cybercrimetracker', 'malware', dict['description'].lower()])
evil = Evil()
evil['value'] = "%s (%s CC)" % (url['value'],
dict['description'].lower())
evil['tags'] = [
'cybercrimetracker', 'malware', 'cc', dict['description'].lower()
]
evil['info'] = "%s CC. Published on %s" % (dict['description'],
dict['pubDate'])
return url, evil
def analyze(self, dict):
try:
url = toolbox.find_urls(dict["title"])[0]
except Exception:
return # if no URL is found, bail
url = Url(url=url, tags=[dict["description"].lower()])
evil = {}
evil["description"] = "%s CC" % (dict["description"].lower())
evil["date_added"] = datetime.datetime.strptime(dict["pubDate"], "%d-%m-%Y")
evil["id"] = md5.new(dict["title"] + dict["pubDate"] + dict["description"]).hexdigest()
evil["source"] = self.name
url.seen(first=evil["date_added"])
url.add_evil(evil)
self.commit_to_db(url)
def analyze(self, dict):
evil = {}
evil['description'] = dict['title']
try:
evil['date_added'] = datetime.datetime.strptime(dict['description'], "%d/%b/%Y")
except ValueError:
evil['date_added'] = datetime.datetime.strptime(dict['description'], "%b/%Y")
evil['source'] = self.name
# nasty hack because of utf-8 encoded strings
evil['id'] = md5.new(dict['title'].encode('utf-8').encode('hex') + dict['link'] + dict['description']).hexdigest()
url = Url(url=dict['link'], tags=[dict['title'].lower()])
url.seen(first=evil['date_added'])
url.add_evil(evil)
self.commit_to_db(url)
def analyze(self, dict):
g = re.match(
r"^URL: (?P<url>.+), IP Address: (?P<ip>[\d.]+), Country: (?P<country>[A-Z]{2}), ASN: (?P<asn>\d+), MD5: (?P<md5>[a-f0-9]+)$",
dict["description"],
)
if g:
evil = g.groupdict()
evil["description"] = "N/A"
evil["link"] = dict["link"]
try:
d = dict["description"].encode("UTF-8")
evil["id"] = md5.new(d).hexdigest()
evil["source"] = self.name
url = Url(url=evil["url"])
url.add_evil(evil)
url.seen()
self.commit_to_db(url)
except UnicodeError:
sys.stderr.write("error Unicode : %s" % dict["description"])
def analyze(self, dict):
evil = dict
url = Url(re.search("URL: (?P<url>\S+),", dict['description']).group('url'))
evil['id'] = md5.new(re.search(r"id=(?P<id>[a-f0-9]+)", dict['guid']).group('id')).hexdigest()
try:
date_string = re.search(r"\((?P<date>[0-9\-]+)\)", dict['title']).group('date')
evil['date_added'] = datetime.datetime.strptime(date_string, "%Y-%m-%d")
except AttributeError, e:
print "Date not found!"
def analyze(self, dict):
evil = dict
url = Url(re.search("URL: (?P<url>\S+),", dict["description"]).group("url"))
evil["id"] = md5.new(re.search(r"id=(?P<id>[a-f0-9]+)", dict["guid"]).group("id")).hexdigest()
try:
date_string = re.search(r"\((?P<date>[0-9\-]+)\)", dict["title"]).group("date")
evil["date_added"] = datetime.datetime.strptime(date_string, "%Y-%m-%d")
except AttributeError:
pass
try:
evil["status"] = re.search(r"status: (?P<status>[^,]+)", dict["description"]).group("status")
except Exception:
pass
url.add_evil(evil)
url.seen(first=evil["date_added"])
self.commit_to_db(url)
def analyze(self, dict):
evil = dict
evil['date_added'] = datetime.datetime.strptime(dict['first_seen'], "%d-%m-%Y")
# url
evil['url'] = dict['url']
evil['id'] = md5.new(evil['url'] + dict['first_seen']).hexdigest()
evil['description'] = self.description
evil['source'] = self.name
url = Url(url=evil['url'], tags=[dict['malware']])
url.seen(first=evil['date_added'])
url.add_evil(evil)
self.commit_to_db(url)
# ip
evil['url'] = dict['ip']
evil['id'] = md5.new(evil['url'] + dict['first_seen']).hexdigest()
ip = Ip(ip=dict['ip'], tags=[dict['malware']])
ip.seen(first=evil['date_added'])
ip.add_evil(evil)
self.commit_to_db(ip)
def analyze(self, dict):
evil = dict
url = Url(
re.search("URL: (?P<url>\S+),", dict['description']).group('url'))
evil['id'] = md5.new(
re.search(r"id=(?P<id>[a-f0-9]+)",
dict['guid']).group('id')).hexdigest()
try:
date_string = re.search(r"\((?P<date>[0-9\-]+)\)",
dict['title']).group('date')
evil['date_added'] = datetime.datetime.strptime(
date_string, "%Y-%m-%d")
except AttributeError:
pass
try:
evil['status'] = re.search(r"status: (?P<status>[^,]+)",
dict['description']).group('status')
except Exception:
pass
url.add_evil(evil)
url.seen(first=evil['date_added'])
self.commit_to_db(url)
def analyze(self, dict):
# We create an Evil object. Evil objects are what Malcom uses
# to store anything it considers evil. Malware, spam sources, etc.
# Remember that you can create your own datatypes, if need be.
evil = Evil()
# We start populating the Evil() object's attributes with
# information from the dict we parsed earlier
# description
evil['description'] = dict['description']
# status
if dict['description'].find("offline") != -1:
evil['status'] = "offline"
else:
evil['status'] = "online"
# md5
md5 = re.search("MD5 hash: (?P<md5>[0-9a-f]{32,32})",
dict['description'])
if md5 != None:
evil['md5'] = md5.group('md5')
else:
evil['md5'] = "No MD5"
# linkback
evil['guid'] = dict['guid']
# tags
evil['tags'] += ['zeus', 'malware', 'ZeusTrackerDropzones']
# Create an URL element
url = Url(
re.search("URL: (?P<url>\S+),", dict['description']).group('url'))
# This is important. Values have to be unique, since it's this way that
# Malcom will identify them in the database.
# This is probably not the best way, but it will do for now.
evil['value'] = "ZeuS Dropzone"
if md5:
evil['value'] += " (MD5: %s)" % evil['md5']
else:
evil['value'] += " (URL: %s)" % url['value']
# Save elements to DB. The status field will contain information on
# whether this element already existed in the DB.
return url, evil
class MalwaredRu(Feed):
def __init__(self, name):
super(MalwaredRu, self).__init__(name, run_every="1h")
self.name = "MalwaredRu"
self.source = "http://malwared.malwaremustdie.org/rss.php"
self.description = "Collects results from http://malwared.malwaremustdie.org/, last C&C domains seen"
def update(self):
request = urllib2.Request(
self.source,
headers={
"User-agent":
"Mozilla/5.0 (X11; U; Linux i686) Gecko/20071127 Firefox/2.0.0.11"
})
feed = urllib2.urlopen(request).read()
feed = feed[:feed.find('<script>')]
soup = BeautifulSoup(feed, 'xml')
for item in soup.find_all('item'):
strings = list(item.strings)
dict = {
'title': strings[0],
'description': strings[1],
'link': strings[2]
}
self.analyze(dict)
return True
def analyze(self, dict):
evil = {}
evil['description'] = dict['title']
try:
evil['date_added'] = datetime.datetime.strptime(
dict['description'], "%d/%b/%Y")
except ValueError, e:
evil['date_added'] = datetime.datetime.strptime(
dict['description'], "%b/%Y")
evil['source'] = self.name
# nasty hack because of utf-8 encoded strings
evil['id'] = md5.new(dict['title'].encode('utf-8').encode('hex') +
dict['link'] + dict['description']).hexdigest()
url = Url(url=dict['link'], tags=[dict['title'].lower()])
url.seen(first=evil['date_added'])
url.add_evil(evil)
self.commit_to_db(url)
def analyze(self, line):
Number, Status, CC, Host, Port, Protocol = line.split(
',')[:6] # split the entry into elements
_url = Url(url="{}://{}:{}".format(Protocol, Host, Port))
_url['tags'] = ['asprox']
evil = Evil()
evil['tags'] = ['asprox', 'cc']
evil['value'] = 'Asprox C2: {}'.format(_url['value'])
evil['status'] = Status
return _url, evil
def analyze(self, dict):
# Create the new URL and store it in the DB
url = re.search("Host: (?P<url>[^,]+),",
dict['description']).group('url')
url = Url(url=url)
evil = Evil()
evil['details'] = dict['description']
threat_type = re.search('Description: (?P<tag>.+)',
dict['description']).group('tag')
evil['tags'] = ['malwaredomainlist', threat_type]
evil['value'] = "%s (%s)" % (threat_type, url['value'])
evil['link'] = dict['link']
evil['guid'] = dict['guid']
return url, evil
def analyze(self, dict):
evil = dict
evil['url'] = dict['url']
evil['id'] = md5.new(evil['url'] + 'HostsFileEXP').hexdigest()
evil['description'] = self.description
evil['source'] = self.name
url = Url(url=evil['url'])
url.seen()
url.add_evil(evil)
self.commit_to_db(url)
def add_text(self, text, tags=[]):
added = []
for t in text:
elt = None
if t.strip() != "":
if is_url(t):
elt = Url(is_url(t), [])
elif is_hostname(t):
elt = Hostname(is_hostname(t), [])
elif is_ip(t):
elt = Ip(is_ip(t), [])
if elt:
added.append(self.save_element(elt, tags))
if len(added) == 1:
return added[0]
else:
return added
def analyze(self, dict):
# We create an Evil object. Evil objects are what Malcom uses
# to store anything it considers evil. Malware, spam sources, etc.
# Remember that you can create your own datatypes, if need be.
#print dict
#return
mdl = Url()
# We start populating the Evil() object's attributes with
# information from the dict we parsed earlier
mdl['feed'] = "MDLTracker"
try:
mdl['value'] = toolbox.find_urls(dict['description'])[0]
except Exception, e:
return
def analyze(self, dict):
evil = dict
evil['url'] = dict['url']
evil['id'] = md5.new('fumik0' + evil['url']).hexdigest()
evil['description'] = 'Mark by tracker.fumik0.com'
evil['source'] = self.name
url = Url(url=evil['url'])
url.seen()
url.add_evil(evil)
self.commit_to_db(url)
def analyze(self, dict):
url_re = re.compile(
'URL</td><td style=\'color:black;vertical-align:top;\'>(.+)</td>')
exploit_kit_re = re.compile('Detected\s(.+)\sexploit\skit',
re.IGNORECASE)
iframe_re = re.compile('iframe\sinjection', re.IGNORECASE)
cookiebomb_re = re.compile('CookieBomb', re.IGNORECASE)
dynamicdns_re = re.compile('Dynamic\sDNS', re.IGNORECASE)
tds_re = re.compile('TDS\sURL', re.IGNORECASE)
page_data = urllib2.urlopen(dict['link']).read()
self.status = "OK"
url = url_re.findall(page_data)
exploit_kit = exploit_kit_re.findall(page_data)
iframe = iframe_re.findall(page_data)
cookiebomb = cookiebomb_re.findall(page_data)
dynamicdns = dynamicdns_re.findall(page_data)
tds = tds_re.findall(page_data)
if url:
dict["link"] = url[0]
else:
return False
tags = ['urlquery']
if exploit_kit: tags.append(exploit_kit[0])
if iframe: tags.append('iframe infection')
if cookiebomb: tags.append('cookiebomb')
if dynamicdns: tags.append('dynamic dns')
if tds: tags.append('tds')
# Create the new url and store it in the DB
url = Url(url=url[0], tags=tags)
url, new = self.model.save(url, with_status=True)
if new:
self.elements_fetched += 1
def analyze(self, dict):
try:
url = toolbox.find_urls(dict['title'])[0]
except Exception:
return # if no URL is found, bail
url = Url(url=url, tags=['evil'])
evil = {}
dict['pubDate'] = dict['pubDate'].split('+')[0]
evil['description'] = "%s CC" % (dict['description'].lower())
evil['date_added'] = datetime.datetime.strptime(dict['pubDate'], "%a, %d %b %Y %X ")
evil['id'] = md5.new(dict['title']+dict['pubDate']+dict['description']).hexdigest()
evil['source'] = self.name
url.seen(first=evil['date_added'])
url.add_evil(evil)
self.commit_to_db(url)
def analyze(self, dict):
evil = {}
evil['description'] = dict['title']
try:
evil['date_added'] = datetime.datetime.strptime(
dict['description'], "%d/%b/%Y")
except ValueError:
evil['date_added'] = datetime.datetime.strptime(
dict['description'], "%b/%Y")
evil['source'] = self.name
# nasty hack because of utf-8 encoded strings
evil['id'] = md5.new(dict['title'].encode('utf-8').encode('hex') +
dict['link'] + dict['description']).hexdigest()
url = Url(url=dict['link'], tags=[dict['title'].lower()])
url.seen(first=evil['date_added'])
url.add_evil(evil)
self.commit_to_db(url)
def analyze(self, dict):
# We create an Evil object. Evil objects are what Malcom uses
# to store anything it considers evil. Malware, spam sources, etc.
# Remember that you can create your own datatypes, if need be.
evil = Evil()
# We start populating the Evil() object's attributes with
# information from the dict we parsed earlier
evil['feed'] = "SpyEyeConfigs"
evil['url'] = toolbox.find_urls(dict['description'])[0]
# description
evil['description'] = dict['link'] + " " + dict['description']
# status
if dict['description'].find("offline") != -1:
evil['status'] = "offline"
else:
evil['status'] = "online"
# md5
md5 = re.search("MD5 hash: (?P<md5>[0-9a-f]{32,32})",
dict['description'])
if md5 != None:
evil['md5'] = md5.group('md5')
else:
evil['md5'] = "No MD5"
# linkback
evil['source'] = dict['guid']
# type
evil['type'] = 'evil'
# tags
evil['tags'] += ['spyeye', 'malware', 'SpyEyeConfigs']
# date_retreived
evil['date_retreived'] = datetime.datetime.utcnow()
# This is important. Values have to be unique, since it's this way that
# Malcom will identify them in the database.
# This is probably not the best way, but it will do for now.
evil['value'] = "SpyEye Config"
if md5:
evil['value'] += " (MD5: %s)" % evil['md5']
else:
evil['value'] += " (URL: %s)" % evil['url']
# Save elements to DB. The status field will contain information on
# whether this element already existed in the DB.
evil, status = self.analytics.save_element(evil, with_status=True)
if status['updatedExisting'] == False:
self.elements_fetched += 1
# Create an URL element
url = Url(evil['url'], ['evil', 'SpyEyeConfigs'])
# Save it to the DB.
url, status = self.analytics.save_element(url, with_status=True)
if status['updatedExisting'] == False:
self.elements_fetched += 1
# Connect the URL element to the Evil element
self.analytics.data.connect(url, evil, 'hosting')
feed = urllib2.urlopen("http://cybercrime-tracker.net/rss.xml") #Xylitol's tracker
self.status = "OK"
except Exception, e:
self.status = "ERROR: " + str(e)
return False
children = ["title", "link", "pubDate", "description"]
main_node = "item"
tree = etree.parse(feed)
for item in tree.findall("//%s"%main_node):
dict = {}
for field in children:
dict[field] = item.findtext(field)
self.analyze(dict)
return True
def analyze(self, dict):
try:
url = toolbox.find_urls(dict['title'])[0]
except Exception, e:
return
# Create the new url and store it in the DB
url =Url(url=url, tags=['cybercrimetracker', 'malware', dict['description'].lower()])
url, status = self.analytics.save_element(url, with_status=True)
if status['updatedExisting'] == False:
self.elements_fetched += 1
except Exception, e:
self.status = "ERROR: " + str(e)
return False
url = url_re.findall(page_data)
exploit_kit = exploit_kit_re.findall(page_data)
iframe = iframe_re.findall(page_data)
cookiebomb = cookiebomb_re.findall(page_data)
dynamicdns = dynamicdns_re.findall(page_data)
tds = tds_re.findall(page_data)
if url:
dict["link"] = url[0]
else:
return False
tags = ['urlquery']
if exploit_kit: tags.append(exploit_kit[0])
if iframe: tags.append('iframe infection')
if cookiebomb: tags.append('cookiebomb')
if dynamicdns: tags.append('dynamic dns')
if tds: tags.append('tds')
# Create the new url and store it in the DB
url = Url(url=url, tags=tags)
url, status = self.analytics.save_element(url, with_status=True)
if status['updatedExisting'] == False:
self.elements_fetched += 1
feed = urllib2.urlopen(
"http://www.malwarepatrol.net/cgi/submit?action=list"
).readlines()
self.status = "OK"
except Exception, e:
self.status = "ERROR: " + str(e)
return False
for line in feed:
self.analyze(line)
return True
def analyze(self, line):
if line.startswith('#') or line.startswith('\n'):
return
try:
url = toolbox.find_urls(line)[0]
except Exception, e:
# if find_urls raises an exception, it means no ip
# was found in the line, so we return
return
# Create the new ip and store it in the DB
url = Url(url=url, tags=['malwarepatrol'])
url, status = self.analytics.save_element(url, with_status=True)
if status['updatedExisting'] == False:
self.elements_fetched += 1
try:
feed = urllib2.urlopen(
"http://vxvault.siri-urz.net/URL_List.php").readlines()
self.status = "OK"
except Exception, e:
self.status = "ERROR: " + str(e)
return False
for line in feed:
self.analyze(line)
return True
def analyze(self, line):
if line.startswith('#') or line.startswith('\n'):
return
try:
url = toolbox.find_urls(line)[0]
except Exception, e:
# if find_ip raises an exception, it means no ip
# was found in the line, so we return
return
# Create the new ip and store it in the DB
url = Url(url=url, tags=['siri-urz'])
url, status = self.analytics.save_element(url, with_status=True)
if status['updatedExisting'] == False:
self.elements_fetched += 1
class MalcodeBinaries(Feed):
def __init__(self, name):
super(MalcodeBinaries, self).__init__(name, run_every="1h")
self.name = "MalcodeBinaries"
self.description = "Updated Feed of Malicious Executables"
self.source = "http://malc0de.com/rss/"
def update(self):
request = urllib2.Request(
self.source,
headers={
"User-agent":
"Mozilla/5.0 (X11; U; Linux i686) Gecko/20071127 Firefox/2.0.0.11"
})
feed = urllib2.urlopen(request)
children = ["title", "description", "link"]
main_node = "item"
tree = etree.parse(feed)
for item in tree.findall("//%s" % main_node):
dict = {}
for field in children:
dict[field] = item.findtext(field)
self.analyze(dict)
return True
def analyze(self, dict):
try:
url = toolbox.find_urls(dict['description'])[0]
except Exception, e:
return # no URL found, bail
url = Url(url=url, tags=['exe'])
# We create an Evil object. Evil objects are what Malcom uses
# to store anything it considers evil. Malware, spam sources, etc.
# Remember that you can create your own datatypes, if need be.
evil = Evil()
# We start populating the Evil() object's attributes with
# information from the dict we parsed earlier
evil['info'] = dict['description'] # description
evil['tags'] = [self.name, 'malware']
md5 = re.search("MD5 hash: (?P<md5>[0-9a-f]{32,32})",
dict['description']) # md5
if md5 != None:
evil['md5'] = md5.group('md5')
else:
evil['md5'] = "No MD5"
evil['link'] = dict['link'] # linkback
# This is important. Values have to be unique, since it's this way that
# Malcom will identify them in the database.
# This is probably not the best way, but it will do for now.
evil['value'] = "Malcode malware URL"
if md5:
evil['value'] += " (MD5: %s)" % evil['md5']
else:
evil['value'] += " (URL: %s)" % url['value']
# Save elements to DB. The status field will contain information on
# whether this element already existed in the DB.
self.commit_to_db(url, evil)
return False
# <item><title>Solar</title><description>Mar/2014</description><link>http://...</link></item>
children = ["title", "description", "link"]
main_node = "item"
tree = etree.parse(feed)
for item in tree.findall("//%s" % main_node):
dict = {}
for field in children:
dict[field] = item.findtext(field)
self.analyze(dict)
return True
def analyze(self, dict):
try:
url = toolbox.find_urls(dict['link'])[0]
except Exception, e:
return
# Create the new url and store it in the DB
url = Url(url=url,
tags=['Malwared.ru', 'malware', dict['title'].lower()])
url, status = self.analytics.save_element(url, with_status=True)
if status['updatedExisting'] == False:
self.elements_fetched += 1