def thugOnly(url):
if not re.match('http', url):
url = 'http://' + url
if isthug:
logging.info("Thug candidate: HTML code in %s", url)
try:
thug.Thug([url])()
except Exception, e:
logging.error("thug error: %s", e)
return
def decisor(site, url):
if not re.match('http', url):
url = 'http://' + url
try:
url_dl = urllib3.urlopen(url).read()
except Exception as e:
print("-- %s Error: %s %s" % (site, str(e), url))
return
filetype = gettype(url_dl).split(' ')[0]
if (filetype == 'HTML'):
if isthug:
print("-- Thug candidate: HTML code in %s" % url)
try:
thug.Thug([url])()
except Exception as e:
print("- Thug error: %s" % e)
return
else:
dest = './malware/' + filetype
temp_fpath = dest + '/temp'
if not os.path.exists(dest):
os.makedirs(dest)
if not os.path.exists(temp_fpath):
file = open(temp_fpath, 'wb')
file.write(url_dl)
file.close
md5_val = md5(temp_fpath)
os.rename(temp_fpath, dest + "/" + md5_val)
fpath = dest + "/" + md5_val
time.sleep(1)
print("-- " + site + " Saved file type %s with md5: %s" %
(filetype, md5_val))
try:
vtquery.get_vt_result(md5_val, fpath, url, site)
except:
pass
def decisor(url):
if not re.match('http',url):
url = 'http://'+url
try:
url_dl = urllib2.urlopen(url).read()
#except Exception, e:
except:
#print ("-- Error: %s" % e)
return
filetype = gettype(url_dl).split(' ')[0]
md5 = hashlib.md5(url_dl).hexdigest()
if (filetype == 'HTML'):
if isthug:
print ("-- Thug candidate: HTML code in %s" % url)
try:
thug.Thug([url])()
except:
try:
url_dl = urllib2.urlopen(url).read()
except Exception, e:
#print "-- Error: %s" % e
return
filetype = gettype(url_dl).split(' ')[0]
md5 = hashlib.md5(url_dl).hexdigest()
if (filetype == 'HTML'):
if isthug:
print "-- Thug candidate: HTML code in %s" % url
try:
thug.Thug([url])()
except Exception, e:
print "- Thug error: %s" % e
return
else:
dest = '/opt/mwcrawler/unsorted/' + filetype + '/' + md5[:3] + "/" + md5[
3:6] + "/" + md5[6:9] + "/"
fpath = dest + str(md5)
print "Saving to" + dest
if not os.path.exists(dest):
os.makedirs(dest)
if not os.path.exists(fpath):
file = open(fpath, 'wb')