def manual_download(captured_sha1):
util.setup_socks()
conn = util.connect_to_db()
cursor = conn.cursor()
# Database query to get the relevant recent record
cursor.execute(
"""
SELECT dump_id,host,url,referer,client,server FROM pe_dumps WHERE sha1 = %s
ORDER BY timestamp DESC;""", (captured_sha1, ))
row = cursor.fetchone()
dump_id = row[0]
host = row[1]
url = row[2]
referer = row[3]
client = row[4]
server = row[5]
full_url = "http://"
ordered_host = server # if host is null, we use ther server IP
if host:
ordered_host = util.reorder_domain(host)
full_url += ordered_host
if url:
full_url += url
print "Starting manual download from :", full_url
# Prepare the urllib2 request
req = urllib2.Request(full_url)
req.add_header("User-Agent", USER_AGENT)
download_time = time.time()
sha1, md5, different, is_interesting_file = download_file(
dump_id, req, captured_sha1)
# Database statement
cursor.execute(
"""
INSERT INTO manual_download_checksums(dump_id, sha1,
md5, different, referer_exists, timestamp, is_pe)
VALUES (%s, %s, %s, %s, %s, TO_TIMESTAMP(%s), %s)""",
(dump_id, sha1, md5, different, False, download_time,
is_interesting_file))
cursor.close()
conn.close()
def manual_download(captured_sha1):
util.setup_socks()
conn = util.connect_to_db()
cursor = conn.cursor()
# Database query to get the relevant recent record
cursor.execute("""
SELECT dump_id,host,url,referer,client,server FROM pe_dumps WHERE sha1 = %s
ORDER BY timestamp DESC;""", (captured_sha1,))
row = cursor.fetchone()
dump_id = row[0]
host = row[1]
url = row[2]
referer = row[3]
client = row[4]
server = row[5]
if host is None:
host = server
ordered_host = util.reorder_domain(host)
full_url = "http://" + ordered_host + url
#print full_url
# Prepare the urllib2 request
req = urllib2.Request(full_url)
req.add_header("User-Agent", USER_AGENT)
download_time = time.time()
sha1, md5, different, is_pe = download_file(dump_id, req, captured_sha1)
# Database statement
cursor.execute("""
INSERT INTO manual_download_checksums(dump_id, sha1,
md5, different, referer_exists, timestamp, is_pe)
VALUES (%s, %s, %s, %s, %s, TO_TIMESTAMP(%s), %s)""",
(dump_id, sha1, md5, different, False, download_time, is_pe))
cursor.close()
conn.close()
def __init__(self):
self.QUERY_RATE_LIMIT = 10
self.ONE_MIN = 60
logging.config.fileConfig(LOG_CONF_FILE)
self.logger = logging.getLogger("amico_logger")
#stdout_handler = logging.StreamHandler(sys.stdout)
#stdout_handler.setLevel(logging.DEBUG)
#formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s'
#'- %(message)s')
#stdout_handler.setFormatter(formatter)
#self.logger.addHandler(stdout_handler)
util.setup_socks()
self.conn = util.connect_to_db()
self.cursor = self.conn.cursor()
self.today = date.today().strftime("%Y-%m-%d")
self.yesterday = (date.today() -
timedelta(days=1)).strftime("%Y-%m-%d")
self.last_month = (date.today() -
timedelta(days=30)).strftime("%Y-%m-%d")
def __init__(self):
self.QUERY_RATE_LIMIT = 10
self.ONE_MIN = 60
logging.config.fileConfig(LOG_CONF_FILE)
self.logger = logging.getLogger("amico_logger")
#stdout_handler = logging.StreamHandler(sys.stdout)
#stdout_handler.setLevel(logging.DEBUG)
#formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s'
#'- %(message)s')
#stdout_handler.setFormatter(formatter)
#self.logger.addHandler(stdout_handler)
util.setup_socks()
self.conn = util.connect_to_db()
self.cursor = self.conn.cursor()
self.today = date.today().strftime("%Y-%m-%d")
self.yesterday = (date.today() -
timedelta(days=1)).strftime("%Y-%m-%d")
self.last_month = (date.today() -
timedelta(days=30)).strftime("%Y-%m-%d")
def db_virus_total(dump_id):
logging.config.fileConfig(LOG_CONF_FILE)
logger = logging.getLogger("amico_logger")
util.setup_socks()
conn = util.connect_to_db()
cursor = conn.cursor()
# Exit if this sha1 has been queried in the past VT_QUERY_INTERVAL period
prev_query_time = datetime(MINYEAR, 1, 1, 0, 0, 0, 0)
time_now = datetime.now()
cursor.execute(
"""
SELECT sha1, md5
FROM pe_dumps
WHERE dump_id = %s""", (dump_id, ))
(sha1, md5) = cursor.fetchone()
try:
cursor.execute(
"SELECT query_time, vt_id FROM virus_total_scans "
"WHERE sha1 = %s "
"ORDER by query_time DESC", (sha1, ))
res = cursor.fetchone()
if res:
prev_query_time = res[0]
vt_id = res[1]
except:
print "sha1:%s no previous VT query" % (sha1, )
pass
vt_query_period = timedelta(days=VT_QUERY_INTERVAL)
if (time_now - prev_query_time) < vt_query_period:
print "sha1:%s has been queried recently. Skipping..." % (sha1, )
cursor.execute(
"""
INSERT INTO ped_vts_mapping (dump_id, vt_id)
VALUES (%s, %s)""", (dump_id, vt_id))
conn.close()
return
tries = 0
success = False
while tries < MAX_TRIES:
try:
tries += 1
json = vt_api.get_vt_report(md5)
if not json:
continue
report = simplejson.loads(json)
if report["response_code"] == 1:
insert_report(cursor, report, sha1, md5, json, dump_id)
success = True
break
elif report["response_code"] == 0:
cursor.execute(
"""
INSERT INTO virus_total_scans(sha1, md5, query_time)
VALUES (%s, %s, CLOCK_TIMESTAMP())
RETURNING vt_id
""", (sha1, md5))
vt_id = cursor.fetchone()[0]
cursor.execute(
"""
INSERT INTO ped_vts_mapping (dump_id, vt_id)
VALUES (%s, %s)""", (dump_id, vt_id))
print "Virus Total: No scan report exists in the VT database"
success = True
break
else:
logger.exception("Unknown response code! %s" %
(report["response_code"], ))
time.sleep(1)
except Exception as e:
print e
logger.exception(
"Try %s. Error in fetching report for md5 %s: %s" %
(tries, md5, e))
time.sleep(5)
if not success:
cursor.execute(
"""
INSERT INTO ped_vts_mapping (dump_id)
VALUES (%s)""", (dump_id, ))
logger.warning("Giving up on dump_id: %s's VT report" % (dump_id, ))
cursor.close()
conn.close()
def db_virus_total(dump_id):
logging.config.fileConfig(LOG_CONF_FILE)
logger = logging.getLogger("amico_logger")
util.setup_socks()
conn = util.connect_to_db()
cursor = conn.cursor()
# Exit if this sha1 has been queried in the past VT_QUERY_INTERVAL period
prev_query_time = datetime(MINYEAR, 1, 1, 0, 0, 0, 0)
time_now = datetime.now()
cursor.execute("""
SELECT sha1, md5
FROM pe_dumps
WHERE dump_id = %s""",
(dump_id,))
(sha1, md5) = cursor.fetchone()
try:
cursor.execute("SELECT query_time, vt_id FROM virus_total_scans "
"WHERE sha1 = %s "
"ORDER by query_time DESC", (sha1,))
res = cursor.fetchone()
if res:
prev_query_time = res[0]
vt_id = res[1]
except:
print "sha1:%s no previous VT query" % (sha1, )
pass
vt_query_period = timedelta(days=VT_QUERY_INTERVAL)
if (time_now - prev_query_time) < vt_query_period:
print "sha1:%s has been queried recently. Skipping..." % (sha1, )
cursor.execute("""
INSERT INTO ped_vts_mapping (dump_id, vt_id)
VALUES (%s, %s)""",
(dump_id, vt_id))
conn.close()
return
tries = 0
success = False
while tries < MAX_TRIES:
try:
tries += 1
json = vt_api.get_vt_report(md5)
if not json:
continue
report = simplejson.loads(json)
if report["response_code"] == 1:
insert_report(cursor, report, sha1, md5, json, dump_id)
success = True
break
elif report["response_code"] == 0:
cursor.execute("""
INSERT INTO virus_total_scans(sha1, md5, query_time)
VALUES (%s, %s, CLOCK_TIMESTAMP())
RETURNING vt_id
""", (sha1, md5))
vt_id = cursor.fetchone()[0]
cursor.execute("""
INSERT INTO ped_vts_mapping (dump_id, vt_id)
VALUES (%s, %s)""",
(dump_id, vt_id))
print "Virus Total: No scan report exists in the VT database"
success = True
break
else:
logger.exception("Unknown response code! %s" %
(report["response_code"],))
time.sleep(1)
except Exception as e:
print e
logger.exception("Try %s. Error in fetching report for md5 %s: %s"
% (tries, md5, e))
time.sleep(5)
if not success:
cursor.execute("""
INSERT INTO ped_vts_mapping (dump_id)
VALUES (%s)""",
(dump_id,))
logger.warning("Giving up on dump_id: %s's VT report" % (dump_id,))
cursor.close()
conn.close()