def add_httplog(fqdn, db, c):
for fqdn_part in iterate_fqdn_parts(fqdn):
execute_with_retry(db, c, """
INSERT INTO httplog ( host, numconnections, firstconnectdate )
VALUES ( LOWER(%s), 1, UNIX_TIMESTAMP(NOW()) )
ON DUPLICATE KEY UPDATE numconnections = numconnections + 1""", ( fqdn_part, ))
db.commit()
def add_httplog(fqdn):
with get_db_connection('brocess') as db:
c = db.cursor()
for fqdn_part in iterate_fqdn_parts(fqdn):
c.execute(
"""INSERT INTO httplog ( host, numconnections, firstconnectdate )
VALUES ( LOWER(%s), 1, UNIX_TIMESTAMP(NOW()) )
ON DUPLICATE KEY UPDATE numconnections = numconnections + 1""",
(fqdn_part, ))
db.commit()
def _is_uncommon_fqdn(self, fqdn):
"""Returns True if the given fqnd is considered "uncommon"."""
# consider a.b.c.d
# if d is common then we want to see if c.d is uncommon
# if c.d is common then we look at b.c.d, and so forth
# if they are all common then we return False
for partial_fqdn in iterate_fqdn_parts(fqdn):
count = query_brocess_by_fqdn(partial_fqdn)
if count is None:
continue
if count < saq.CONFIG[analysis_module].getint('uncommon_network_threshold'):
logging.info("{} is an uncommon network with count {}".format(partial_fqdn, count))
return True
else:
pass
#logging.debug("{} is a common network with count {}".format(partial_fqdn, count))
return False
def is_in_cache_db(self, value, cache_path):
"""Is this URL in crits? value is the result of calling process_url on a URL."""
assert isinstance(value, ParseResult)
with sqlite3.connect('file:{}?mode=ro'.format(cache_path), uri=True) as db:
db_cursor = db.cursor()
row = None
# check ipv4
if is_ipv4(value.hostname):
db_cursor.execute("SELECT id FROM indicators WHERE type = ? AND value = ?",
(CRITS_IPV4, value.hostname))
row = db_cursor.fetchone()
if row:
logging.debug("{} matched ipv4 indicator {}".format(value.hostname, row[0]))
return True
else:
# check fqdn
for partial_fqdn in iterate_fqdn_parts(value.hostname):
#logging.debug("checking crits for {}".format(partial_fqdn))
db_cursor.execute("SELECT id FROM indicators WHERE type = ? AND value = ?",
(CRITS_FQDN, partial_fqdn.lower()))
row = db_cursor.fetchone()
if row:
logging.debug("{} matched fqdn indicator {}".format(partial_fqdn, row[0]))
return True
# check full url
db_cursor.execute("SELECT id FROM indicators WHERE type = ? AND value = LOWER(?)",
(CRITS_URL, value.geturl()))
row = db_cursor.fetchone()
if row:
logging.debug("{} matched url indicator{}".format(value.geturl(), row[0]))
return True
# check url path
path = urlunparse(('', '', value.path, value.params, value.query, value.fragment))
if path:
db_cursor.execute("SELECT id FROM indicators WHERE type = ? AND value = LOWER(?)",
(CRITS_URL_PATH, path))
row = db_cursor.fetchone()
if row:
logging.debug("{} matched url_path indicator {}".format(value.path, row[0]))
return True
# check url file name
if value.path:
if not value.path.endswith('/'):
file_name = value.path.split('/')[-1]
db_cursor.execute("SELECT id FROM indicators WHERE type = ? AND value = LOWER(?)",
(CRITS_FILE_NAME, file_name))
row = db_cursor.fetchone()
if row:
logging.debug("{} matched file_name indicator {}".format(file_name, row[0]))
return True
return False