def test_time_based_judgment_should_deny__add_new_entry_to_log(
_prepare_test_data):
global test_db_path
ip = log_parser.ip_to_int('8.7.6.5')
line = 512
log_file = "some-log-file.log"
log_entry = log_parser.LogEntry(log_file,
line,
ip=ip,
time=datetime.strptime(
"2019-03-28 11:12:30.000+0100",
judgment.DATETIME_FORMAT_PATTERN),
status=401,
byte=4286)
conn = sqlite3.connect(test_db_path)
with conn:
blocker = judgment.TimeBasedIpJudgment(conn)
to_be_deny, cause = blocker.should_deny(log_entry)
assert to_be_deny is False
c = conn.cursor()
c.execute("SELECT COUNT(*) FROM log_ip WHERE ip = ?", (ip, ))
row = c.fetchone()
ip_count = row[0]
assert ip_count == 1
c.execute(
"SELECT COUNT(*) FROM processed_log_ip WHERE ip = ? AND line = ? AND log_file = ?",
(ip, line, log_file))
row2 = c.fetchone()
ip_count = row2[0]
assert ip_count == 1
conn.close()
def test_update_deny(_prepare_test_data):
global test_db_path
ip_network = "123.456.789.321/22"
log_entry = log_parser.LogEntry("some-log-file.log",
2,
ip=log_parser.ip_to_int("1.2.3.4"),
time=datetime.strptime(
"2019-03-28 11:15:33.000+0100",
judgment.DATETIME_FORMAT_PATTERN),
status=401,
request="GET /manager/html",
byte=4286)
judge = "judge of party"
cause = "just for fun"
judgment.update_deny(ip_network, log_entry, judge, cause, test_db_path)
conn = sqlite3.connect(test_db_path)
c = conn.cursor()
c.execute(
"SELECT COUNT(*), cause_of_block FROM block_network WHERE ip = ?",
(log_entry.ip, ))
row = c.fetchone()
ip_count = row[0]
cause_of_block = row[1]
assert ip_count == 1
assert cause_of_block == cause
def test_user_agent_based_judgment_2():
ip = log_parser.ip_to_int('54.36.150.103')
log_entry = log_parser.LogEntry(
"some-log-file.log",
2,
ip=ip,
user_agent=
"Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)")
blacklist_agent = ['http://ahrefs.com', 'http://www.semrush.com']
blocker = judgment.UserAgentBasedIpJudgment(blacklist_agent)
deny, cause = blocker.should_deny(log_entry)
assert deny is True
def test_parse_tomcat_log_line():
line = '127.0.0.1 134.96.214.161 - someone [27/Mar/2019:13:11:45 +0100] "GET /mathcoach/gfx/muetze.ico HTTP/1.1" 200 4286'
pattern = '%h %{X-Forwarded-For}i %l %u %t "%r" %s %b'.replace(
'"', '"').split(' ')
entry = log_parser.parser_tomcat_log_line("no-name.log", 1024, line,
pattern)
logging.debug("%s", entry)
assert entry['ip'] == log_parser.ip_to_int('134.96.214.161')
assert entry['user'] == 'someone'
assert entry['time'] == datetime.strptime('27/Mar/2019:13:11:45 +0100',
'%d/%b/%Y:%H:%M:%S %z')
assert entry['request'].index('GET') >= 0
assert entry['status'] == 200
assert entry['byte'] == 4286
def test_path_based_judgment_free():
bot_path = {
"/phpMyAdmin/", "/pma/", "/myadmin", "/MyAdmin/", "/wp-login",
"/webdav/", "/manager/html"
}
blocker = judgment.PathBasedIpJudgment(bot_path)
entry = log_parser.LogEntry("dummy-log.txt",
1234,
ip=log_parser.ip_to_int('111.21.253.2'),
time=datetime.strptime(
"2019-03-28 11:15:33.000+0100",
judgment.DATETIME_FORMAT_PATTERN),
status=401,
request="GET /test",
byte=4286)
(block, cause) = blocker.should_deny(entry)
assert not block
def test_parse_tomcat_log_line_with_user_agent_3():
line = r'54.36.150.103 - - [02/Oct/2019:08:38:31 +0200] "GET /openolat/dmz/1%3A1%3A1119026850%3A2%3A0%3Acid%3Amenu.guest/ HTTP/1.1" 200 15072 "-" "Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)"'
pattern = '%h %l %u %t "%r" %>s %O "%{Referer}i" "%{User-Agent}i"'.split(
' ')
entry = log_parser.parser_tomcat_log_line("no-name.log", 1024, line,
pattern)
logging.debug("%s", entry)
assert entry['ip'] == log_parser.ip_to_int('54.36.150.103')
assert entry['user'] == '-'
assert entry['time'] == datetime.strptime('02/Oct/2019:08:38:31 +0200',
'%d/%b/%Y:%H:%M:%S %z')
assert entry['request'].index('GET') >= 0
assert entry['status'] == 200
assert entry['byte'] == 0 # Cannot parse '%O'
assert entry[
'user_agent'] == 'Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)'
def test_parse_tomcat_log_line_with_user_agent():
line = '93.242.172.189 - - [01/Apr/2019:07:11:42 +0000] "POST /mathcoach/ui/j_security_check HTTP/1.1" 200 738 "http://local.host/path/to/resource.html" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36"'
pattern = '%h %l %u %t "%r" %>s %O "%{Referer}i" "%{User-Agent}i"'.split(
' ')
entry = log_parser.parser_tomcat_log_line("no-name.log", 1024, line,
pattern)
logging.debug("%s", entry)
assert entry['ip'] == log_parser.ip_to_int('93.242.172.189')
assert entry['user'] == '-'
assert entry['time'] == datetime.strptime('01/Apr/2019:07:11:42 +0000',
'%d/%b/%Y:%H:%M:%S %z')
assert entry['request'].index('POST') >= 0
assert entry['status'] == 200
assert entry['byte'] == 0 # Cannot parse '%O'
assert entry[
'user_agent'] == 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'
def test_lookup():
ip = "134.96.210.150"
expected_network = "134.96.0.0/16"
first_lookup_start = time.perf_counter()
network = judgment.lookup_ip(ip)
first_lookup_stop = time.perf_counter()
first_lookup_duration = first_lookup_stop - first_lookup_start
logging.info("Lookup time: %s", first_lookup_duration)
assert network == expected_network
cache_lookup_start = time.perf_counter()
network_cache = judgment.lookup_ip(log_parser.ip_to_int(ip))
cache_lookup_stop = time.perf_counter()
cache_lookup_duration = cache_lookup_stop - cache_lookup_start
logging.info("Caching time: %s", cache_lookup_duration)
assert network_cache == expected_network
assert cache_lookup_duration <= first_lookup_duration
def test_FileBasedUFWBlock(clean_output_file):
log = [log_parser.LogEntry(
"some-file",
1234,
ip=log_parser.ip_to_int("1.2.3.4"),
network="1.2.3.4/"+str(x),
time=None,
status=404,
request="GET /abcd",
byte=1024,
user=None
)for x in range(10)]
blocker = execution.FileBasedUWFBlock(ufw_script)
blocker.begin_execute()
for l in log:
blocker.block(l)
blocker.end_execute()
pass
def test_time_based_judgment_update_access_time(_prepare_test_data):
global test_db_path
ip = log_parser.ip_to_int('9.10.11.12')
log_entry = log_parser.LogEntry("some-log-file.log",
2,
ip=ip,
time=datetime.strptime(
"2019-03-28 11:15:33.000+0100",
judgment.DATETIME_FORMAT_PATTERN),
status=401,
byte=4286)
conn = sqlite3.connect(test_db_path)
with conn:
blocker = judgment.TimeBasedIpJudgment(test_db_path)
to_be_deny, cause = blocker.should_deny(log_entry)
assert to_be_deny == False
c = conn.cursor()
c.execute("SELECT access_count FROM log_ip WHERE ip = ?", (ip, ))
row = c.fetchone()
ip_count = row[0]
assert ip_count == 5
conn.close()
#!/usr/bin/python3
from datetime import datetime
import time
import pytest
import logging
from find2deny import log_parser
from find2deny import judgment
import sqlite3
test_db_path = './test-data/ipdb.sqlite'
ip_data = [
(log_parser.ip_to_int('1.2.3.4'), '2019-03-28 11:12:13.000+0100',
'2019-03-28 11:12:15.000+0100', 2),
(log_parser.ip_to_int('5.6.7.8'), '2019-03-28 11:12:13.000+0100',
'2019-03-28 11:12:22.000+0100', 30),
(log_parser.ip_to_int('9.10.11.12'), '2019-03-28 11:12:13.000+0100',
'2019-03-28 11:12:13.000+0100', 4),
]
ip_processed_data = [(log_parser.ip_to_int("1.2.3.4"), 1024,
'some-log-file.log')]
def test_path_based_judgment_block():
bot_path = {
"/phpMyAdmin/", "/pma/", "/myadmin", "/MyAdmin/", "/wp-login",
"/webdav/", "/manager/html"
}
blocker = judgment.PathBasedIpJudgment(bot_path)
entry = log_parser.LogEntry("dummy-log.txt",
def test_ip_to_int():
ip = '134.96.214.161'
ip_int = log_parser.ip_to_int(ip)
print(ip_int)
assert ip_int > 0
