def test_WebTimeExp_redirect():
w = WebTimeExp(qhost='nist.gov',
protocol='https',
ipaddr='50.17.216.216',
config=db.get_mysql_config(CONFIG_INI_TEST),
db=None,
cname='')
assert w.rcode in [301, 302, 303]
assert w.redirect == 'www.nist.gov'
def test_WebTimeExp():
w = WebTimeExp(qhost=GOOD_TIME,
ipaddr=GOOD_TIME_IP,
protocol='http',
config=db.get_mysql_config(CONFIG_INI_TEST),
cname='',
db=None)
assert w.offset() < datetime.timedelta(
seconds=GOOD_TIME_CORRECT) # we should be off by less than 5 seconds
def get_test_database():
assert os.path.exists(CONFIG_INI_TEST)
config = db.get_mysql_config(CONFIG_INI_TEST)
# Safety -- make sure that the test database is different from the real database, or else
# that the word 'test' appears in the tst database name
dbname = config.get('mysql','db')
testdbname = config.get('mysql','testdb')
assert ('test' in testdbname) or (dbname != testdbname)
dbc = db.mysql(config)
dbc.connect(db=testdbname)
return dbc
def test_get_hosts():
import configparser
config = configparser.ConfigParser()
config.add_section('hosts')
config.set('hosts', 'source', 'webtime_test.some_hosts')
config.set('hosts', 'order', 'as_is')
hosts = get_hosts(config)
assert hosts == SOME_HOSTS
# test random order
config.set('hosts', 'order', 'random')
rhosts = get_hosts(config)
assert len(SOME_HOSTS) == len(rhosts)
assert SOME_HOSTS != rhosts
assert sorted(SOME_HOSTS) == sorted(rhosts)
# test the one that's there
hosts = get_hosts(db.get_mysql_config(db_test.CONFIG_INI_TEST))
assert 100 < len(hosts) < 100000
def test_QueryHostEngine_Redirect():
import time, datetime
config = db.get_mysql_config(CONFIG_INI_TEST)
mdb = db.mysql(config)
qhe = QueryHostEngine(config)
# Enable debugging. It will print to stdout but only generate output if the test fails
mdb.debug = 1
qhe.debug = 1
qhe.db.debug = 1
t0 = time.time()
qhe.queryhost(KNOWN_REDIR_SOURCE, force_record=True)
t1 = time.time()
# Now make sure that there was a query done on KNOWN_REDIR_DEST during the time
(id, diff) = mdb.select1(
"select id,NOW()-qdatetime from times where host=%s order by qdatetime desc limit 1",
(KNOWN_REDIR_DEST, ))
assert diff < (t1 - t0) + 4
def test_create_schema():
"""Test creating a database and upgrading it and killing it"""
config = db.get_mysql_config(CONFIG_INI_TEST)
dbc = get_test_database()
#
# Make sure that we are in the correct database
#
assert dbc.select1("select database();")[0]==config.get('mysql','testdb')
# Make sure that log and metadata tables are not present
dbc.execute("DROP TABLE IF EXISTS `metadata`")
dbc.execute("DROP TABLE IF EXISTS `log`")
#
# Send the schema
#
dbc.send_schema(db.file_contents(SCHEMA))
assert dbc.table_exists('dated')==True
assert dbc.table_exists('xxx')==False
def test_QueryHostEngine(wb=False):
import time, datetime
config = db.get_mysql_config(CONFIG_INI_TEST)
mdb = db.mysql(config)
qhe = QueryHostEngine(config)
assert (type(qhe.db) == type(mdb))
assert (qhe.debug == False)
# Enable debugging. It will print to stdout but only generate output if the test fails
mdb.debug = 1
qhe.debug = 1
qhe.db.debug = 1
# Make sure that host is in the database now
# We do this by making sure that there is an entry in the database for 'today'
# Because 'today' may change between the start and the end, we measure it twice,
# and we only do the assert if the day hasn't changed
day0 = datetime.datetime.fromtimestamp(time.time(), pytz.utc).date()
# Run a query!
qhost = GOOD_TIME
qhe.queryhost(qhost)
qdate = day0.isoformat()
windowdays = 30
# Check if host is well behaved
windowstart = (day0 - datetime.timedelta(days=windowdays)).isoformat()
check_between_cmd = "SELECT SUM(wtcount) FROM dated WHERE host=%s and qdate BETWEEN %s and %s"
# if wtcount is 0 (the host is well behaved)
if not mdb.select1(check_between_cmd, (qhost, windowstart, qdate))[0]:
qlast_query = mdb.select1("SELECT qlast FROM wbhosts WHERE host=%s",
(qhost, ))
qlast = None if qlast_query == None else qlast_query[0]
# case: host was not well behaved before, now is well behaved
if not qlast:
mdb.execute(
"INSERT IGNORE INTO wbhosts (host, qlast) VALUES (%s,%s)",
(qhost, qdate))
elif qlast == qdate:
print("***Queried today***")
# case: host was not queried today
else:
mdb.execute("UPDATE wbhosts SET qlast=%s WHERE host=%s",
(qdate, qhost))
(id, ipaddr, qdate) = mdb.select1(
"select id,ipaddr,qdate from dated where host=%s order by id desc limit 1",
(GOOD_TIME, ))
if wb:
qlast = mdb.select1("SELECT qlast FROM wbhosts WHERE host=%s",
(GOOD_TIME, ))[0]
assert qlast.isoformat() == day0.isoformat()
day1 = datetime.datetime.fromtimestamp(time.time(), pytz.utc).date()
assert id > 0 # make sure id is good
assert ipaddr > '' #
assert qdate in [day0, day1
] # the day must be when we started or when we stopped
# Now make sure we got both a http and an https value
(id, ipaddr, http) = mdb.select1(
"select id,ipaddr,https from dated where host=%s and https=0 order by id desc limit 1",
(GOOD_TIME, ))
assert id > 0
(id, ipaddr, http) = mdb.select1(
"select id,ipaddr,https from dated where host=%s and https=1 order by id desc limit 1",
(GOOD_TIME, ))
assert id > 0
# print ("Images created: " + str(num_hosts))
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--debug",action="store_true",help="write results to STDOUT")
parser.add_argument("--verbose",action="store_true",help="output to STDOUT")
parser.add_argument("--config",help="config file",required=True)
parser.add_argument("--host",help="Specify a host; just do the report for that one host")
parser.add_argument("--usg",action="store_true",help="Only USG")
parser.add_argument("--outdir",help="Where to put the output.",default='plots')
parser.add_argument("--nosizes",help="Do not report the size of the tables",action='store_true')
args = parser.parse_args()
config = db.get_mysql_config(args.config)
dbc = db.mysql(config)
if args.debug:
dbc.debug = args.debug
print("debug mode")
dbc.execute("set innodb_lock_wait_timeout=20")
dbc.execute("set tx_isolation='READ-COMMITTED'")
dbc.execute("set time_zone = '+00:00'")
print("Starting at {}".format(time.asctime()))
t0 = time.time()
page_by_host(dbc, args.outdir)
#page_by_ip(dbc, os.path.join(args.outdir,'ipplots'), args.outdir)
t1 = time.time()
def test_get_mysql_config():
config = db.get_mysql_config()
assert int(config['mysql']['port']) == db.DEFAULT_MYSQL_PORT
assert config['mysql']['db'] == db.DEFAULT_MYSQL_DB
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--debug",
action="store_true",
help="write results to STDOUT")
parser.add_argument("--verbose",
action="store_true",
help="Be more verbose")
parser.add_argument("--config", help="config file", required=True)
parser.add_argument("--db",
action="store_true",
help="When running interactive, write to the database")
args = parser.parse_args()
config = db.get_mysql_config(
args.config) # prep it with default MySQL parameters
dbc = db.mysql(config)
ver = dbc.mysql_version()
if args.debug:
print("MySQL Version {}".format(ver))
#table='times'
#for c in dbc.execute("select distinct ipaddr from "+table+" where ipaddr RLIKE ':.{1,3}:' LIMIT 1000").fetchall():
# print("{}: {:40} -> {:40}".format(table,c[0],fix_ipv6(c[0])))
# dbc.execute("update "+table+" set ipaddr=%s where ipaddr=%s",(fix_ipv6(c[0]),c[0]))
# dated need to be automatically for 2017-11-03
#table='dated';
#for c in dbc.execute("select distinct ipaddr from "+table+" where ipaddr RLIKE ':.{1,3}:' and qdate!='2017-11-03' LIMIT 1000").fetchall():
# print("{}: {:40} -> {:40}".format(table,c[0],fix_ipv6(c[0])))
def query(dbc,cmd,desc=None):
c = dbc.execute(cmd)
field_names = [i[0] for i in c.description]
headers = field_names
content = c.fetchall()
print(tabulate.tabulate(content,headers=headers))
#print(tabulate.tabulate([headers],content))
if __name__=="__main__":
import argparse
import sys
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--config",help="config file",default=CONFIG_INI)
args = parser.parse_args()
if not os.path.exists(args.config):
raise RuntimeError("{} does not exist; specify config file with --config FILENAME".
format(args.config))
config = db.get_mysql_config(args.config) # prep it with default MySQL parameters
if config.getint('mysql','debug'):
args.debug = 1 # override
dbc = db.mysql(config)
query(dbc,"select * from log where level='ERR' order by modified desc limit 10")
query(dbc,"select * from log where level='INFO' order by modified desc limit 10")
exit(1)
query(dbc,"select host,offset,qdatetime,ipaddr,cname from times order by qdatetime desc limit 10","Last 10 bad times:")
query(dbc,"select host,qdate,qlast,ipaddr,qcount,ecount,wtcount from dated order by qdate desc,qlast desc limit 10","Last 10 queried")
my_logger.addHandler(
logging.handlers.SysLogHandler(address='/dev/log'))
my_logger.info(msg)
if __name__ == "__main__":
logger_info('{} PID {} Started'.format(__file__, os.getpid()))
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--debug", action="store_true")
parser.add_argument("--config", help="config file", required=True)
args = parser.parse_args()
config = db.get_mysql_config(args.config)
# Running from cron. Make sure only one of us is running. If another is running, exit
try:
fd = getlock(__file__)
logger_info('{} PID {} config={} acquired lock'.format(
__file__, os.getpid(), args.config))
except RuntimeError as e:
logger_info('{} PID {} config={} could not acquire lock'.format(
__file__, os.getpid(), args.config))
print("{}: Could not acquire lock".format(__file__))
exit(0)
# Make sure mySQL works. We do this here so that we don't report
# that we can't connect to MySQL after the loop starts. We cache
# the results in w to avoid reundent connections to the MySQL
if not my_logger:
my_logger = logging.getLogger(__file__)
my_logger.setLevel(logging.INFO)
my_logger.addHandler(logging.handlers.SysLogHandler(address = '/dev/log'))
my_logger.info(msg)
if __name__=="__main__":
logger_info('{} PID {} Started'.format(__file__,os.getpid()))
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--debug",action="store_true")
parser.add_argument("--config",help="config file",required=True)
args = parser.parse_args()
config = db.get_mysql_config(args.config)
# Running from cron. Make sure only one of us is running. If another is running, exit
try:
fd = getlock(__file__)
logger_info('{} PID {} config={} acquired lock'.format(__file__,os.getpid(),args.config))
except RuntimeError as e:
logger_info('{} PID {} config={} could not acquire lock'.format(__file__,os.getpid(),args.config))
print("{}: Could not acquire lock".format(__file__))
exit(0)
# Make sure mySQL works. We do this here so that we don't report
# that we can't connect to MySQL after the loop starts. We cache
# the results in w to avoid reundent connections to the MySQL
# server.
