def test_WebTimeExp_redirect(): w = WebTimeExp(qhost='nist.gov', protocol='https', ipaddr='50.17.216.216', config=db.get_mysql_config(CONFIG_INI_TEST), db=None, cname='') assert w.rcode in [301, 302, 303] assert w.redirect == 'www.nist.gov'
def test_WebTimeExp(): w = WebTimeExp(qhost=GOOD_TIME, ipaddr=GOOD_TIME_IP, protocol='http', config=db.get_mysql_config(CONFIG_INI_TEST), cname='', db=None) assert w.offset() < datetime.timedelta( seconds=GOOD_TIME_CORRECT) # we should be off by less than 5 seconds
def get_test_database(): assert os.path.exists(CONFIG_INI_TEST) config = db.get_mysql_config(CONFIG_INI_TEST) # Safety -- make sure that the test database is different from the real database, or else # that the word 'test' appears in the tst database name dbname = config.get('mysql','db') testdbname = config.get('mysql','testdb') assert ('test' in testdbname) or (dbname != testdbname) dbc = db.mysql(config) dbc.connect(db=testdbname) return dbc
def test_get_hosts(): import configparser config = configparser.ConfigParser() config.add_section('hosts') config.set('hosts', 'source', 'webtime_test.some_hosts') config.set('hosts', 'order', 'as_is') hosts = get_hosts(config) assert hosts == SOME_HOSTS # test random order config.set('hosts', 'order', 'random') rhosts = get_hosts(config) assert len(SOME_HOSTS) == len(rhosts) assert SOME_HOSTS != rhosts assert sorted(SOME_HOSTS) == sorted(rhosts) # test the one that's there hosts = get_hosts(db.get_mysql_config(db_test.CONFIG_INI_TEST)) assert 100 < len(hosts) < 100000
def test_QueryHostEngine_Redirect(): import time, datetime config = db.get_mysql_config(CONFIG_INI_TEST) mdb = db.mysql(config) qhe = QueryHostEngine(config) # Enable debugging. It will print to stdout but only generate output if the test fails mdb.debug = 1 qhe.debug = 1 qhe.db.debug = 1 t0 = time.time() qhe.queryhost(KNOWN_REDIR_SOURCE, force_record=True) t1 = time.time() # Now make sure that there was a query done on KNOWN_REDIR_DEST during the time (id, diff) = mdb.select1( "select id,NOW()-qdatetime from times where host=%s order by qdatetime desc limit 1", (KNOWN_REDIR_DEST, )) assert diff < (t1 - t0) + 4
def test_create_schema(): """Test creating a database and upgrading it and killing it""" config = db.get_mysql_config(CONFIG_INI_TEST) dbc = get_test_database() # # Make sure that we are in the correct database # assert dbc.select1("select database();")[0]==config.get('mysql','testdb') # Make sure that log and metadata tables are not present dbc.execute("DROP TABLE IF EXISTS `metadata`") dbc.execute("DROP TABLE IF EXISTS `log`") # # Send the schema # dbc.send_schema(db.file_contents(SCHEMA)) assert dbc.table_exists('dated')==True assert dbc.table_exists('xxx')==False
def test_QueryHostEngine(wb=False): import time, datetime config = db.get_mysql_config(CONFIG_INI_TEST) mdb = db.mysql(config) qhe = QueryHostEngine(config) assert (type(qhe.db) == type(mdb)) assert (qhe.debug == False) # Enable debugging. It will print to stdout but only generate output if the test fails mdb.debug = 1 qhe.debug = 1 qhe.db.debug = 1 # Make sure that host is in the database now # We do this by making sure that there is an entry in the database for 'today' # Because 'today' may change between the start and the end, we measure it twice, # and we only do the assert if the day hasn't changed day0 = datetime.datetime.fromtimestamp(time.time(), pytz.utc).date() # Run a query! qhost = GOOD_TIME qhe.queryhost(qhost) qdate = day0.isoformat() windowdays = 30 # Check if host is well behaved windowstart = (day0 - datetime.timedelta(days=windowdays)).isoformat() check_between_cmd = "SELECT SUM(wtcount) FROM dated WHERE host=%s and qdate BETWEEN %s and %s" # if wtcount is 0 (the host is well behaved) if not mdb.select1(check_between_cmd, (qhost, windowstart, qdate))[0]: qlast_query = mdb.select1("SELECT qlast FROM wbhosts WHERE host=%s", (qhost, )) qlast = None if qlast_query == None else qlast_query[0] # case: host was not well behaved before, now is well behaved if not qlast: mdb.execute( "INSERT IGNORE INTO wbhosts (host, qlast) VALUES (%s,%s)", (qhost, qdate)) elif qlast == qdate: print("***Queried today***") # case: host was not queried today else: mdb.execute("UPDATE wbhosts SET qlast=%s WHERE host=%s", (qdate, qhost)) (id, ipaddr, qdate) = mdb.select1( "select id,ipaddr,qdate from dated where host=%s order by id desc limit 1", (GOOD_TIME, )) if wb: qlast = mdb.select1("SELECT qlast FROM wbhosts WHERE host=%s", (GOOD_TIME, ))[0] assert qlast.isoformat() == day0.isoformat() day1 = datetime.datetime.fromtimestamp(time.time(), pytz.utc).date() assert id > 0 # make sure id is good assert ipaddr > '' # assert qdate in [day0, day1 ] # the day must be when we started or when we stopped # Now make sure we got both a http and an https value (id, ipaddr, http) = mdb.select1( "select id,ipaddr,https from dated where host=%s and https=0 order by id desc limit 1", (GOOD_TIME, )) assert id > 0 (id, ipaddr, http) = mdb.select1( "select id,ipaddr,https from dated where host=%s and https=1 order by id desc limit 1", (GOOD_TIME, )) assert id > 0
# print ("Images created: " + str(num_hosts)) if __name__ == "__main__": import argparse parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("--debug",action="store_true",help="write results to STDOUT") parser.add_argument("--verbose",action="store_true",help="output to STDOUT") parser.add_argument("--config",help="config file",required=True) parser.add_argument("--host",help="Specify a host; just do the report for that one host") parser.add_argument("--usg",action="store_true",help="Only USG") parser.add_argument("--outdir",help="Where to put the output.",default='plots') parser.add_argument("--nosizes",help="Do not report the size of the tables",action='store_true') args = parser.parse_args() config = db.get_mysql_config(args.config) dbc = db.mysql(config) if args.debug: dbc.debug = args.debug print("debug mode") dbc.execute("set innodb_lock_wait_timeout=20") dbc.execute("set tx_isolation='READ-COMMITTED'") dbc.execute("set time_zone = '+00:00'") print("Starting at {}".format(time.asctime())) t0 = time.time() page_by_host(dbc, args.outdir) #page_by_ip(dbc, os.path.join(args.outdir,'ipplots'), args.outdir) t1 = time.time()
def test_get_mysql_config(): config = db.get_mysql_config() assert int(config['mysql']['port']) == db.DEFAULT_MYSQL_PORT assert config['mysql']['db'] == db.DEFAULT_MYSQL_DB
parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("--debug", action="store_true", help="write results to STDOUT") parser.add_argument("--verbose", action="store_true", help="Be more verbose") parser.add_argument("--config", help="config file", required=True) parser.add_argument("--db", action="store_true", help="When running interactive, write to the database") args = parser.parse_args() config = db.get_mysql_config( args.config) # prep it with default MySQL parameters dbc = db.mysql(config) ver = dbc.mysql_version() if args.debug: print("MySQL Version {}".format(ver)) #table='times' #for c in dbc.execute("select distinct ipaddr from "+table+" where ipaddr RLIKE ':.{1,3}:' LIMIT 1000").fetchall(): # print("{}: {:40} -> {:40}".format(table,c[0],fix_ipv6(c[0]))) # dbc.execute("update "+table+" set ipaddr=%s where ipaddr=%s",(fix_ipv6(c[0]),c[0])) # dated need to be automatically for 2017-11-03 #table='dated'; #for c in dbc.execute("select distinct ipaddr from "+table+" where ipaddr RLIKE ':.{1,3}:' and qdate!='2017-11-03' LIMIT 1000").fetchall(): # print("{}: {:40} -> {:40}".format(table,c[0],fix_ipv6(c[0])))
def query(dbc,cmd,desc=None): c = dbc.execute(cmd) field_names = [i[0] for i in c.description] headers = field_names content = c.fetchall() print(tabulate.tabulate(content,headers=headers)) #print(tabulate.tabulate([headers],content)) if __name__=="__main__": import argparse import sys parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("--config",help="config file",default=CONFIG_INI) args = parser.parse_args() if not os.path.exists(args.config): raise RuntimeError("{} does not exist; specify config file with --config FILENAME". format(args.config)) config = db.get_mysql_config(args.config) # prep it with default MySQL parameters if config.getint('mysql','debug'): args.debug = 1 # override dbc = db.mysql(config) query(dbc,"select * from log where level='ERR' order by modified desc limit 10") query(dbc,"select * from log where level='INFO' order by modified desc limit 10") exit(1) query(dbc,"select host,offset,qdatetime,ipaddr,cname from times order by qdatetime desc limit 10","Last 10 bad times:") query(dbc,"select host,qdate,qlast,ipaddr,qcount,ecount,wtcount from dated order by qdate desc,qlast desc limit 10","Last 10 queried")
my_logger.addHandler( logging.handlers.SysLogHandler(address='/dev/log')) my_logger.info(msg) if __name__ == "__main__": logger_info('{} PID {} Started'.format(__file__, os.getpid())) parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("--debug", action="store_true") parser.add_argument("--config", help="config file", required=True) args = parser.parse_args() config = db.get_mysql_config(args.config) # Running from cron. Make sure only one of us is running. If another is running, exit try: fd = getlock(__file__) logger_info('{} PID {} config={} acquired lock'.format( __file__, os.getpid(), args.config)) except RuntimeError as e: logger_info('{} PID {} config={} could not acquire lock'.format( __file__, os.getpid(), args.config)) print("{}: Could not acquire lock".format(__file__)) exit(0) # Make sure mySQL works. We do this here so that we don't report # that we can't connect to MySQL after the loop starts. We cache # the results in w to avoid reundent connections to the MySQL
if not my_logger: my_logger = logging.getLogger(__file__) my_logger.setLevel(logging.INFO) my_logger.addHandler(logging.handlers.SysLogHandler(address = '/dev/log')) my_logger.info(msg) if __name__=="__main__": logger_info('{} PID {} Started'.format(__file__,os.getpid())) parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("--debug",action="store_true") parser.add_argument("--config",help="config file",required=True) args = parser.parse_args() config = db.get_mysql_config(args.config) # Running from cron. Make sure only one of us is running. If another is running, exit try: fd = getlock(__file__) logger_info('{} PID {} config={} acquired lock'.format(__file__,os.getpid(),args.config)) except RuntimeError as e: logger_info('{} PID {} config={} could not acquire lock'.format(__file__,os.getpid(),args.config)) print("{}: Could not acquire lock".format(__file__)) exit(0) # Make sure mySQL works. We do this here so that we don't report # that we can't connect to MySQL after the loop starts. We cache # the results in w to avoid reundent connections to the MySQL # server.