def get29AzkabanAlert(endTime):
sql = """
select * from execution_flows
where (flow_data like '%FAILED%' or flow_data like '%KILLED%')
and end_time>={}
""".format(endTime)
db = mysql("azkaban29")
columns = db.query(sql).fetchall()
return columns
def __init__(self, refresh = False, quick = False): self.firstYear = 2008 #self.firstYear = 2015 self.currentYear = date.today().year self.db = mysql() #self.db = mysql(True) self.refresh = refresh self.quick = quick self.complaint = complaint(self.refresh, self.quick)
def get_test_database():
assert os.path.exists(CONFIG_INI_TEST)
config = db.get_mysql_config(CONFIG_INI_TEST)
# Safety -- make sure that the test database is different from the real database, or else
# that the word 'test' appears in the tst database name
dbname = config.get('mysql','db')
testdbname = config.get('mysql','testdb')
assert ('test' in testdbname) or (dbname != testdbname)
dbc = db.mysql(config)
dbc.connect(db=testdbname)
return dbc
def check_db(check_sql):
check_sql_list = check_sql.split(';')
for i in check_sql_list:
if i:
j = i.split('|')
if j:
sql = j[0]
value = j[1]
mysql = db.mysql()
db_value = mysql.query(sql)
unittest.TestCase().assertEqual(
value, str(db_value),
'校验数据库 sql:%s 预期值:%s 数据库中值:%s' % (sql, value, db_value))
def __init__(self, config, debug=False, runid=None):
"""Create the object.
@param db - a proxied database connection
@param debug - if we are debugging
"""
assert type(config) == configparser.ConfigParser
self.windowdays = 30
self.config = config
self.db = db.mysql(config)
self.debug = debug
self.runid = runid
if debug:
self.db.debug = debug
def __init__(self,url=None,timeout=10):
Thread.__init__(self)
if not url:
return None
else:
self.url = url
# Initializing
#super(GetPages,self).__init__(url,timeout)
self.timeout = timeout
# Get the name of googs.
self.namere = re.compile('<h1>.+</h1>')
#self.linkre = patern = re.compile('<li>*?href=')
self.goodsid = url.split('com/')[1].split('.html')[0]
self.mysql = db.mysql(hostname='192.168.88.129',passwd='lilo',user='******',db='test')
def test_QueryHostEngine_Redirect():
import time, datetime
config = db.get_mysql_config(CONFIG_INI_TEST)
mdb = db.mysql(config)
qhe = QueryHostEngine(config)
# Enable debugging. It will print to stdout but only generate output if the test fails
mdb.debug = 1
qhe.debug = 1
qhe.db.debug = 1
t0 = time.time()
qhe.queryhost(KNOWN_REDIR_SOURCE, force_record=True)
t1 = time.time()
# Now make sure that there was a query done on KNOWN_REDIR_DEST during the time
(id, diff) = mdb.select1(
"select id,NOW()-qdatetime from times where host=%s order by qdatetime desc limit 1",
(KNOWN_REDIR_DEST, ))
assert diff < (t1 - t0) + 4
def test_QueryHostEngine(wb=False):
import time, datetime
config = db.get_mysql_config(CONFIG_INI_TEST)
mdb = db.mysql(config)
qhe = QueryHostEngine(config)
assert (type(qhe.db) == type(mdb))
assert (qhe.debug == False)
# Enable debugging. It will print to stdout but only generate output if the test fails
mdb.debug = 1
qhe.debug = 1
qhe.db.debug = 1
# Make sure that host is in the database now
# We do this by making sure that there is an entry in the database for 'today'
# Because 'today' may change between the start and the end, we measure it twice,
# and we only do the assert if the day hasn't changed
day0 = datetime.datetime.fromtimestamp(time.time(), pytz.utc).date()
# Run a query!
qhost = GOOD_TIME
qhe.queryhost(qhost)
qdate = day0.isoformat()
windowdays = 30
# Check if host is well behaved
windowstart = (day0 - datetime.timedelta(days=windowdays)).isoformat()
check_between_cmd = "SELECT SUM(wtcount) FROM dated WHERE host=%s and qdate BETWEEN %s and %s"
# if wtcount is 0 (the host is well behaved)
if not mdb.select1(check_between_cmd, (qhost, windowstart, qdate))[0]:
qlast_query = mdb.select1("SELECT qlast FROM wbhosts WHERE host=%s",
(qhost, ))
qlast = None if qlast_query == None else qlast_query[0]
# case: host was not well behaved before, now is well behaved
if not qlast:
mdb.execute(
"INSERT IGNORE INTO wbhosts (host, qlast) VALUES (%s,%s)",
(qhost, qdate))
elif qlast == qdate:
print("***Queried today***")
# case: host was not queried today
else:
mdb.execute("UPDATE wbhosts SET qlast=%s WHERE host=%s",
(qdate, qhost))
(id, ipaddr, qdate) = mdb.select1(
"select id,ipaddr,qdate from dated where host=%s order by id desc limit 1",
(GOOD_TIME, ))
if wb:
qlast = mdb.select1("SELECT qlast FROM wbhosts WHERE host=%s",
(GOOD_TIME, ))[0]
assert qlast.isoformat() == day0.isoformat()
day1 = datetime.datetime.fromtimestamp(time.time(), pytz.utc).date()
assert id > 0 # make sure id is good
assert ipaddr > '' #
assert qdate in [day0, day1
] # the day must be when we started or when we stopped
# Now make sure we got both a http and an https value
(id, ipaddr, http) = mdb.select1(
"select id,ipaddr,https from dated where host=%s and https=0 order by id desc limit 1",
(GOOD_TIME, ))
assert id > 0
(id, ipaddr, http) = mdb.select1(
"select id,ipaddr,https from dated where host=%s and https=1 order by id desc limit 1",
(GOOD_TIME, ))
assert id > 0
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--debug",action="store_true",help="write results to STDOUT")
parser.add_argument("--verbose",action="store_true",help="output to STDOUT")
parser.add_argument("--config",help="config file",required=True)
parser.add_argument("--host",help="Specify a host; just do the report for that one host")
parser.add_argument("--usg",action="store_true",help="Only USG")
parser.add_argument("--outdir",help="Where to put the output.",default='plots')
parser.add_argument("--nosizes",help="Do not report the size of the tables",action='store_true')
args = parser.parse_args()
config = db.get_mysql_config(args.config)
dbc = db.mysql(config)
if args.debug:
dbc.debug = args.debug
print("debug mode")
dbc.execute("set innodb_lock_wait_timeout=20")
dbc.execute("set tx_isolation='READ-COMMITTED'")
dbc.execute("set time_zone = '+00:00'")
print("Starting at {}".format(time.asctime()))
t0 = time.time()
page_by_host(dbc, args.outdir)
#page_by_ip(dbc, os.path.join(args.outdir,'ipplots'), args.outdir)
t1 = time.time()
print("Took {:.2} seconds".format(t1-t0))
#!/usr/bin/env python
#-*-coding:utf8-*-
import db
#with open('../userdata','r') as f:
# for line in f:
# username = line.split()[0]
# password = line.split()[1]
# sqli = "INSERT INTO user VALUES('%s','%s')"%(username,password)
# db.mysql(sqli)g
with open('../bookdata', 'r') as f:
for line in f:
bookname = line.split()[0]
nums = line.split()[1]
sqli = "INSERT INTO bookinfo VALUES('%s','%s','')" % (bookname, nums)
db.mysql(sqli)
#!/usr/bin/env python
#-*-coding:utf8-*-
import db
#with open('../userdata','r') as f:
# for line in f:
# username = line.split()[0]
# password = line.split()[1]
# sqli = "INSERT INTO user VALUES('%s','%s')"%(username,password)
# db.mysql(sqli)g
with open('../bookdata','r') as f:
for line in f:
bookname = line.split()[0]
nums = line.split()[1]
sqli = "INSERT INTO bookinfo VALUES('%s','%s','')"%(bookname,nums)
db.mysql(sqli)
try:
fd = getlock(__file__)
logger_info('{} PID {} config={} acquired lock'.format(
__file__, os.getpid(), args.config))
except RuntimeError as e:
logger_info('{} PID {} config={} could not acquire lock'.format(
__file__, os.getpid(), args.config))
print("{}: Could not acquire lock".format(__file__))
exit(0)
# Make sure mySQL works. We do this here so that we don't report
# that we can't connect to MySQL after the loop starts. We cache
# the results in w to avoid reundent connections to the MySQL
# server.
d = db.mysql(db.get_mysql_config(args.config))
d.connect()
ver = d.select1("select version();")[0]
assert type(ver) == str and ver[0] >= '5'
d.close()
t0 = time.time()
res = subprocess.call(
[sys.executable, 'webtime.py', '--config', args.config])
t1 = time.time()
took = t1 - t0
# TODO: Log in the database our start and end time
logger_info('{} PID {} Completed. res={} took={:8.2f} seconds'.format(
__file__, os.getpid(), res, took))
help="write results to STDOUT")
parser.add_argument("--verbose",
action="store_true",
help="output to STDOUT")
parser.add_argument("--config", help="config file", required=True)
parser.add_argument("--host", help="Specify a host")
parser.add_argument(
"report",
nargs="*",
help=
"Specify which reports are requested. options: counts hosts size offset all"
)
args = parser.parse_args()
config = db.get_mysql_config(args.config)
dbc = db.mysql(config, mode='ro')
if args.debug:
dbc.debug = args.debug
print("debug mode")
print("Report generated: {}".format(datetime.datetime.now().isoformat()))
if not args.report:
print("No report specified; running counts")
args.report = ['counts']
if 'counts' in args.report:
cmd = "select min(qdate),max(qdate) from dated "
(date_min, date_max) = dbc.select1(cmd)
# Running from cron. Make sure only one of us is running. If another is running, exit
try:
fd = getlock(__file__)
logger_info('{} PID {} config={} acquired lock'.format(__file__,os.getpid(),args.config))
except RuntimeError as e:
logger_info('{} PID {} config={} could not acquire lock'.format(__file__,os.getpid(),args.config))
print("{}: Could not acquire lock".format(__file__))
exit(0)
# Make sure mySQL works. We do this here so that we don't report
# that we can't connect to MySQL after the loop starts. We cache
# the results in w to avoid reundent connections to the MySQL
# server.
d = db.mysql( db.get_mysql_config(args.config) )
d.connect()
ver = d.select1("select version();")[0]
assert type(ver)==str and ver[0]>='5'
d.close()
t0 = time.time()
res = subprocess.call([sys.executable,'webtime.py','--config',args.config])
t1 = time.time()
took = t1-t0
# TODO: Log in the database our start and end time
logger_info('{} PID {} Completed. res={} took={:8.2f} seconds'.format(__file__,os.getpid(),res,took))
# finally, release our lock, so we can catch it again