def run(args):
spec = {
'-v': '',
'-h': '=+',
'-p': '=+',
'-c': '=+',
'--help': '',
'--host': '-h',
'--project': '-p',
'--context': '-c',
'--verbose': '-v', # arg is an alias for some other arg
'-help': '--help',
}
opt, args = easyargs.get(spec, args)
if opt['--help']:
print(__doc__)
return
verbose = opt['-v']
try:
test_run_type = args[0]
test_run = args[1]
except:
print("\ncan't get args, try:\n pdk check_expected --help\n")
return 1
# normalize the test run, so they can say stuff like "daily_latest"
test_run = common.find_test_run(test_run)
print("TYPE %s" % test_run_type)
print("test_run %s" % test_run)
if test_run.endswith('latest'):
print("this test run name is probably a mistake")
return 1
# construct the query for the set of tests that we are expecting
select_args = [('test_run_type', test_run_type)]
if '-h' in opt:
select_args.append(('host', opt['-h']))
if '-p' in opt:
select_args.append(('project', opt['-p']))
if '-c' in opt:
select_args.append(('context', opt['-c']))
where_text, where_dict = pdk_db.where_dict(select_args)
s = "SELECT project, host, test_name, context FROM expected %s " % where_text
if verbose > 1:
print(s)
print(where_dict)
# perform the query
c = pdk_db.execute(s, where_dict)
if verbose > 1:
print("query done")
# check each test reported in the query result
detected = 0
for (project, host, test_name, context) in c:
if verbose > 2:
print("CHECK %s %s %s" % (project, host, test_name))
c1 = pdk_db.execute("""SELECT status FROM result_scalar
WHERE test_run = :1 AND project = :2 AND host = :3 AND
test_name = :4 AND context = :5 """,
(test_run, project, host, test_name, context)
)
if c1.fetchone() is None:
# it wasn't there
if verbose:
print(" MISSING: %s %s %s" % (project, host, test_name))
pdk_db.execute(
"""INSERT INTO result_scalar
( test_run, project, host, context, test_name, status, attn )
VALUES ( :1, :2, :3, :4, :5, :6, :7 )""",
(test_run,
project,
host,
context,
test_name,
'M',
'Y'))
detected = detected + 1
# do some commits from time to time to avoid lock timeouts
if detected % 10000 == 0:
pdk_db.commit()
# must commit after db updates
pdk_db.commit()
print("detected %d" % detected)
def run(args):
spec = {
'-v': '',
'-h': '=+',
'-p': '=+',
'-c': '=+',
'--help': '',
'--host': '-h',
'--project': '-p',
'--context': '-c',
'--verbose': '-v', # arg is an alias for some other arg
'-help': '--help',
}
opt, args = easyargs.get(spec, args)
if opt['--help']:
print __doc__
return
verbose = opt['-v']
try:
test_run_type = args[0]
test_run = args[1]
except:
print "\ncan't get args, try:\n pdk check_expected --help\n"
return 1
# normalize the test run, so they can say stuff like "daily_latest"
test_run = common.find_test_run(test_run)
print "TYPE ", test_run_type
print "test_run", test_run
if test_run.endswith('latest'):
print "this test run name is probably a mistake"
return 1
# construct the query for the set of tests that we are expecting
select_args = [('test_run_type', test_run_type)]
if '-h' in opt:
select_args.append(('host', opt['-h']))
if '-p' in opt:
select_args.append(('project', opt['-p']))
if '-c' in opt:
select_args.append(('context', opt['-c']))
where_text, where_dict = pdk_db.where_dict(select_args)
s = "SELECT project, host, test_name, context FROM expected %s " % where_text
if verbose > 1:
print s
print where_dict
# perform the query
c = pdk_db.execute(s, where_dict)
if verbose > 1:
print "query done"
# check each test reported in the query result
detected = 0
for (project, host, test_name, context) in c:
if verbose > 2:
print "CHECK", project, host, test_name
c1 = pdk_db.execute(
"""SELECT status FROM result_scalar
WHERE test_run = :1 AND project = :2 AND host = :3 AND
test_name = :4 AND context = :5 """,
(test_run, project, host, test_name, context))
if c1.fetchone() is None:
# it wasn't there
if verbose:
print " MISSING:", project, host, test_name
pdk_db.execute(
"""INSERT INTO result_scalar
( test_run, project, host, context, test_name, status, attn )
VALUES ( :1, :2, :3, :4, :5, :6, :7 )""",
(test_run, project, host, context, test_name, 'M', 'Y'))
detected = detected + 1
# do some commits from time to time to avoid lock timeouts
if detected % 10000 == 0:
pdk_db.commit()
# must commit after db updates
pdk_db.commit()
print "detected ", detected
def delete(args):
'''pdk delete -field value -otherfield othervalue ...
deletes records from the database where all the parameters match
This command can remove a subset of the records for a test run.
It can also remove all the records for a test run, but leave
the test run in the index as an empty dataset.
The count of records in a test run will be incorrect until
you update it.
pdk delete -test_run foo -project bar -context baz -host xyzzy -custom yzzyx
also accepts:
-wild allow wildcards; some queries are dramatically
inefficient when you use wildcards, so you have to
actively ask to use them. if you use * or ?, you
must specify -wild
-count do not delete the records - just show how many are
affected
'''
import pandokia.helpers.easyargs as easyargs
import pandokia.common as common
opt, args = easyargs.get(
{
'-test_run': '=+',
'-project': '=+',
'-context': '=+',
'-custom': '=+',
'-host': '=+',
'-status': '=+',
'-wild': '',
'-count': '',
'-c': '',
}, args)
dont = 0
if len(args) != 0:
if '-test_run' not in opt:
opt['-test_run'] = args
else:
print("error: -test_run and non-option args used together")
wild = opt['-wild']
del opt['-wild']
count = opt['-count'] | opt['-c']
del opt['-count']
del opt['-c']
if '-test_run' not in opt:
print("You really have to specify -test_run")
return 1
# expand wild cards in test run
lll = []
for x in opt['-test_run']:
lll = lll + common.expand_test_run(x)
opt['-test_run'] = lll
#
if not count:
lll = []
for x in opt['-test_run']:
if check_valuable(x):
print("test run %s is marked valuable - cannot delete" % x)
else:
lll.append(x)
#
opt['-test_run'] = lll
if len(lll) == 0:
# print "No deleteable test runs found"
dont = 1
lll = []
for x in sorted(opt.keys()):
if x == '-test_run':
continue
v = opt[x]
lll.append((x[1:], v))
if not wild:
if ('*' in v) or ('?' in v) or ('%' in v):
print('\nError: must use -wild for %s %s\n' % (x, v))
dont = 1
if dont:
return
for x in opt['-test_run']:
where_str, where_dict = pdk_db.where_dict(lll + [('test_run', x)])
if count:
c = pdk_db.execute(
"SELECT count(*) FROM result_scalar %s" % (where_str, ),
where_dict)
print("%s %s" % (x, c.fetchone()[0]))
else:
print(x)
delete_by_query(where_str, where_dict)
recount([x], verbose=0)
return 0
def main() :
global xnodes
global no_run
# read all the input files
if readline :
history = os.path.join(os.path.expanduser("~"), ".steuermann_history")
try :
readline.read_history_file(history)
except IOError :
pass
import atexit
atexit.register(readline.write_history_file, history)
# easyargs spec definition:
#
# '-v' : '', # arg takes no parameter, opt['-v'] is
# # how many times it occurred
# '-f' : '=', # arg takes a parameter
# '-mf' : '=+', # arg takes a parameter, may be specified
# # several times to get a list
# '--verbose' : '-v', # arg is an alias for some other arg
opt, args = easyargs.get(allowed_flags, allow_unexpected = True)
do_all = opt['-a']
no_run = opt['-n']
'''
NOTE: moving this functionality into nodes.py
# find any unknown arguments like --something=whatever, set as conditions
arguments = sys.argv[1:]
for a in arguments:
if '--' in a and '=' in a:
not_allowed_flag = True
for f in allowed_flags.keys():
if a.startswith(f):
not_allowed_flag = False
break
if not_allowed_flag:
a = a.lstrip('--')
k, v = a.split('=')
nodes.saved_conditions[k] = eval(v)
'''
sm_files = [os.path.abspath(a) for a in args if ('--' not in a and '=' not in a)]
di_nodes = nodes.read_file_list( sm_files )
xnodes = di_nodes.node_index
# get run name
if '-r' in opt :
run_name = opt['-r']
else :
run_name = "user_%s_%s"%(username,str(datetime.datetime.now()).replace(' ','_'))
# get hosts (*.ini) file name
if '-h' in opt :
hosts_ini = opt['-h']
else :
hosts_ini = os.path.join(os.path.dirname(__file__), 'hosts.ini')
# Use a user-defined config if it exists
if os.path.exists(config.hosts_config):
hosts_ini = config.hosts_config
# parse common resources from hosts INI file
get_common_resources(hosts_ini)
db = config.open_db()
if do_all:
run_all(xnodes, run_name, hosts_ini, db)
else :
run_interactive( xnodes, run_name, hosts_ini, db )
def run(args):
argspec = {
'-r': '=+',
'-s': '=',
'--help': '',
'--test_run': '-r',
'--subject': '-s',
}
opt, args = easyargs.get(argspec, args)
if opt['--help']:
print("""
-r
--test_run
specify a list of test runs to report on
-s
--subject
specify a subject for the email
other parameters are users to send email to (not email addresses)
""")
return 0
if '-r' in opt:
test_runs = [pandokia.common.find_test_run(x) for x in opt['-r']]
else:
test_runs = [pandokia.common.find_test_run("daily_latest")]
if args:
# for each user name, look it up the email address in the table
users = []
for user in args:
found = 0
c = pdk_db.execute(
"SELECT email FROM user_prefs WHERE username = :1", (user, ))
for email, in c:
users.append((user, email))
found = 1
if not found:
print("No email address known for user %s" % user)
else:
# get a list of all the (user, email) from the user prefs
query = """SELECT username, email FROM user_prefs"""
user_res = pdk_db.execute(query)
users = [(user, email) for user, email in user_res]
if '-s' in args:
subject = args['-s']
else:
subject = 'Test Results: %s' % ', '.join([x for x in test_runs])
# compute the email to send to each user; send it.
for user, email in users:
if email is None:
continue
msg = ''
for x in test_runs:
newmsg = create_email(user, x)
if newmsg is not None:
msg = msg + newmsg
if len(msg) > 0 and email:
sendmail(email, subject,
'%s\n\n\nThis report created for %s' % (msg, email))
else:
print("suppress blank email to %s" % email)
def run(args):
opts, args = easyargs.get(
{
'-c': 'list',
'-p': 'list',
'-h': 'list',
'--context': '-c',
'--project': '-p',
'--host': '-h',
}, args)
try:
test_run_type = args[0]
test_run_pattern = args[1]
except:
print("can't get args")
print(" pdk gen_expected test_run_type test_run_pattern")
sys.exit(1)
test_run_pattern = common.find_test_run(test_run_pattern)
print("test_run_pattern = %s" % test_run_pattern)
l = [('test_run', test_run_pattern)]
if '-c' in opts:
l = l + [('context', x) for x in opts['-c']]
if '-p' in opts:
l = l + [('project', x) for x in opts['-p']]
if '-h' in opts:
l = l + [('host', x) for x in opts['-h']]
if debug:
for x in l:
print(" %s" % x)
print("?")
sys.stdin.readline()
where_str, where_dict = pdk_db.where_dict(l)
sql = "select distinct project, host, context, test_name from result_scalar %s " % where_str
c = pdk_db.execute(sql, where_dict)
for (project, host, context, test_name) in c:
if test_name.endswith("nose.failure.Failure.runTest"):
# Sometimes nose generates this test name. I don't want it in the database at all, because
# the name is not unique, and the record does not contain any useful information about the problem.
# In any case, we never want to list this test as "expected", even
# if it leaks into the database.
continue
if debug:
print("expect %s %s %s %s %s" %
(test_run_type, project, host, context, test_name))
# insert to the expected table; if the record is already there, it's
# ok.
try:
pdk_db.execute(
'insert into expected ( test_run_type, project, host, context, test_name ) values ( :1, :2, :3, :4, :5 )',
(test_run_type, project, host, context, test_name))
except pdk_db.IntegrityError as e:
if debug:
print("exception %s" % e)
pass
pdk_db.commit()
def run(args):
argspec = {
'-r': '=+',
'-s': '=',
'--help': '',
'--test_run': '-r',
'--subject': '-s',
}
opt, args = easyargs.get(argspec, args)
if opt['--help']:
print("""
-r
--test_run
specify a list of test runs to report on
-s
--subject
specify a subject for the email
other parameters are users to send email to (not email addresses)
""")
return 0
if '-r' in opt:
test_runs = [pandokia.common.find_test_run(x) for x in opt['-r']]
else:
test_runs = [pandokia.common.find_test_run("daily_latest")]
if args:
# for each user name, look it up the email address in the table
users = []
for user in args:
found = 0
c = pdk_db.execute(
"SELECT email FROM user_prefs WHERE username = :1", (user,))
for email, in c:
users.append((user, email))
found = 1
if not found:
print("No email address known for user %s" % user)
else:
# get a list of all the (user, email) from the user prefs
query = """SELECT username, email FROM user_prefs"""
user_res = pdk_db.execute(query)
users = [(user, email) for user, email in user_res]
if '-s' in args:
subject = args['-s']
else:
subject = 'Test Results: %s' % ', '.join([x for x in test_runs])
# compute the email to send to each user; send it.
for user, email in users:
if email is None:
continue
msg = ''
for x in test_runs:
newmsg = create_email(user, x)
if newmsg is not None:
msg = msg + newmsg
if len(msg) > 0 and email:
sendmail(
email, subject, '%s\n\n\nThis report created for %s' %
(msg, email))
else:
print("suppress blank email to %s" % email)
def main():
global xnodes
global no_run
# read all the input files
if readline:
history = os.path.join(os.path.expanduser("~"), ".steuermann_history")
try:
readline.read_history_file(history)
except IOError:
pass
import atexit
atexit.register(readline.write_history_file, history)
# easyargs spec definition:
#
# '-v' : '', # arg takes no parameter, opt['-v'] is
# # how many times it occurred
# '-f' : '=', # arg takes a parameter
# '-mf' : '=+', # arg takes a parameter, may be specified
# # several times to get a list
# '--verbose' : '-v', # arg is an alias for some other arg
opt, args = easyargs.get(allowed_flags, allow_unexpected=True)
do_all = opt['-a']
no_run = opt['-n']
'''
NOTE: moving this functionality into nodes.py
# find any unknown arguments like --something=whatever, set as conditions
arguments = sys.argv[1:]
for a in arguments:
if '--' in a and '=' in a:
not_allowed_flag = True
for f in allowed_flags.keys():
if a.startswith(f):
not_allowed_flag = False
break
if not_allowed_flag:
a = a.lstrip('--')
k, v = a.split('=')
nodes.saved_conditions[k] = eval(v)
'''
sm_files = [
os.path.abspath(a) for a in args if ('--' not in a and '=' not in a)
]
di_nodes = nodes.read_file_list(sm_files)
xnodes = di_nodes.node_index
# get run name
if '-r' in opt:
run_name = opt['-r']
else:
run_name = "user_%s_%s" % (username, str(
datetime.datetime.now()).replace(' ', '_'))
# get hosts (*.ini) file name
if '-h' in opt:
hosts_ini = opt['-h']
else:
hosts_ini = os.path.join(os.path.dirname(__file__), 'hosts.ini')
# Use a user-defined config if it exists
if os.path.exists(config.hosts_config):
hosts_ini = config.hosts_config
# parse common resources from hosts INI file
get_common_resources(hosts_ini)
db = config.open_db()
if do_all:
run_all(xnodes, run_name, hosts_ini, db)
else:
run_interactive(xnodes, run_name, hosts_ini, db)
try:
f = urllib2.urlopen(req)
except urllib2.HTTPError as e:
print("HTTP ERROR", e.code)
print(e.read())
return 1
print(f.read())
f.close()
return 0
if __name__ == '__main__':
opt, args = easyargs.get({
'-p': '=', # password
'-f': '=', # password from file
'-u': '', # upload
'-d': '=', # upload destination directory (default .)
'-h': '=', # host name
})
if '-p' in opt:
password = opt['-p']
elif '-f' in opt:
password = open(opt['-f'], 'r').readline().strip()
if not ('-h' in opt):
print("must give host name with -h")
sys.exit(2)
host = opt['-h']
if '-d' in opt:
directory = opt['-d']
else:
def delete(args):
'''pdk delete -field value -otherfield othervalue ...
deletes records from the database where all the parameters match
This command can remove a subset of the records for a test run.
It can also remove all the records for a test run, but leave
the test run in the index as an empty dataset.
The count of records in a test run will be incorrect until
you update it.
pdk delete -test_run foo -project bar -context baz -host xyzzy
also accepts:
-wild allow wildcards; some queries are dramatically
inefficient when you use wildcards, so you have to
actively ask to use them. if you use * or ?, you
must specify -wild
-count do not delete the records - just show how many are
affected
'''
import pandokia.helpers.easyargs as easyargs
import pandokia.common as common
opt, args = easyargs.get(
{
'-test_run': '=+',
'-project': '=+',
'-context': '=+',
'-host': '=+',
'-status': '=+',
'-wild': '',
'-count': '',
'-c': '',
},
args
)
dont = 0
if len(args) != 0:
if '-test_run' not in opt:
opt['-test_run'] = args
else:
print("error: -test_run and non-option args used together")
wild = opt['-wild']
del opt['-wild']
count = opt['-count'] | opt['-c']
del opt['-count']
del opt['-c']
if '-test_run' not in opt:
print("You really have to specify -test_run")
return 1
# expand wild cards in test run
lll = []
for x in opt['-test_run']:
lll = lll + common.expand_test_run(x)
opt['-test_run'] = lll
#
if not count:
lll = []
for x in opt['-test_run']:
if check_valuable(x):
print("test run %s is marked valuable - cannot delete" % x)
else:
lll.append(x)
#
opt['-test_run'] = lll
if len(lll) == 0:
# print "No deleteable test runs found"
dont = 1
lll = []
for x in sorted(opt.keys()):
if x == '-test_run':
continue
v = opt[x]
lll.append((x[1:], v))
if not wild:
if ('*' in v) or ('?' in v) or ('%' in v):
print('\nError: must use -wild for %s %s\n' % (x, v))
dont = 1
if dont:
return
for x in opt['-test_run']:
where_str, where_dict = pdk_db.where_dict(lll + [('test_run', x)])
if count:
c = pdk_db.execute(
"SELECT count(*) FROM result_scalar %s" %
(where_str,), where_dict)
print("%s %s" % (x, c.fetchone()[0]))
else:
print(x)
delete_by_query(where_str, where_dict)
recount([x], verbose=0)
return 0
def run(args):
opts, args = easyargs.get({
'-c': 'list',
'-p': 'list',
'-h': 'list',
'--context': '-c',
'--project': '-p',
'--host': '-h',
}, args)
try:
test_run_type = args[0]
test_run_pattern = args[1]
except:
print("can't get args")
print(" pdk gen_expected test_run_type test_run_pattern")
sys.exit(1)
test_run_pattern = common.find_test_run(test_run_pattern)
print("test_run_pattern = %s" % test_run_pattern)
l = [('test_run', test_run_pattern)]
if '-c' in opts:
l = l + [('context', x) for x in opts['-c']]
if '-p' in opts:
l = l + [('project', x) for x in opts['-p']]
if '-h' in opts:
l = l + [('host', x) for x in opts['-h']]
if debug:
for x in l:
print(" %s" % x)
print("?")
sys.stdin.readline()
where_str, where_dict = pdk_db.where_dict(l)
sql = "select distinct project, host, context, test_name from result_scalar %s " % where_str
c = pdk_db.execute(sql, where_dict)
for (project, host, context, test_name) in c:
if test_name.endswith("nose.failure.Failure.runTest"):
# Sometimes nose generates this test name. I don't want it in the database at all, because
# the name is not unique, and the record does not contain any useful information about the problem.
# In any case, we never want to list this test as "expected", even
# if it leaks into the database.
continue
if debug:
print(
"expect %s %s %s %s %s" %
(test_run_type, project, host, context, test_name))
# check if the row already exists before insert since the expected table does not have unique enforcement
# on its columns
a = pdk_db.execute('select * from expected where test_run_type = :1 and project = :2 and host = :3 and context = :4 and test_name = :5', (test_run_type, project, host, context, test_name))
y = a.fetchone()
if y is None:
try:
pdk_db.execute(
'insert into expected ( test_run_type, project, host, context, test_name ) values ( :1, :2, :3, :4, :5 )',
(test_run_type,
project,
host,
context,
test_name))
except Exception as e:
if debug:
print("exception %s" % e)
pass
pdk_db.commit()
def run(args):
opts, args = easyargs.get(
{
'-c': 'list',
'-p': 'list',
'-h': 'list',
'-m': 'list',
'--context': '-c',
'--project': '-p',
'--host': '-h',
'--custom': '-m',
}, args)
try:
test_run_type = args[0]
test_run_pattern = args[1]
except:
print("can't get args")
print(" pdk gen_expected test_run_type test_run_pattern")
sys.exit(1)
test_run_pattern = common.find_test_run(test_run_pattern)
print("test_run_pattern = ", test_run_pattern)
l = [('test_run', test_run_pattern)]
if '-c' in opts:
l = l + [('context', x) for x in opts['-c']]
if '-p' in opts:
l = l + [('project', x) for x in opts['-p']]
if '-h' in opts:
l = l + [('host', x) for x in opts['-h']]
if '-m' in opts:
l = l + [('custom', x) for x in opts['-m']]
if debug:
for x in l:
print(" ", x)
print("?")
sys.stdin.readline()
where_str, where_dict = pdk_db.where_dict(l)
sql = "select distinct project, host, context, custom, test_name from result_scalar %s " % where_str
c = pdk_db.execute(sql, where_dict)
for (project, host, context, custom, test_name) in c:
if test_name.endswith("nose.failure.Failure.runTest"):
# Sometimes nose generates this test name. I don't want it in the database at all, because
# the name is not unique, and the record does not contain any useful information about the problem.
# In any case, we never want to list this test as "expected", even if it leaks into the database.
continue
if debug:
print("expect ", test_run_type, project, host, context, custom,
test_name)
a = pdk_db.execute(
'select * from expected where test_run_type = :1 and project = :2 and host = :3 and context = :4 and custom = :5 and test_name = :6',
(test_run_type, project, host, context, custom, test_name))
y = a.fetchone()
if y is None:
try:
pdk_db.execute(
'insert into expected ( test_run_type, project, host, context, custom, test_name ) values ( :1, :2, :3, :4, :5, :6 )',
(test_run_type, project, host, context, custom, test_name))
except Exception as e:
if debug:
print("exception", e)
pass
pdk_db.commit()
