def run(args): spec = { '-v': '', '-h': '=+', '-p': '=+', '-c': '=+', '--help': '', '--host': '-h', '--project': '-p', '--context': '-c', '--verbose': '-v', # arg is an alias for some other arg '-help': '--help', } opt, args = easyargs.get(spec, args) if opt['--help']: print(__doc__) return verbose = opt['-v'] try: test_run_type = args[0] test_run = args[1] except: print("\ncan't get args, try:\n pdk check_expected --help\n") return 1 # normalize the test run, so they can say stuff like "daily_latest" test_run = common.find_test_run(test_run) print("TYPE %s" % test_run_type) print("test_run %s" % test_run) if test_run.endswith('latest'): print("this test run name is probably a mistake") return 1 # construct the query for the set of tests that we are expecting select_args = [('test_run_type', test_run_type)] if '-h' in opt: select_args.append(('host', opt['-h'])) if '-p' in opt: select_args.append(('project', opt['-p'])) if '-c' in opt: select_args.append(('context', opt['-c'])) where_text, where_dict = pdk_db.where_dict(select_args) s = "SELECT project, host, test_name, context FROM expected %s " % where_text if verbose > 1: print(s) print(where_dict) # perform the query c = pdk_db.execute(s, where_dict) if verbose > 1: print("query done") # check each test reported in the query result detected = 0 for (project, host, test_name, context) in c: if verbose > 2: print("CHECK %s %s %s" % (project, host, test_name)) c1 = pdk_db.execute("""SELECT status FROM result_scalar WHERE test_run = :1 AND project = :2 AND host = :3 AND test_name = :4 AND context = :5 """, (test_run, project, host, test_name, context) ) if c1.fetchone() is None: # it wasn't there if verbose: print(" MISSING: %s %s %s" % (project, host, test_name)) pdk_db.execute( """INSERT INTO result_scalar ( test_run, project, host, context, test_name, status, attn ) VALUES ( :1, :2, :3, :4, :5, :6, :7 )""", (test_run, project, host, context, test_name, 'M', 'Y')) detected = detected + 1 # do some commits from time to time to avoid lock timeouts if detected % 10000 == 0: pdk_db.commit() # must commit after db updates pdk_db.commit() print("detected %d" % detected)
def run(args): spec = { '-v': '', '-h': '=+', '-p': '=+', '-c': '=+', '--help': '', '--host': '-h', '--project': '-p', '--context': '-c', '--verbose': '-v', # arg is an alias for some other arg '-help': '--help', } opt, args = easyargs.get(spec, args) if opt['--help']: print __doc__ return verbose = opt['-v'] try: test_run_type = args[0] test_run = args[1] except: print "\ncan't get args, try:\n pdk check_expected --help\n" return 1 # normalize the test run, so they can say stuff like "daily_latest" test_run = common.find_test_run(test_run) print "TYPE ", test_run_type print "test_run", test_run if test_run.endswith('latest'): print "this test run name is probably a mistake" return 1 # construct the query for the set of tests that we are expecting select_args = [('test_run_type', test_run_type)] if '-h' in opt: select_args.append(('host', opt['-h'])) if '-p' in opt: select_args.append(('project', opt['-p'])) if '-c' in opt: select_args.append(('context', opt['-c'])) where_text, where_dict = pdk_db.where_dict(select_args) s = "SELECT project, host, test_name, context FROM expected %s " % where_text if verbose > 1: print s print where_dict # perform the query c = pdk_db.execute(s, where_dict) if verbose > 1: print "query done" # check each test reported in the query result detected = 0 for (project, host, test_name, context) in c: if verbose > 2: print "CHECK", project, host, test_name c1 = pdk_db.execute( """SELECT status FROM result_scalar WHERE test_run = :1 AND project = :2 AND host = :3 AND test_name = :4 AND context = :5 """, (test_run, project, host, test_name, context)) if c1.fetchone() is None: # it wasn't there if verbose: print " MISSING:", project, host, test_name pdk_db.execute( """INSERT INTO result_scalar ( test_run, project, host, context, test_name, status, attn ) VALUES ( :1, :2, :3, :4, :5, :6, :7 )""", (test_run, project, host, context, test_name, 'M', 'Y')) detected = detected + 1 # do some commits from time to time to avoid lock timeouts if detected % 10000 == 0: pdk_db.commit() # must commit after db updates pdk_db.commit() print "detected ", detected
def delete(args): '''pdk delete -field value -otherfield othervalue ... deletes records from the database where all the parameters match This command can remove a subset of the records for a test run. It can also remove all the records for a test run, but leave the test run in the index as an empty dataset. The count of records in a test run will be incorrect until you update it. pdk delete -test_run foo -project bar -context baz -host xyzzy -custom yzzyx also accepts: -wild allow wildcards; some queries are dramatically inefficient when you use wildcards, so you have to actively ask to use them. if you use * or ?, you must specify -wild -count do not delete the records - just show how many are affected ''' import pandokia.helpers.easyargs as easyargs import pandokia.common as common opt, args = easyargs.get( { '-test_run': '=+', '-project': '=+', '-context': '=+', '-custom': '=+', '-host': '=+', '-status': '=+', '-wild': '', '-count': '', '-c': '', }, args) dont = 0 if len(args) != 0: if '-test_run' not in opt: opt['-test_run'] = args else: print("error: -test_run and non-option args used together") wild = opt['-wild'] del opt['-wild'] count = opt['-count'] | opt['-c'] del opt['-count'] del opt['-c'] if '-test_run' not in opt: print("You really have to specify -test_run") return 1 # expand wild cards in test run lll = [] for x in opt['-test_run']: lll = lll + common.expand_test_run(x) opt['-test_run'] = lll # if not count: lll = [] for x in opt['-test_run']: if check_valuable(x): print("test run %s is marked valuable - cannot delete" % x) else: lll.append(x) # opt['-test_run'] = lll if len(lll) == 0: # print "No deleteable test runs found" dont = 1 lll = [] for x in sorted(opt.keys()): if x == '-test_run': continue v = opt[x] lll.append((x[1:], v)) if not wild: if ('*' in v) or ('?' in v) or ('%' in v): print('\nError: must use -wild for %s %s\n' % (x, v)) dont = 1 if dont: return for x in opt['-test_run']: where_str, where_dict = pdk_db.where_dict(lll + [('test_run', x)]) if count: c = pdk_db.execute( "SELECT count(*) FROM result_scalar %s" % (where_str, ), where_dict) print("%s %s" % (x, c.fetchone()[0])) else: print(x) delete_by_query(where_str, where_dict) recount([x], verbose=0) return 0
def main() : global xnodes global no_run # read all the input files if readline : history = os.path.join(os.path.expanduser("~"), ".steuermann_history") try : readline.read_history_file(history) except IOError : pass import atexit atexit.register(readline.write_history_file, history) # easyargs spec definition: # # '-v' : '', # arg takes no parameter, opt['-v'] is # # how many times it occurred # '-f' : '=', # arg takes a parameter # '-mf' : '=+', # arg takes a parameter, may be specified # # several times to get a list # '--verbose' : '-v', # arg is an alias for some other arg opt, args = easyargs.get(allowed_flags, allow_unexpected = True) do_all = opt['-a'] no_run = opt['-n'] ''' NOTE: moving this functionality into nodes.py # find any unknown arguments like --something=whatever, set as conditions arguments = sys.argv[1:] for a in arguments: if '--' in a and '=' in a: not_allowed_flag = True for f in allowed_flags.keys(): if a.startswith(f): not_allowed_flag = False break if not_allowed_flag: a = a.lstrip('--') k, v = a.split('=') nodes.saved_conditions[k] = eval(v) ''' sm_files = [os.path.abspath(a) for a in args if ('--' not in a and '=' not in a)] di_nodes = nodes.read_file_list( sm_files ) xnodes = di_nodes.node_index # get run name if '-r' in opt : run_name = opt['-r'] else : run_name = "user_%s_%s"%(username,str(datetime.datetime.now()).replace(' ','_')) # get hosts (*.ini) file name if '-h' in opt : hosts_ini = opt['-h'] else : hosts_ini = os.path.join(os.path.dirname(__file__), 'hosts.ini') # Use a user-defined config if it exists if os.path.exists(config.hosts_config): hosts_ini = config.hosts_config # parse common resources from hosts INI file get_common_resources(hosts_ini) db = config.open_db() if do_all: run_all(xnodes, run_name, hosts_ini, db) else : run_interactive( xnodes, run_name, hosts_ini, db )
def run(args): argspec = { '-r': '=+', '-s': '=', '--help': '', '--test_run': '-r', '--subject': '-s', } opt, args = easyargs.get(argspec, args) if opt['--help']: print(""" -r --test_run specify a list of test runs to report on -s --subject specify a subject for the email other parameters are users to send email to (not email addresses) """) return 0 if '-r' in opt: test_runs = [pandokia.common.find_test_run(x) for x in opt['-r']] else: test_runs = [pandokia.common.find_test_run("daily_latest")] if args: # for each user name, look it up the email address in the table users = [] for user in args: found = 0 c = pdk_db.execute( "SELECT email FROM user_prefs WHERE username = :1", (user, )) for email, in c: users.append((user, email)) found = 1 if not found: print("No email address known for user %s" % user) else: # get a list of all the (user, email) from the user prefs query = """SELECT username, email FROM user_prefs""" user_res = pdk_db.execute(query) users = [(user, email) for user, email in user_res] if '-s' in args: subject = args['-s'] else: subject = 'Test Results: %s' % ', '.join([x for x in test_runs]) # compute the email to send to each user; send it. for user, email in users: if email is None: continue msg = '' for x in test_runs: newmsg = create_email(user, x) if newmsg is not None: msg = msg + newmsg if len(msg) > 0 and email: sendmail(email, subject, '%s\n\n\nThis report created for %s' % (msg, email)) else: print("suppress blank email to %s" % email)
def run(args): opts, args = easyargs.get( { '-c': 'list', '-p': 'list', '-h': 'list', '--context': '-c', '--project': '-p', '--host': '-h', }, args) try: test_run_type = args[0] test_run_pattern = args[1] except: print("can't get args") print(" pdk gen_expected test_run_type test_run_pattern") sys.exit(1) test_run_pattern = common.find_test_run(test_run_pattern) print("test_run_pattern = %s" % test_run_pattern) l = [('test_run', test_run_pattern)] if '-c' in opts: l = l + [('context', x) for x in opts['-c']] if '-p' in opts: l = l + [('project', x) for x in opts['-p']] if '-h' in opts: l = l + [('host', x) for x in opts['-h']] if debug: for x in l: print(" %s" % x) print("?") sys.stdin.readline() where_str, where_dict = pdk_db.where_dict(l) sql = "select distinct project, host, context, test_name from result_scalar %s " % where_str c = pdk_db.execute(sql, where_dict) for (project, host, context, test_name) in c: if test_name.endswith("nose.failure.Failure.runTest"): # Sometimes nose generates this test name. I don't want it in the database at all, because # the name is not unique, and the record does not contain any useful information about the problem. # In any case, we never want to list this test as "expected", even # if it leaks into the database. continue if debug: print("expect %s %s %s %s %s" % (test_run_type, project, host, context, test_name)) # insert to the expected table; if the record is already there, it's # ok. try: pdk_db.execute( 'insert into expected ( test_run_type, project, host, context, test_name ) values ( :1, :2, :3, :4, :5 )', (test_run_type, project, host, context, test_name)) except pdk_db.IntegrityError as e: if debug: print("exception %s" % e) pass pdk_db.commit()
def run(args): argspec = { '-r': '=+', '-s': '=', '--help': '', '--test_run': '-r', '--subject': '-s', } opt, args = easyargs.get(argspec, args) if opt['--help']: print(""" -r --test_run specify a list of test runs to report on -s --subject specify a subject for the email other parameters are users to send email to (not email addresses) """) return 0 if '-r' in opt: test_runs = [pandokia.common.find_test_run(x) for x in opt['-r']] else: test_runs = [pandokia.common.find_test_run("daily_latest")] if args: # for each user name, look it up the email address in the table users = [] for user in args: found = 0 c = pdk_db.execute( "SELECT email FROM user_prefs WHERE username = :1", (user,)) for email, in c: users.append((user, email)) found = 1 if not found: print("No email address known for user %s" % user) else: # get a list of all the (user, email) from the user prefs query = """SELECT username, email FROM user_prefs""" user_res = pdk_db.execute(query) users = [(user, email) for user, email in user_res] if '-s' in args: subject = args['-s'] else: subject = 'Test Results: %s' % ', '.join([x for x in test_runs]) # compute the email to send to each user; send it. for user, email in users: if email is None: continue msg = '' for x in test_runs: newmsg = create_email(user, x) if newmsg is not None: msg = msg + newmsg if len(msg) > 0 and email: sendmail( email, subject, '%s\n\n\nThis report created for %s' % (msg, email)) else: print("suppress blank email to %s" % email)
def main(): global xnodes global no_run # read all the input files if readline: history = os.path.join(os.path.expanduser("~"), ".steuermann_history") try: readline.read_history_file(history) except IOError: pass import atexit atexit.register(readline.write_history_file, history) # easyargs spec definition: # # '-v' : '', # arg takes no parameter, opt['-v'] is # # how many times it occurred # '-f' : '=', # arg takes a parameter # '-mf' : '=+', # arg takes a parameter, may be specified # # several times to get a list # '--verbose' : '-v', # arg is an alias for some other arg opt, args = easyargs.get(allowed_flags, allow_unexpected=True) do_all = opt['-a'] no_run = opt['-n'] ''' NOTE: moving this functionality into nodes.py # find any unknown arguments like --something=whatever, set as conditions arguments = sys.argv[1:] for a in arguments: if '--' in a and '=' in a: not_allowed_flag = True for f in allowed_flags.keys(): if a.startswith(f): not_allowed_flag = False break if not_allowed_flag: a = a.lstrip('--') k, v = a.split('=') nodes.saved_conditions[k] = eval(v) ''' sm_files = [ os.path.abspath(a) for a in args if ('--' not in a and '=' not in a) ] di_nodes = nodes.read_file_list(sm_files) xnodes = di_nodes.node_index # get run name if '-r' in opt: run_name = opt['-r'] else: run_name = "user_%s_%s" % (username, str( datetime.datetime.now()).replace(' ', '_')) # get hosts (*.ini) file name if '-h' in opt: hosts_ini = opt['-h'] else: hosts_ini = os.path.join(os.path.dirname(__file__), 'hosts.ini') # Use a user-defined config if it exists if os.path.exists(config.hosts_config): hosts_ini = config.hosts_config # parse common resources from hosts INI file get_common_resources(hosts_ini) db = config.open_db() if do_all: run_all(xnodes, run_name, hosts_ini, db) else: run_interactive(xnodes, run_name, hosts_ini, db)
try: f = urllib2.urlopen(req) except urllib2.HTTPError as e: print("HTTP ERROR", e.code) print(e.read()) return 1 print(f.read()) f.close() return 0 if __name__ == '__main__': opt, args = easyargs.get({ '-p': '=', # password '-f': '=', # password from file '-u': '', # upload '-d': '=', # upload destination directory (default .) '-h': '=', # host name }) if '-p' in opt: password = opt['-p'] elif '-f' in opt: password = open(opt['-f'], 'r').readline().strip() if not ('-h' in opt): print("must give host name with -h") sys.exit(2) host = opt['-h'] if '-d' in opt: directory = opt['-d'] else:
def delete(args): '''pdk delete -field value -otherfield othervalue ... deletes records from the database where all the parameters match This command can remove a subset of the records for a test run. It can also remove all the records for a test run, but leave the test run in the index as an empty dataset. The count of records in a test run will be incorrect until you update it. pdk delete -test_run foo -project bar -context baz -host xyzzy also accepts: -wild allow wildcards; some queries are dramatically inefficient when you use wildcards, so you have to actively ask to use them. if you use * or ?, you must specify -wild -count do not delete the records - just show how many are affected ''' import pandokia.helpers.easyargs as easyargs import pandokia.common as common opt, args = easyargs.get( { '-test_run': '=+', '-project': '=+', '-context': '=+', '-host': '=+', '-status': '=+', '-wild': '', '-count': '', '-c': '', }, args ) dont = 0 if len(args) != 0: if '-test_run' not in opt: opt['-test_run'] = args else: print("error: -test_run and non-option args used together") wild = opt['-wild'] del opt['-wild'] count = opt['-count'] | opt['-c'] del opt['-count'] del opt['-c'] if '-test_run' not in opt: print("You really have to specify -test_run") return 1 # expand wild cards in test run lll = [] for x in opt['-test_run']: lll = lll + common.expand_test_run(x) opt['-test_run'] = lll # if not count: lll = [] for x in opt['-test_run']: if check_valuable(x): print("test run %s is marked valuable - cannot delete" % x) else: lll.append(x) # opt['-test_run'] = lll if len(lll) == 0: # print "No deleteable test runs found" dont = 1 lll = [] for x in sorted(opt.keys()): if x == '-test_run': continue v = opt[x] lll.append((x[1:], v)) if not wild: if ('*' in v) or ('?' in v) or ('%' in v): print('\nError: must use -wild for %s %s\n' % (x, v)) dont = 1 if dont: return for x in opt['-test_run']: where_str, where_dict = pdk_db.where_dict(lll + [('test_run', x)]) if count: c = pdk_db.execute( "SELECT count(*) FROM result_scalar %s" % (where_str,), where_dict) print("%s %s" % (x, c.fetchone()[0])) else: print(x) delete_by_query(where_str, where_dict) recount([x], verbose=0) return 0
def run(args): opts, args = easyargs.get({ '-c': 'list', '-p': 'list', '-h': 'list', '--context': '-c', '--project': '-p', '--host': '-h', }, args) try: test_run_type = args[0] test_run_pattern = args[1] except: print("can't get args") print(" pdk gen_expected test_run_type test_run_pattern") sys.exit(1) test_run_pattern = common.find_test_run(test_run_pattern) print("test_run_pattern = %s" % test_run_pattern) l = [('test_run', test_run_pattern)] if '-c' in opts: l = l + [('context', x) for x in opts['-c']] if '-p' in opts: l = l + [('project', x) for x in opts['-p']] if '-h' in opts: l = l + [('host', x) for x in opts['-h']] if debug: for x in l: print(" %s" % x) print("?") sys.stdin.readline() where_str, where_dict = pdk_db.where_dict(l) sql = "select distinct project, host, context, test_name from result_scalar %s " % where_str c = pdk_db.execute(sql, where_dict) for (project, host, context, test_name) in c: if test_name.endswith("nose.failure.Failure.runTest"): # Sometimes nose generates this test name. I don't want it in the database at all, because # the name is not unique, and the record does not contain any useful information about the problem. # In any case, we never want to list this test as "expected", even # if it leaks into the database. continue if debug: print( "expect %s %s %s %s %s" % (test_run_type, project, host, context, test_name)) # check if the row already exists before insert since the expected table does not have unique enforcement # on its columns a = pdk_db.execute('select * from expected where test_run_type = :1 and project = :2 and host = :3 and context = :4 and test_name = :5', (test_run_type, project, host, context, test_name)) y = a.fetchone() if y is None: try: pdk_db.execute( 'insert into expected ( test_run_type, project, host, context, test_name ) values ( :1, :2, :3, :4, :5 )', (test_run_type, project, host, context, test_name)) except Exception as e: if debug: print("exception %s" % e) pass pdk_db.commit()
def run(args): opts, args = easyargs.get( { '-c': 'list', '-p': 'list', '-h': 'list', '-m': 'list', '--context': '-c', '--project': '-p', '--host': '-h', '--custom': '-m', }, args) try: test_run_type = args[0] test_run_pattern = args[1] except: print("can't get args") print(" pdk gen_expected test_run_type test_run_pattern") sys.exit(1) test_run_pattern = common.find_test_run(test_run_pattern) print("test_run_pattern = ", test_run_pattern) l = [('test_run', test_run_pattern)] if '-c' in opts: l = l + [('context', x) for x in opts['-c']] if '-p' in opts: l = l + [('project', x) for x in opts['-p']] if '-h' in opts: l = l + [('host', x) for x in opts['-h']] if '-m' in opts: l = l + [('custom', x) for x in opts['-m']] if debug: for x in l: print(" ", x) print("?") sys.stdin.readline() where_str, where_dict = pdk_db.where_dict(l) sql = "select distinct project, host, context, custom, test_name from result_scalar %s " % where_str c = pdk_db.execute(sql, where_dict) for (project, host, context, custom, test_name) in c: if test_name.endswith("nose.failure.Failure.runTest"): # Sometimes nose generates this test name. I don't want it in the database at all, because # the name is not unique, and the record does not contain any useful information about the problem. # In any case, we never want to list this test as "expected", even if it leaks into the database. continue if debug: print("expect ", test_run_type, project, host, context, custom, test_name) a = pdk_db.execute( 'select * from expected where test_run_type = :1 and project = :2 and host = :3 and context = :4 and custom = :5 and test_name = :6', (test_run_type, project, host, context, custom, test_name)) y = a.fetchone() if y is None: try: pdk_db.execute( 'insert into expected ( test_run_type, project, host, context, custom, test_name ) values ( :1, :2, :3, :4, :5, :6 )', (test_run_type, project, host, context, custom, test_name)) except Exception as e: if debug: print("exception", e) pass pdk_db.commit()