def main(): test = tester.Core('webserver test', 15) # start server at some random port. this may cause collisions. random.seed(None) port = random.randint(2049, 65534) print 'starting server at port ' + str(port) # timeout is 3600, 60 minutes test.start_program('./run-experiment ' + str(port), 3600) test.look("Threads experiment done.\r\n") count = file_line_count('plot-threads.out', \ [0, 1, 2, 4, 8, 16, 32, 64, 128]) test.look("Requests experiment done.\r\n") count += file_line_count('plot-requests.out', [1, 2, 4, 8, 16, 32]) test.look("Cachesize experiment done.\r\n") cache_count = file_line_count('plot-cachesize.out', \ [16384, 65536, 262144, \ 1048576, 4194304, 16777216]) # give a maximum of 3 marks for first two experiments, # and a max of 12 marks for the cache experiments newcount = int(round(count/5 + 2 * cache_count)) test.add_mark(newcount) test.start_program('./plot-experiment')
def main(): test = tester.Core('table test', 5) tester.includepath() import table t = table.Table(int, "abcdefg", range(7)) init = random.randint(0, 99) skip = random.randint(1, 9) curr = init for x in t.xlabels: for y in t.ylabels: t[y, x] = curr curr += skip curr = init for x in t.xlabels: for y in t.ylabels: if t[y, x] != curr: return curr += skip test.add_mark(4) del t[3, 'b'] if t[3, 'b'] is not None: return test.add_mark(1)
def main(): test = tester.Core('basic test', 44) test.start_program('./test_basic') test.lookA('starting basic test') test.lookA('initial thread returns from yield\(SELF\)', 1) test.lookA('initial thread returns from yield\(0\)', 1) test.lookA('initial thread returns from yield\(ANY\)', 1) test.lookA('initial thread returns from yield\(INVALID\)', 1) test.lookA('initial thread returns from yield\(INVALID2\)', 1) test.lookA('my id is 0', 1) test.lookA('message: hello from first thread', 1) test.lookA('thread returns from first yield', 1) test.lookA('thread returns from second yield', 1) test.lookA('my id is 0', 1) for i in range(tester.NTHREADS): if (test.lookA('message: hello from thread\s+(\d+)') < 0): break else: num = test.program.match.group(1) num = int(num) if (num < 0 or num >= tester.NTHREADS): break threads[num] = 1 if (test.lookA('thread returns from first yield') < 0): break if (test.lookA('thread returns from second yield') < 0): break mark = 0 for i in range(tester.NTHREADS): mark = mark + threads[i] if (mark == tester.NTHREADS): test.add_mark(20) test.lookA('destroying all threads', 1) test.lookA('creating 1023 threads', 1) test.lookA('running 1023 threads', 1) test.lookA('creating 1023 threads', 1) test.lookA('destroying 512 threads', 1) test.lookA('testing some destroys even though I\'m the only thread', 1) test.lookA('testing destroy self', 1) test.lookA('for my grand finale, I will destroy myself', 1) test.lookA('while my talented assistant prints \"basic test done\"', 0) test.lookA('finale running', 1) test.lookA('basic test done', 1) # wait so that the test finishes correctly if test.wait_until_end(): test.add_mark(4)
def main(): mark = tester.LOOPS + 2 test = tester.Core('cv test', mark) print 'running ' + str(test.nr_times) + ' times' for i in range(test.nr_times): experiment(test) test.program.close() mark = min(mark, test.get_mark()) test.reset_mark() test.add_mark(mark)
def run_script(filename, marks): """ filename: path to script marks: marks assigned to this test case """ path = os.path.join(TEST_FOLDER, filename) test = tester.Core(filename, marks) test.start_program(path) test.lookA("PASS", marks) test.program.close()
def main(): test = tester.Core('arguments test', 15) wizard = tester.datapath('wizard.txt', 'asst1') shoe = tester.datapath('shoe.txt', 'asst1') test.start_program('./asst1 -i %d' % (random.randint(-int(maxsize), 0))) check_and_wait(test, r'Error: SEED must be a non-negative integer\.') test.start_program('./asst1 -a %s 0' % wizard) check_and_wait(test, r'Error: NUM must be a natural number\.') test.start_program('./asst1 -a %s' % wizard) check_and_wait( test, r'Error: must specify number of hands when playing automatically\.') test.start_program('./asst1 -s') check_and_wait( test, r'Error: silent mode is only available when playing automatically\.') seed = 123 test.start_program('./asst1 -f %s -i %d' % (shoe, seed)) check_and_wait(test, r'Error: cannot choose both file and random-based shoe\.', 0) temp = "record.txt" test.start_program('./asst1 -f %s -r %s' % (shoe, temp)) check_and_wait( test, r'Error: recording is only available for random-based shoe\.', 0) try: os.remove(temp) except OSError as e: # the student may not have created this file yet pass test.start_program('./asst1 -f') check_and_wait(test, r"\.\/asst1: option requires an argument -- 'f'", 0) test.start_program('./asst1 -r') check_and_wait(test, r"\.\/asst1: option requires an argument -- 'r'", 0) test.start_program('./asst1 -i') check_and_wait(test, r"\.\/asst1: option requires an argument -- 'i'", 0) test.start_program('./asst1 -a') check_and_wait(test, r"\.\/asst1: option requires an argument -- 'a'", 0) test.start_program('\./asst1 -h') check_and_wait( test, r"usage: \.\/asst1 \[-h\] \[-f FILE\|-i SEED \[-r FILE\]\] \[\[-s\] -a FILE NUM\]", 0)
def main(): test = tester.Core('warmup test', 40) test.start_program('./hello') test.lookA('Hello world', 1) test.start_program('./words how many words') test.lookA('how', 1) test.lookA('many', 1) test.lookA('words', 1) test.start_program('./fact 9') test.lookA('362880', 1) test.start_program('./fact 12') test.lookA('479001600', 1) test.start_program('./fact 0') test.lookA('Huh\?', 1) test.start_program('./fact hello') test.lookA('Huh\?', 1) test.start_program('./fact 20') test.lookA('Overflow', 1) test.start_program('./fact 1.2') test.lookA('Huh\?', 1) test.start_program('./test_point') test.lookA('OK', 2) # test.start_program('./test_sorted_points', 20) # test.lookA('starting sorted points test.'); # test.lookA('this test may take a minute.'); # test.lookA('this test may also take a minute.'); # test.lookA('OK', 8) test.start_program('./run_small_test_wc') test.lookA('OK', 10) # terminate program if it runs for more than 35 seconds test.start_program('./run_big_test_wc', 35) test.lookA('OK', 10) # handle test_wc program not being killed properly kill_process('test_wc') test.start_program('./test_cpr', 5) test.lookA('OK', 1) test.lookA('OK', 5) test.lookA('OK', 2)
def main(): test = tester.Core('webserver test', 15) # start server at some random port. this may cause collisions. random.seed(None) port = random.randint(2049, 65534) print 'starting server at port ' + str(port) # timeout is 3600, 60 minutes test.start_program('./run-experiment ' + str(port) + " nocache", 3600) test.look("Threads experiment done.\r\n") count = file_line_count('plot-threads.out', \ [0, 1, 2, 4, 8, 16, 32, 64, 128]) test.look("Requests experiment done.\r\n") count += file_line_count('plot-requests.out', [1, 2, 4, 8, 16, 32]) test.add_mark(count) test.start_program('./plot-experiment')
def main(): test = tester.Core('memory test', 5) path = tester.datapath("wizard.txt", 'asst1') test.start_program('valgrind ./asst1 -s -i %d -a %s %d' % (4, path, 10000), timeout=10) find_or_exit(test, 'Hands Played: 10000') find_or_exit(test, 'Final Balance: \+\$1103\.00') find_or_exit(test, 'Player Advantage: 11\.03\%') test.set_timeout(1) exit_on_leak(test, 'definitely lost: .+? bytes in (\d+) blocks') exit_on_leak(test, 'indirectly lost: .+? bytes in (\d+) blocks') exit_on_leak(test, 'possibly lost: .+? bytes in (\d+) blocks') test.add_mark(3) exit_on_leak(test, 'still reachable: .+? bytes in (\d+) blocks') test.add_mark(2)
def main(): test = tester.Core('table test', 5) tester.includepath() import table t = table.Table(int, "abcdefg", range(7)) init = random.randint(0, 99) skip = random.randint(1, 9) curr = init for x in t.xlabels: for y in t.ylabels: t[y, x] = curr curr += skip curr = init for x in t.xlabels: for y in t.ylabels: if t[y, x] != curr: print("table value mismatch (what is set cannot be gotten)") return curr += skip test.add_mark(2) t2 = table.Table(float, string.digits, range(10)) for x in t2.xlabels: for y in t2.ylabels: t2[y, x] = random.random() * 2 - .8 curr = init for x in t.xlabels: for y in t.ylabels: if t[y, x] != curr: print("write to another table affects this table!") return curr += skip test.add_mark(2) del t[3, 'b'] if t[3, 'b'] is not None: print("cannot delete from table") return test.add_mark(1)
def main(): test = tester.Core('webserver test', 16) # start server at some random port. this may cause collisions. random.seed(None) port = random.randint(2049, 65534) print 'starting server at port ' + str(port) # timeout for each look below is 20 minutes test.start_program('./run-cache-experiment ' + str(port), 1200) test.look("Cachesize experiment done.\r\n") mark = process_experiment(test) test.add_mark(mark) test.start_program('./plot-cache-experiment') # handle various programs not being killed properly kill_process('./run-one-experiment') kill_process('./server')
def main(): test = tester.Core('warmup test', 30) test.start_program('./hello') test.lookA('Hello world', 1) test.start_program('./words how many words') test.lookA('how', 1) test.lookA('many', 1) test.lookA('words', 1) test.start_program('./fact 9') test.lookA('362880', 1) test.start_program('./fact 12') test.lookA('479001600', 1) test.start_program('./fact 0') test.lookA('Huh\?', 1) test.start_program('./fact hello') test.lookA('Huh\?', 1) test.start_program('./fact 20') test.lookA('Overflow', 1) test.start_program('./fact 1.2') test.lookA('Huh\?', 1) test.start_program('./test_point') test.lookA('OK', 2) test.start_program('./test_sorted_points', 20) test.look('OK\r\n', 8) test.start_program('./run_test_wc') test.lookA('OK', 10)
def main(): test = tester.Core('manual test', 40) # test if either or both have blackjack begin(test, 'bjshoe.txt') test.look('Dealer: A K \(blackjack\)', 1) test.look('Result: -\$1\.00', 1) prompt(test) test.look('Hand 1: T A \(blackjack\)', 1) test.look('Result: \+\$1\.50', 1) prompt(test) test.look('Result: \$0\.00', 1) prompt(test, 'Q') # test output of hitting begin(test, 'hitshoe.txt') action(test, "SHDR", 'H', 1) action(test, "SH", 'H') action(test, "SH", 'r') # not allowed action(test, "SH", 'p') # not allowed action(test, "SH", 'd') # not allowed action(test, "SH", 'H') action(test, "SH", 'x') # random character action(test, "SH", 'hit', 1) # test if lowercase accepted # bust, no more action test.look('Result: -\$1\.00', 1) prompt(test) action(test, "SHDR", 'H') # got 21, should have no more action test.look('Dealer: A 6 8 8 \(bust\)', 1) test.look('Result: \+\$1\.00', 1) prompt(test) action(test, "SHDR", 'H') action(test, "SH", 'S', 1) test.look('Dealer: 7 8 6 \(21\)', 1) test.look('Result: -\$1\.00', 1) prompt(test, 'Q') begin(test, 'dblshoe.txt') action(test, "SHDR", 'please') action(test, "SHDR", 'd', 1) test.look('Dealer: K 6 3 \(19\)', 1) test.look('Hand 1: 5 6 Q \(21\)', 1) test.look('Result: \+\$2\.00', 1) prompt(test) action(test, "SHDR", 'r') # dealer should not hit after a surrender test.look('Dealer: 2 9 \(11\)', 1) test.look('Hand 1: 5 J \(15\) SURRENDER', 1) test.look('Result: -\$0\.50', 1) prompt(test) action(test, "SHDPR", 'ddd') test.look('Dealer: 7 7 \(14\)', 1) test.look('Hand 1: 6 6 Q \(bust\) DOUBLE', 1) test.look('Result: -\$2\.00', 1) prompt(test, 'Q') # test output of splitting begin(test, 'splshoe.txt') action(test, "SHDPR", 'P', 1) action(test, "SHDP", 'P', 1) action(test, "SHDP", 'P') # no more split now action(test, "SHD", 'P', 1) # should not be allowed action(test, "SHD", 'R') action(test, "SHD", 'S', 1) action(test, "SHD", 'D') action(test, "SHD", 'h') action(test, "SH", 'h', 1) action(test, "SHD", 'HIT') action(test, "SH", 'stand') test.look('Hand 1: 8 8 \(16\)', 1) test.look('Hand 2: 8 8 2 \(18\) DOUBLE', 1) test.look('Hand 3: 8 8 A J \(bust\)', 1) test.look('Hand 4: 8 8 A \(17\)', 1) test.look('Result: -\$5\.00', 1) prompt(test) action(test, "SHDPR", 'P') test.look('Dealer: A A 7 \(soft 19\)', 1) test.look('Hand 1: A 5 \(soft 16\)', 1) test.look('Hand 2: A 6 \(soft 17\)') test.look('Result: -\$2\.00', 1) test.look('Current Balance: -\$7\.00', 1) prompt(test) # after split, soft 21 is not blackjack action(test, "SHDPR", 'P') test.look('Hand 1: A J \(soft 21\)', 1) prompt(test) # dealer should not hit after double bust action(test, "SHDPR", 'P') action(test, "SHD", 'h', 1) action(test, "SHD", 'h') test.look('Dealer: T 3 \(13\)', 1) prompt(test, 'Q')
def main(): test = tester.Core('ucontext test', 10) test.start_program('./show_ucontext') state = test.look('start = (0x[0-9a-fA-F]+)\r\n') if (state == 0): startaddr = test.program.match.group(1) startaddr = int(startaddr, 0) else: print 'ERROR: start address seems suspicious' return state = test.lookA('end = (0x[0-9a-fA-F]+)') if (state == 0): endaddr = test.program.match.group(1) endaddr = int(endaddr, 0) else: print 'ERROR: end address seems suspicious' return state = test.lookA('ucontext_t size = ([-]*\d+) bytes') if (state == 0): bytes = test.program.match.group(1) bytes = int(bytes) if (bytes > 600 and bytes < 1200): test.add_mark(1) else: print 'ERROR: ucontext size seems suspicious' state = test.lookA('memory address of main\(\) = (0x[0-9a-fA-F]+)') if (state == 0): mainaddr = test.program.match.group(1) mainaddr = int(mainaddr, 0) if (mainaddr > startaddr and mainaddr < endaddr): test.add_mark(1) else: print 'ERROR: main address seems suspicious' state = test.lookA( 'memory address of the program counter \(RIP\) saved in mycontext = (0x[0-9a-fA-F]+)' ) if (state == 0): eip = test.program.match.group(1) eip = int(eip, 0) if (eip > startaddr and eip < endaddr and \ eip > mainaddr + 0x8 and eip < mainaddr + 0x30): test.add_mark(1) else: print 'ERROR: RIP address seems suspicious' state = test.look( 'memory address of the variable setcontext_called = (0x[0-9a-fA-F]+)\r\n' ) if (state == 0): setc = test.program.match.group(1) setc = int(setc, 0) if (setc > 0x7f0000000000 and setc < 0x800000000000): test.add_mark(1) else: print 'ERROR: variable SETCONTEXT_CALLED address seems suspicious' state = test.lookA('memory address of the variable err = (0x[0-9a-fA-F]+)') if (state == 0): err = test.program.match.group(1) err = int(err, 0) if (err > 0x7f0000000000 and err < 0x800000000000 and \ (setc - err) >= -0x10 and (setc - err) <= 0x10): test.add_mark(1) else: print 'ERROR: variable ERR address seems suspicious' state = test.lookA( 'number of bytes pushed to the stack between setcontext_called and err = ([-]*\d+)' ) if (state == 0): setc_err = test.program.match.group(1) setc_err = int(setc_err) if (setc_err >= -0x10 and setc_err <= 0x10): test.add_mark(1) else: print 'ERROR: SETCONTEXT_CALLED to ERR bytes seems suspicious' state = test.lookA( 'stack pointer register \(RSP\) stored in mycontext = (0x[0-9a-fA-F]+)' ) if (state == 0): rsp = test.program.match.group(1) rsp = int(rsp, 0) if (rsp > 0x7f0000000000 and rsp < 0x800000000000 and \ (err - rsp) >= 0x4 and (err - rsp) <= 0x20): test.add_mark(1) else: print 'ERROR: RSP address seems suspicious' state = test.lookA( 'number of bytes between err and the saved stack in mycontext = ([-]*\d+)' ) if (state == 0): err_rsp = test.program.match.group(1) err_rsp = int(err_rsp) if (err_rsp >= 0x4 and err_rsp <= 0x20): test.add_mark(1) else: print 'ERROR: ERR to saved RSP bytes seems suspicious' state = test.look('interrupt is disabled = ([-]*\d+)\r\n') if (state == 0): intr = test.program.match.group(1) intr = int(intr) if (intr == 0): test.add_mark(1) else: print 'ERROR: interrupt value seems suspicious' state = test.lookA('interrupt is disabled = ([-]*\d+)') if (state == 0): intr = test.program.match.group(1) intr = int(intr) if (intr == 1): test.add_mark(1) else: print 'ERROR: interrupt value seems suspicious'
def main(): mark = 14 test = tester.Core('wait test', mark) test_wait(test) test_wait_kill(test) test_wait_parent(test)
def start_test(testname, marks): test = tester.Core(testname, marks) tester.includepath() return test
def main(): test = tester.Core('performance test', 20) for testcase in TEST_CASES: run_test_case(test, testcase)
def main(): test = tester.Core('auto test', 20) for testcase in TEST_CASES: run_test_case(test, testcase)