def test_tutorial5_hello_notrace(self, tmpdir, baremetal_apps_hello, sim_system_2x2_cccc_sim_dualcore_debug): """ Tutorial 5: use osd-target-run and system with debug system to run software (no trace enabled) """ hello_elf = str(baremetal_apps_hello.join('hello.elf')) cmd_targetrun = ['osd-target-run', '-e', hello_elf, '-vvv'] p_targetrun = util.Process(cmd_targetrun, logdir=str(tmpdir), cwd=str(tmpdir)) p_targetrun.run() logging.getLogger(__name__).info("Program should terminate itself; give it up to 5 minutes") p_targetrun.proc.wait(timeout=300) assert p_targetrun.proc.returncode == 0 logging.getLogger(__name__).info("Now wait 60 seconds until the program has finished execution") time.sleep(60) try: p_targetrun.terminate() except ProcessLookupError: # process is already dead pass # check the simulation-generated stdout files for i in range(0, 7): f = "stdout.{:03d}".format(i) assert tmpdir.join(f).isfile() assert util.matches_golden_reference(str(tmpdir), f, filter_func=util.filter_timestamps)
def test_tutorial6_hello(self, tmpdir, baremetal_apps_hello, sim_system_2x2_cccc_sim_dualcore_debug, opensocdebugd_tcp): """ Tutorial 6: Same as tutorial 5, but using the python interface of osd-cli instead of manually typing commands """ hello_elf = str(baremetal_apps_hello.join('hello.elf')) cmd_runelf = [ 'python2', '{}/host/share/opensocdebug/examples/runelf.py'.format( os.environ['OPTIMSOC']), hello_elf ] p_runelf = util.Process(cmd_runelf, logdir=str(tmpdir), cwd=str(tmpdir)) p_runelf.run() logging.getLogger(__name__).info( "Waiting 10 minutes for process to finish") p_runelf.proc.wait(timeout=600) assert p_runelf.proc.returncode == 0 try: p_runelf.terminate() except ProcessLookupError: # process is already dead pass # check all output files for i in range(0, 7): f = "stdout.{:03d}".format(i) assert tmpdir.join(f).isfile() assert util.matches_golden_reference( str(tmpdir), f, filter_func=util.filter_timestamps)
def sim_system_2x2_cccc_sim_dualcore_debug(self, tmpdir): cmd_sim = '{}/examples/sim/system_2x2_cccc/system_2x2_cccc_sim_dualcore_debug'.format(os.environ['OPTIMSOC']) p_sim = util.Process(cmd_sim, logdir=str(tmpdir), cwd=str(tmpdir), startup_done_expect='Glip TCP DPI listening', startup_timeout=10) p_sim.run() yield p_sim p_sim.terminate()
def test_tutorial5(self, tmpdir, baremetal_apps_hello, sim_system_2x2_cccc_sim_dualcore_debug, opensocdebugd_tcp): """ Tutorial 5: Verilator with debug system and interactive osd-cli """ cmd_osdcli = 'osd-cli' p_osdcli = util.Process(cmd_osdcli, logdir=str(tmpdir), cwd=str(tmpdir), startup_done_expect="osd>", startup_timeout=10) p_osdcli.run() assert p_osdcli.expect(stdin_data="help\n", pattern="Available commands:") time.sleep(0.5) # XXX: match end of the output to avoid sleep assert p_osdcli.expect(stdin_data="mem help\n", pattern="Available subcommands:") time.sleep(0.5) # XXX: match end of the output to avoid sleep assert p_osdcli.expect(stdin_data="reset -halt\n", pattern="osd>") hello_elf = str(baremetal_apps_hello.join('hello.elf')) assert p_osdcli.expect( stdin_data="mem loadelf {} 2 -verify\n".format(hello_elf), pattern="Verify program header 1") assert p_osdcli.expect(stdin_data="stm log stm000.log 3\n", pattern="osd>") assert p_osdcli.expect( stdin_data="ctm log ctm000.log 4 {}\n".format(hello_elf), pattern="osd>") assert p_osdcli.expect( stdin_data="start\n", pattern=re.compile(r"\[STM 003\] [a-f0-9]{8} rank 3 is tile 3"), timeout=60) assert p_osdcli.expect(stdin_data="quit\n") # wait up to 2 seconds for osd-cli to terminate, then kill try: p_osdcli.proc.wait(timeout=2) except subprocess.TimeoutExpired: p_osdcli.proc.kill() # XXX: also check the contents of the STM/CTM logs assert tmpdir.join('stm000.log').isfile() assert tmpdir.join('ctm000.log').isfile() return
def opensocdebugd_tcp(self, tmpdir): cmd_opensocdebugd = ['opensocdebugd', 'tcp'] p_opensocdebugd = util.Process( cmd_opensocdebugd, logdir=str(tmpdir), cwd=str(tmpdir), startup_done_expect="Wait for connection", startup_timeout=30) p_opensocdebugd.run() yield p_opensocdebugd p_opensocdebugd.terminate()
def do_it(user, dpath, dname): p_list = [] reset = True num = 0 cursor = None connection = None max_download = None sql = db_interface.SqlLite(dpath + dname) loc = loc_interface.GeoPy('user_agent = "zoby"') q = util.Queue() counter = util.Counter(0) lock = Lock() while True: data = (num, user, dpath, dname, sql, reset, cursor, connection, loc, counter, lock) p = util.Process(target=build_user_db, args=(data, q, max_download)) p.start() if num == 0: p_num = q.get() cursor, connection = sql.open_db() else: p_list.append(p) reset = False if num == p_num or num > 20000: print(num, ' == ', p_num) break num += 1 for p in p_list: p.join() # print('process done: ', p.name) print('\ndo_it qsize: ', q.qsize()) items = [q.get() for _ in range(q.qsize())] tot = 0 for item in items: num, dirpath, file_cnt = item tot += file_cnt print('tot files: ', tot) sql.close_db()
def test_tutorial5_hello_coretrace(self, tmpdir, baremetal_apps_hello, sim_system_2x2_cccc_sim_dualcore_debug): """ Tutorial 5: use osd-target-run and system with debug system to run software (with core trace) """ startup_done_string = '[INFO] osd-target-run: System is now running. Press CTRL-C to end tracing.' hello_elf = str(baremetal_apps_hello.join('hello.elf')) cmd_targetrun = [ 'osd-target-run', '-e', hello_elf, '--coretrace', '-vvv' ] p_targetrun = util.Process(cmd_targetrun, logdir=str(tmpdir), cwd=str(tmpdir), startup_done_expect=startup_done_string, startup_timeout=300) p_targetrun.run() logging.getLogger(__name__).info("Record traces for 1 minute") time.sleep(60) p_targetrun.send_ctrl_c() # Give the process some time to clean up p_targetrun.proc.wait(timeout=30) assert p_targetrun.proc.returncode == 0 try: p_targetrun.terminate() except ProcessLookupError: # process is already dead pass # Ensure that the CTM logs are written ctmlogs = [ 'coretrace.0003.log', 'coretrace.0005.log', 'coretrace.0008.log', 'coretrace.0010.log', 'coretrace.0013.log', 'coretrace.0015.log', 'coretrace.0018.log', 'coretrace.0020.log' ] for f in ctmlogs: # CTM log file exists assert tmpdir.join(f).isfile() # CTM log >= 0 bytes # Currently some of the CTM logs don't contain a full trace due to # overload in the debug system causing dropped packets. Checking for # such partial logs cannot be done reliably. f_stat = os.stat(str(tmpdir.join(f))) assert f_stat.st_size > 0
def test_tutorial_linux(self, tmpdir, localconf, linux_compute_tile_singlecore): """ Tutorial 8: Execute the Linux example in the tutorial """ # program FPGA with bitstream bitstream = "{}/examples/fpga/nexys4ddr/compute_tile/compute_tile_nexys4ddr_singlecore.bit".format(os.environ['OPTIMSOC']) cmd_pgm = ['optimsoc-pgm-fpga', bitstream, 'xc7a100t_0'] p_pgm = util.Process(cmd_pgm, logdir=str(tmpdir), cwd=str(tmpdir)) p_pgm.run() p_pgm.proc.wait(timeout=300) assert p_pgm.proc.returncode == 0 time.sleep(2) # run hello.elf on target board nexys4ddr_device = localconf['boards']['nexys4ddr']['device'] logging.getLogger(__name__).info("Using Nexys 4 DDR board connected to " + nexys4ddr_device) glip_backend = 'uart' glip_backend_options = 'device=%s,speed=12000000' % nexys4ddr_device vmlinux = str(linux_compute_tile_singlecore) cmd_targetrun = ['osd-target-run', '-e', vmlinux, '-b', glip_backend, '-o', glip_backend_options, '--systrace', '--verify', '--terminal', '-vvv'] startup_done_string = '[INFO] osd-target-run: System is now running. Press CTRL-C to end tracing.' p_targetrun = util.Process(cmd_targetrun, logdir=str(tmpdir), cwd=str(tmpdir), startup_done_expect=startup_done_string, startup_timeout=300) p_targetrun.run() # Find allocated pseudo-terminal (expected to appear within 10 seconds) tty_dev = None search = re.compile(r'^.+DEM-UART pseudo-terminal available at (/dev/pts/\d+)') match = p_targetrun._find_in_output(search, timeout = 5) tty_dev = match.group(1) assert tty_dev logging.getLogger(__name__).info("Linux console available at " + tty_dev) # Wait for Linux to boot logging.getLogger(__name__).info("Run Linux until we get a login prompt") cmd_tty_cat = ['cat', tty_dev] p_tty_cat = util.Process(cmd_tty_cat, logdir=str(tmpdir), cwd=str(tmpdir), startup_done_expect="Welcome to OpTiMSoC", startup_timeout=60) p_tty_cat.run() p_tty_cat.send_ctrl_c() p_tty_cat.proc.wait(timeout=5) try: p_tty_cat.terminate() except ProcessLookupError: # process is already dead pass p_targetrun.send_ctrl_c() # Give the process some time to clean up p_targetrun.proc.wait(timeout=30) try: p_targetrun.terminate() except ProcessLookupError: # process is already dead pass
def test_tutorial7(self, tmpdir, localconf, baremetal_apps_hello): """ Tutorial 7: Program a compute_tile system to a Nexys4 DDR board and run hello world on it. """ # program FPGA with bitstream bitstream = "{}/examples/fpga/nexys4ddr/compute_tile/compute_tile_nexys4ddr_singlecore.bit".format(os.environ['OPTIMSOC']) cmd_pgm = ['optimsoc-pgm-fpga', bitstream, 'xc7a100t_0'] p_pgm = util.Process(cmd_pgm, logdir=str(tmpdir), cwd=str(tmpdir)) p_pgm.run() p_pgm.proc.wait(timeout=300) assert p_pgm.proc.returncode == 0 time.sleep(2) # run hello.elf on target board nexys4ddr_device = localconf['boards']['nexys4ddr']['device'] logging.getLogger(__name__).info("Using Nexys 4 DDR board connected to " + nexys4ddr_device) glip_backend = 'uart' glip_backend_options = 'device=%s,speed=12000000' % nexys4ddr_device hello_elf = str(baremetal_apps_hello.join('hello.elf')) cmd_targetrun = ['osd-target-run', '-e', hello_elf, '-b', glip_backend, '-o', glip_backend_options, '--coretrace', '--systrace', '--verify', '-vvv'] startup_done_string = '[INFO] osd-target-run: System is now running. Press CTRL-C to end tracing.' p_targetrun = util.Process(cmd_targetrun, logdir=str(tmpdir), cwd=str(tmpdir), startup_done_expect=startup_done_string, startup_timeout=300) p_targetrun.run() logging.getLogger(__name__).info("Record traces for 30 seconds") time.sleep(30) p_targetrun.send_ctrl_c() # Give the process some time to clean up p_targetrun.proc.wait(timeout=30) assert p_targetrun.proc.returncode == 0 try: p_targetrun.terminate() except ProcessLookupError: # process is already dead pass # Ensure that the STM logs are written stmlogs = [ 'systrace.0002.log' ] for f in stmlogs: assert tmpdir.join(f).isfile() assert util.matches_golden_reference(str(tmpdir), f, filter_func=util.filter_osd_trace_timestamps) # Ensure that the CTM logs are written ctmlogs = [ 'coretrace.0003.log' ] for f in ctmlogs: assert tmpdir.join(f).isfile() assert util.matches_golden_reference(str(tmpdir), f, filter_func=util.filter_osd_trace_timestamps)
def test_tutorial7(self, tmpdir, localconf, baremetal_apps_hello): """ Tutorial 7: Program a 2x2 CCCC system to a Nexys4 DDR board and run hello world on it. """ # program FPGA with bitstream bitstream = "{}/examples/fpga/nexys4ddr/system_2x2_cccc/system_2x2_cccc_nexys4ddr.bit".format( os.environ['OPTIMSOC']) cmd_pgm = ['optimsoc-pgm-fpga', bitstream, 'xc7a100t_0'] p_pgm = util.Process(cmd_pgm, logdir=str(tmpdir), cwd=str(tmpdir)) p_pgm.run() p_pgm.proc.wait(timeout=300) assert p_pgm.proc.returncode == 0 time.sleep(2) # connect to board with opensocdebugd nexys4ddr_device = localconf['boards']['nexys4ddr']['device'] logging.getLogger(__name__).info("Using Nexys 4 board connected to " + nexys4ddr_device) cmd_opensocdebugd = [ 'opensocdebugd', 'uart', 'device=' + nexys4ddr_device, 'speed=12000000' ] p_opensocdebugd = util.Process( cmd_opensocdebugd, logdir=str(tmpdir), cwd=str(tmpdir), startup_done_expect="Wait for connection", startup_timeout=30) p_opensocdebugd.run() # run hello world on FPGA hello_elf = str(baremetal_apps_hello.join('hello.elf')) cmd_runelf = [ 'python2', '{}/host/share/opensocdebug/examples/runelf.py'.format( os.environ['OPTIMSOC']), hello_elf ] p_runelf = util.Process(cmd_runelf, logdir=str(tmpdir), cwd=str(tmpdir)) p_runelf.run() logging.getLogger(__name__).info( "Waiting 10 minutes for process to finish") p_runelf.proc.wait(timeout=600) assert p_runelf.proc.returncode == 0 try: p_runelf.terminate() except ProcessLookupError: # process is already dead pass p_opensocdebugd.terminate() # Ensure that all cores execute some code, as shown in the CTM logs # We currently don't check for the exact software execution, but the # output of the program as shown in the STM logs (below) ctmlogs = ['ctm004.log', 'ctm007.log', 'ctm010.log', 'ctm013.log'] for f in ctmlogs: # CTM log file exists assert tmpdir.join(f).isfile() # CTM log >= 0 bytes f_stat = os.stat(str(tmpdir.join(f))) assert f_stat.st_size > 0 # Ensure that the STMs log the expected printf() messages stmlogs = ['stm003.log', 'stm006.log', 'stm009.log', 'stm012.log'] for f in stmlogs: # STM log file exists assert tmpdir.join(f).isfile() # Check printf() program output in STM logs assert util.matches_golden_reference( str(tmpdir), f, filter_func=util.filter_stm_printf)