def test_complex_good(self): tm = taskrun.TaskManager( observers=[taskrun.VerboseObserver(summary=False)], resource_manager=taskrun.ResourceManager( taskrun.CounterResource('cpu', 10, 64), taskrun.CounterResource('mem', 1024 * 1024 * 1024, 256 * 1024 * 1024 * 1024), taskrun.CounterResource('net', 1000, 10000), taskrun.CounterResource('fds', 500, 50000)), failure_mode=taskrun.FailureMode.AGGRESSIVE_FAIL) tm.run_tasks()
def main(args): # ensure rundir exists, if not make it if not os.path.isdir(args.rundir): try: os.mkdir(args.rundir) except: error('couldn\'t create {0}'.format(args.rundir)) # ensure the supersim environment exists if not os.path.isdir(args.ssenv): error('{0} does not exist'.format(args.ssenv)) # create a task manager to handle all tasks rm = taskrun.ResourceManager( taskrun.CounterResource('cpus', 9999, args.cpus), taskrun.MemoryResource('mem', 9999, args.mem)) ob = taskrun.VerboseObserver(description=args.verbose) tm = taskrun.TaskManager(resource_manager=rm, observer=ob, failure_mode=taskrun.FailureMode.ACTIVE_CONTINUE) # sweep params sweepStart = 1 sweepStop = 100 sweepStep = 9 loads = [ '{0:.02f}'.format(x / 100) for x in range(sweepStart, sweepStop + 1, sweepStep) ] # sweep load parse_tasks = [] for load in loads: runId = '{0:02d}_{1}'.format(args.queue, load) # create file names supersim_bin = os.path.join(args.ssenv, 'poem_supersim', 'bin', 'supersim') sslatency_bin = os.path.join(args.ssenv, 'sslatency', 'bin', 'sslatency') settings_json = os.path.join(args.ssenv, 'poem_supersim', 'json', 'hierarchyhyperx_iq_GlueFactory1.json') simout_log = os.path.join(args.rundir, 'simout_{0}.log'.format(runId)) messages_mpf = os.path.join(args.rundir, 'messages_{0}.mpf'.format(runId)) rates_csv = os.path.join(args.rundir, 'rates_{0}.csv'.format(runId)) channels_csv = os.path.join(args.rundir, 'channels_{0}.mpf'.format(runId)) messages_csv = os.path.join(args.rundir, 'messages_{0}.csv'.format(runId)) packets_csv = os.path.join(args.rundir, 'packets_{0}.csv'.format(runId)) aggregate_csv = os.path.join(args.rundir, 'aggregate_{0}.csv'.format(runId)) packets_png = os.path.join(args.rundir, 'packets_{0}.png'.format(runId)) # create simulation task sim_cmd = ('{0} {1} ' 'application.max_injection_rate=float={2} ' 'application.message_log.file=string={3} ' 'application.rate_log.file=string={4} ' 'network.channel_log.file=string={5} ' 'network.interface.init_credits=uint={6} ' 'network.router.input_queue_depth=uint={6} ').format( supersim_bin, settings_json, load, messages_mpf, rates_csv, channels_csv, args.queue) sim_task = taskrun.ProcessTask(tm, 'sim_{0}'.format(runId), sim_cmd) sim_task.stdout_file = simout_log sim_task.stderr_file = simout_log sim_task.resources = {'cpus': 1, 'mem': 10} sim_task.add_condition( taskrun.FileModificationCondition( [settings_json], [simout_log, messages_mpf, rates_csv, channels_csv])) # create parser task parse_cmd = '{0} -m {1} -p {2} -a {3} {4}'.format( sslatency_bin, messages_csv, packets_csv, aggregate_csv, messages_mpf) parse_task = taskrun.ProcessTask(tm, 'parse_{0}'.format(runId), parse_cmd) parse_task.resources = {'cpus': 1, 'mem': 2} parse_task.add_dependency(sim_task) parse_task.add_condition( taskrun.FileModificationCondition( [messages_mpf], [messages_csv, packets_csv, aggregate_csv])) parse_tasks.append(parse_task) # create plot task plot_cmd = 'sslqp {0} {1}'.format(packets_csv, packets_png) plot_task = taskrun.ProcessTask(tm, 'plot_{0}'.format(runId), plot_cmd) plot_task.resources = {'cpus': 1, 'mem': 2} plot_task.add_dependency(parse_task) plot_task.add_condition( taskrun.FileModificationCondition([packets_csv], [packets_png])) # create a task to make a load latency graph loadlat_file = os.path.join(args.rundir, 'load_latency_{0:02d}.png'.format(args.queue)) loadlat_cmd = ('ssllp --row Packet --title "QueueSize={0}" ' '{1} {2} {3} {4}').format(args.queue, loadlat_file, sweepStart, sweepStop + 1, sweepStep) agg_files = [] for load in loads: runId = '{0:02d}_{1}'.format(args.queue, load) aggregate_csv = os.path.join(args.rundir, 'aggregate_{0}.csv'.format(runId)) loadlat_cmd += ' {0}'.format(aggregate_csv) agg_files.append(aggregate_csv) loadlat_task = taskrun.ProcessTask(tm, 'loadlat_{0}'.format(args.queue), loadlat_cmd) loadlat_task.resources = {'cpus': 1, 'mem': 2} for dep in parse_tasks: loadlat_task.add_dependency(dep) loadlat_task.add_condition( taskrun.FileModificationCondition(agg_files, [loadlat_file])) # run all tasks tm.run_tasks()
help='supersim binary to run') ap.add_argument('--ssparse', type=str, default='../ssparse/', help='ssparse directory') ap.add_argument('--settings', type=str, default='json/fattree_iq_blast.json', help='settings file to use') args = ap.parse_args() # get the current amount of resources cpus = os.cpu_count() mem = taskrun.MemoryResource.current_available_memory_gib(); # build the task manager rm = taskrun.ResourceManager(taskrun.CounterResource('cpus', 9999, cpus), taskrun.MemoryResource('mem', 9999, mem)) cob = taskrun.FileCleanupObserver() vob = taskrun.VerboseObserver(description=False, summary=True) tm = taskrun.TaskManager(resource_manager=rm, observers=[cob, vob], failure_mode=taskrun.FailureMode.AGGRESSIVE_FAIL) # output directory out_dir = args.directory if not os.path.isdir(out_dir): os.mkdir(out_dir) # create task and resources function def set_task_function(tm, name, cmd, console_out, task_type, config): task = taskrun.ProcessTask(tm, name, cmd) if console_out: task.stdout_file = console_out task.stderr_file = console_out
ap.add_argument('-g', '--granularity', type=int, default=6, help='the granularity of the injection rate sweeps') args = ap.parse_args() ############################################################################### # SETUP # get the current amount of resources cpus = os.cpu_count() mem = taskrun.MemoryResource.current_available_memory_gib(); # build the task manager rm = taskrun.ResourceManager(taskrun.CounterResource('cpus', 9999, cpus), taskrun.MemoryResource('mem', 9999, mem)) cob = taskrun.FileCleanupObserver() vob = taskrun.VerboseObserver() tm = taskrun.TaskManager( resource_manager=rm, observers=[cob, vob], failure_mode=taskrun.FailureMode.AGGRESSIVE_FAIL) # output directory out_dir = args.directory if not os.path.isdir(out_dir): os.mkdir(out_dir) # generate an array for the loads to be simulated sweep_start = 0 sweep_stop = 100 sweep_step = args.granularity loads = ['{0:.02f}'.format(x/100)
def main(args): total_cpus = args.cpus if total_cpus == None: total_cpus = os.cpu_count() total_mem = args.mem if total_mem == None: psmem = psutil.virtual_memory() total_mem = math.floor((psmem.free + psmem.cached) / (1024 * 1024 * 1024)) print('using up to {0} CPUs'.format(total_cpus)) print('using up to {0} GiB of memory'.format(total_mem)) if args.check: subprocess.check_call('valgrind -h', shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) rm = taskrun.ResourceManager( taskrun.CounterResource('cpu', 9999, total_cpus), taskrun.MemoryResource('mem', 9999, total_mem)) vob = taskrun.VerboseObserver() cob = taskrun.FileCleanupObserver() tm = taskrun.TaskManager(resource_manager=rm, observers=[vob,cob], failure_mode='passive_fail') # find all files settingsFiles = glob.glob('json/{0}.json'.format(args.glob)) print('config files to test: {0}'.format(settingsFiles)) # generate all tasks for settingsFile in settingsFiles: cmd = 'bin/supersim {0}'.format(settingsFile) if args.check: cmd = ('valgrind --log-fd=1 --leak-check=full --show-reachable=yes ' '--track-origins=yes --track-fds=yes {0}'.format(cmd)) log = logFile(settingsFile) if not args.skip: try: os.remove(log) except OSError: pass cmd = '{0} 2>&1 | tee {1}'.format(cmd, log) task = taskrun.ProcessTask(tm, settingsFile, cmd) task.resources = {'cpu': 1, 'mem': 10 if args.check else 3} # run tasks if args.skip: print('skipping simulations') else: print('running simulations') tm.run_tasks() print('done') # check output for failures anyError = False for settingsFile in settingsFiles: error = False print('analyzing {0} output'.format(settingsFile)) # read in text log = logFile(settingsFile) with open(log, 'r') as fd: lines = fd.readlines(); # analyze output simComplete = False for idx, line in enumerate(lines): if line.find('Simulation complete') >= 0: simComplete = True if args.check: if (line.find('Open file descriptor') >= 0 and lines[idx+1].find('inherited from parent') < 0): error = True bad('open file descriptor') if line.find('blocks are definitely lost') >= 0: error = True bad('definitely lost memory') if line.find('blocks are indirectly lost') >= 0: error = True bad('indirectly lost memory') if (line.find('blocks are still reachable') >= 0 and # TODO(nic): REMOVE ME WHEN G++ STOPS SUCKING not line.find('72,704 bytes') >= 0): error = True bad('still reachable memory') if line.find('depends on uninitialised value') >= 0: error = True bad('depends on uninitialised value') if not simComplete: error = True; bad('no "Simulation complete" message') # show status if error: anyError = True else: good('passed all tests') return 0 if not anyError else -1