def test_sim_config(sim_engine, repeat4times): sim_config = SimConfig.SimConfig(u.CONFIG_FILE_PATH) sim_engine = sim_engine() for (k,v) in sim_config.config['settings']['regular'].items(): assert getattr(sim_engine.settings,k) == v
def create_sim_engine( diff_config = {}, force_initial_routing_and_scheduling_state = False, run_id = None ): engine = None sim_log = None sim_settings = None # add a finalizer def fin(): if engine: engine.connectivity.destroy() engine.destroy() if sim_log: sim_log.destroy() if sim_settings: sim_settings.destroy() request.addfinalizer(fin) # get default configuration sim_config = SimConfig.SimConfig(u.CONFIG_FILE_PATH) config = sim_config.settings['regular'] assert 'exec_numMotes' not in config config['exec_numMotes'] = sim_config.settings['combination']['exec_numMotes'][0] # update default configuration with parameters for (k,v) in list(diff_config.items()): assert k in config config.update(**diff_config) # create sim settings sim_settings = SimSettings.SimSettings(**config) sim_settings.setLogDirectory( '{0}-{1:03d}'.format( time.strftime('%Y%m%d-%H%M%S'), int(round(time.time() * 1000))%1000 ) ) sim_settings.setCombinationKeys([]) # create sim log sim_log = SimEngine.SimLog.SimLog() sim_log.set_log_filters('all') # do not log # create sim engine engine = SimEngine.SimEngine(run_id=run_id) # force initial routing and schedule, if appropriate if force_initial_routing_and_scheduling_state: set_initial_routing_and_scheduling_state(engine) return engine
def runSimCombinations(params): """ Runs simulations for all combinations of simulation settings. This function may run independently on different CPUs. """ cpuID = params['cpuID'] pid = params['pid'] numRuns = params['numRuns'] first_run = params['first_run'] verbose = params['verbose'] config_data = params['config_data'] simconfig = SimConfig.SimConfig(configdata=config_data) # record simulation start time simStartTime = time.time() # compute all the simulation parameter combinations combinationKeys = simconfig.settings.combination.keys() simParams = [] for p in itertools.product( *[simconfig.settings.combination[k] for k in combinationKeys]): simParam = {} for (k, v) in zip(combinationKeys, p): simParam[k] = v for (k, v) in simconfig.settings.regular.items(): if k not in simParam: simParam[k] = v simParams += [simParam] # run a simulation for each set of simParams for (simParamNum, simParam) in enumerate(simParams): # run the simulation runs for run_id in range(first_run, first_run + numRuns): # printOrLog output = 'parameters {0}/{1}, run {2}/{3}'.format( simParamNum + 1, len(simParams), run_id + 1 - first_run, numRuns) printOrLog(cpuID, pid, output, verbose) # create singletons settings = SimSettings.SimSettings(cpuID=cpuID, run_id=run_id, **simParam) settings.setLogDirectory(simconfig.get_log_directory_name()) settings.setCombinationKeys(combinationKeys) simlog = SimLog.SimLog() simlog.set_log_filters(simconfig.logging) parentlog = ParentLogs.ParentLogs() parentlog.set_log_filters(simconfig.logging) simengine = SimEngine.SimEngine(run_id=run_id, verbose=verbose) # start simulation run simengine.start() # wait for simulation run to end simengine.join() # destroy singletons simlog.destroy() #parentlog.destroy() simengine.destroy() Connectivity.Connectivity().destroy() settings.destroy() # destroy last, Connectivity needs it # printOrLog output = 'simulation ended after {0:.0f}s ({1} runs).'.format( time.time() - simStartTime, numRuns * len(simParams)) printOrLog(cpuID, pid, output, verbose)
def main(): #=== initialize # cli params cliparams = parseCliParams() # sim config simconfig = SimConfig.SimConfig(configfile=cliparams['config']) assert simconfig.version == 0 #=== run simulations # decide number of CPUs to run on multiprocessing.freeze_support() max_numCPUs = multiprocessing.cpu_count() if simconfig.execution.numCPUs == -1: numCPUs = max_numCPUs else: numCPUs = simconfig.execution.numCPUs assert numCPUs <= max_numCPUs if numCPUs == 1: # run on single CPU runSimCombinations({ 'cpuID': 0, 'pid': os.getpid(), 'numRuns': simconfig.execution.numRuns, 'first_run': 0, 'verbose': True, 'config_data': simconfig.get_config_data() }) else: # distribute runs on different CPUs runsPerCPU = [ int(math.floor( float(simconfig.execution.numRuns) / float(numCPUs))) ] * numCPUs idx = 0 while sum(runsPerCPU) < simconfig.execution.numRuns: runsPerCPU[idx] += 1 idx += 1 # distribute run ids on different CPUs (transform runsPerCPU into a list of tuples) first_run = 0 for cpuID in range(numCPUs): runs = runsPerCPU[cpuID] runsPerCPU[cpuID] = (runs, first_run) first_run += runs # print progress, wait until done cpuIDs = [i for i in range(numCPUs)] if simconfig.log_directory_name == 'hostname': # We assume the simulator run over a cluster system when # 'log_directory_name' is 'hostname'. Under a cluster system, we # disable "clear" on console because it could cause "'unknown': I # need something more specific." error. clear_console = False else: clear_console = True print_progress_thread = threading.Thread(target=printProgressPerCpu, args=(cpuIDs, os.getpid(), clear_console)) print_progress_thread.start() # wait for the thread ready while print_progress_thread.is_alive() == False: time.sleep(0.5) # start simulations pool = multiprocessing.Pool(numCPUs) async_result = pool.map_async( runSimCombinations, [{ 'cpuID': cpuID, 'pid': os.getpid(), 'numRuns': runs, 'first_run': first_run, 'verbose': False, 'config_data': simconfig.get_config_data() } for [cpuID, (runs, first_run)] in enumerate(runsPerCPU)]) # get() raises an exception raised by a thread if any try: async_result.get() except Exception: raise finally: # stop print_proress_thread if it's alive if print_progress_thread.is_alive(): global keep_printing_progress keep_printing_progress = False print_progress_thread.join() # cleanup hostname = platform.uname()[1] for i in range(numCPUs): os.remove(getTemplogFileName(i, os.getpid())) # merge output files folder_path = os.path.join('simData', simconfig.get_log_directory_name()) merge_output_files(folder_path) # copy config file into output directory with open(os.path.join(folder_path, 'config.json'), 'w') as f: f.write(simconfig.get_config_data()) #=== post-simulation actions if simconfig.log_directory_name == 'hostname': # We assume the simulator run over a cluster system when # 'log_directory_name' is 'hostname'. Under a cluster system, we # disable post actions. Users should perform post actions manually # after merging log files by mergeLogs.py. pass else: for c in simconfig.post: print('calling "{0}"').format(c) rc = subprocess.call(c, shell=True) assert rc == 0