scenario.scenID = stamp # save scenario model instance with timestamp with open(os.path.join(tempDir,str(stamp)+'.pckl'), 'w') as f: pickle.dump(scenario, f) return scenario.scenID if __name__ == "__main__": # set up path rootPath = pathToRepository script = argv[0] _scenType = argv[1] _numFollowers = int(argv[2]) _numAggregators = int(argv[3]) _aggPerFollower = int(argv[4]) _policy = argv[5] _numFacts = int(argv[6]) _valRange = int(argv[7]) _ruleScenario = argv[8] _numPeers = int(argv[9]) scenarios = driver.simpleMAF(_scenType, _numFollowers, _numAggregators, _aggPerFollower, _policy, _numFacts, _valRange, _ruleScenario, _numPeers) # create a list of scenario instances for s in scenarios: generateScenarioFiles( s, rootPath ) # generate each scenario #sys.path.append(os.path.join(rootPath,'/webdamlog-exp')) driver.localSVNCommit( commitPath ) # commit the results exit()
pickle.dump(scenario, f) return scenario.scenID if __name__ == "__main__": # set up path rootPath = pathToRepository script = argv[0] _scenType = argv[1] _numFollowers = int(argv[2]) _numAggregators = int(argv[3]) _aggPerFollower = int(argv[4]) _policy = argv[5] _numFacts = int(argv[6]) _valRange = int(argv[7]) _ruleScenario = argv[8] _numPeers = int(argv[9]) scenarios = driver.simpleMAF( _scenType, _numFollowers, _numAggregators, _aggPerFollower, _policy, _numFacts, _valRange, _ruleScenario, _numPeers) # create a list of scenario instances for s in scenarios: generateScenarioFiles(s, rootPath) # generate each scenario #sys.path.append(os.path.join(rootPath,'/webdamlog-exp')) driver.localSVNCommit(commitPath) # commit the results exit()
def executeScenario(pathToRepository, scenID, scenType, mode, timeToRun, masterDelay): stamp = int(time.time() * 1000) # construct execution record, to be pickled execution = models.Execution( \ execID = stamp, \ scenID = scenID, \ timeToRun = timeToRun, \ mode = mode, \ build = build ) # numTicks is now ignored; remove # this is the directory containing the scenario: e.g. webdamlog-exp/MAF/1385388824301 scenPath = os.path.join('webdamlog-exp', scenType, str(scenID)) # make sure scenario path exists locally localScenPath = os.path.join(pathToRepository, scenPath) assert os.path.exists(localScenPath) # this is the path to dir where output will be written (by remote peers later) # e.g. webdamlog-exp/MAF/1385388824301/exec_str99999999999d execPath = os.path.join(scenPath, 'exec_' + str(stamp)) localExecPath = os.path.join(os.path.join(pathToRepository, execPath)) # create directory for execution within scenario dir, svn add and commit os.makedirs(localExecPath) with open(os.path.join(localExecPath, str(stamp) + '.pckl'), 'w') as f: pickle.dump(execution, f) os.makedirs(os.path.join(localExecPath, 'bench_files') ) # also create this directory to avoid svn conflict at peers driver.localSVNCommit(localScenPath) print localScenPath # inspect scenario for 'out_' directories, infer hosts outs = [] for output in glob.glob(os.path.join(localScenPath, 'out_*')): outs.append(output) print outs outKey = os.path.split(outs[0])[1].split('_')[ 2] # this gets common key from name of out* directories print outKey hosts = [] masterHost = None for out in outs: extractedHostName = os.path.split(out)[1].split('_')[1] if (len(glob.glob(os.path.join(out, 'run_master*')))) == 1: masterHost = extractedHostName if (len(glob.glob(os.path.join(out, 'run_sue*')))) == 1: masterHost = extractedHostName hosts.append(extractedHostName) assert (masterHost != None) execution.success = True start = time.time() env.hosts = hosts env.parallel = True # execute on each host in parallel try: execute(fab.pull_svn, scenPath=scenPath ) # each host should pull latest code and latest exp except: print >> sys.stderr, 'Pull failed: ', sys.exc_info()[0] execution.success = False #only try to run ruby if the pull of files worked if execution.success == True: # prepare parameters for ruby script paramString = '' print "the mode which is set is", mode accessBool = mode & 1 optim1Bool = mode & 2 optim2Bool = mode & 4 if (accessBool): paramString += 'access' + ' ' if (optim1Bool): paramString += 'optim1' + ' ' if (optim2Bool): paramString += 'optim2' + ' ' # run on all hosts try: execute(fab.run_ruby, execPath=execPath, scenPath=scenPath, paramString=paramString, outKey=str(outKey)) except CommandTimeout: execution.success = False #don't want to check the files in, this run is invalid return except: print >> sys.stderr, 'Execution failed: ', sys.exc_info()[0] execution.success = False try: execute(fab.run_commit, execPath=execPath) except: print >> sys.stderr, 'Failed to commit files, please add them manually: ', sys.exc_info( )[0] #this does not change the execution success execution.runTime = time.time() - start # pickle object with open(os.path.join(localExecPath, str(stamp) + '.pckl'), 'w') as f: pickle.dump(execution, f) # refresh database for this execution # execute(fab.pull_svn) # make sure files generated at all hosts are in # loadBenchmark.processExecs( scenID, localExecPath) driver.localSVNCommit(localScenPath) return execution.execID
def executeScenario( pathToRepository, scenID, scenType, mode, timeToRun, masterDelay ): stamp = int(time.time()*1000) # construct execution record, to be pickled execution = models.Execution( \ execID = stamp, \ scenID = scenID, \ timeToRun = timeToRun, \ mode = mode, \ build = build ) # numTicks is now ignored; remove # this is the directory containing the scenario: e.g. webdamlog-exp/MAF/1385388824301 scenPath = os.path.join('webdamlog-exp',scenType,str(scenID)) # make sure scenario path exists locally localScenPath = os.path.join(pathToRepository,scenPath) assert os.path.exists(localScenPath) # this is the path to dir where output will be written (by remote peers later) # e.g. webdamlog-exp/MAF/1385388824301/exec_str99999999999d execPath = os.path.join(scenPath,'exec_'+str(stamp)) localExecPath = os.path.join( os.path.join(pathToRepository,execPath)) # create directory for execution within scenario dir, svn add and commit os.makedirs(localExecPath) with open(os.path.join(localExecPath,str(stamp)+'.pckl'), 'w') as f: pickle.dump(execution, f) os.makedirs(os.path.join(localExecPath,'bench_files')) # also create this directory to avoid svn conflict at peers driver.localSVNCommit(localScenPath) print localScenPath # inspect scenario for 'out_' directories, infer hosts outs =[] for output in glob.glob( os.path.join(localScenPath,'out_*')): outs.append(output) print outs outKey = os.path.split(outs[0])[1].split('_')[2] # this gets common key from name of out* directories print outKey hosts = [] masterHost = None for out in outs: extractedHostName = os.path.split(out)[1].split('_')[1] if (len(glob.glob(os.path.join(out,'run_master*')))) == 1: masterHost = extractedHostName if (len(glob.glob(os.path.join(out,'run_sue*')))) == 1: masterHost = extractedHostName hosts.append(extractedHostName) assert(masterHost != None) execution.success = True start = time.time() env.hosts = hosts env.parallel = True # execute on each host in parallel try: execute(fab.pull_svn, scenPath=scenPath) # each host should pull latest code and latest exp except: print >> sys.stderr, 'Pull failed: ', sys.exc_info()[0] execution.success = False #only try to run ruby if the pull of files worked if execution.success == True: # prepare parameters for ruby script paramString = '' print "the mode which is set is", mode accessBool = mode & 1 optim1Bool = mode & 2 optim2Bool = mode & 4 if (accessBool): paramString += 'access'+' ' if (optim1Bool): paramString += 'optim1'+' ' if (optim2Bool): paramString += 'optim2'+' ' # run on all hosts try: execute(fab.run_ruby, execPath=execPath, scenPath=scenPath, paramString=paramString, outKey=str(outKey)) except CommandTimeout: execution.success = False #don't want to check the files in, this run is invalid return except: print >> sys.stderr, 'Execution failed: ', sys.exc_info()[0] execution.success = False try: execute(fab.run_commit, execPath=execPath) except: print >> sys.stderr, 'Failed to commit files, please add them manually: ', sys.exc_info()[0] #this does not change the execution success execution.runTime = time.time() - start # pickle object with open(os.path.join(localExecPath,str(stamp)+'.pckl'), 'w') as f: pickle.dump(execution, f) # refresh database for this execution # execute(fab.pull_svn) # make sure files generated at all hosts are in # loadBenchmark.processExecs( scenID, localExecPath) driver.localSVNCommit(localScenPath) return execution.execID
def run(configFile): config = ConfigParser.ConfigParser() config.read(configFile) rootPath = os.environ["HOME"] + "/"+ config.get('environment', 'rootPath') scenType = config.get('default', 'scenarioType') scenarioList = [] if scenType == 'MAF': # set-valued parameters (space delimited in config file) policyList = config.get('scenarioMAF', 'policy').split(' ') ruleScenarioList = config.get('scenarioMAF', 'ruleScenario').split(' ') numFollowersList = config.get('scenarioMAF', 'numFollowers').split(' ') numAggregatorsList = config.get('scenarioMAF', 'numAggregators').split(' ') numAgPerFollowerList = config.get('scenarioMAF', 'aggPerFollower').split(' ') numFactsList = config.get('scenarioMAF', 'numFacts').split(' ') # this forms the crossproduct of all set-valued parameters for tup in itertools.product(policyList, ruleScenarioList, numFollowersList, numAggregatorsList, numAgPerFollowerList, numFactsList): print tup scenario = models.Scenario( \ # scenID = _ _ _ (filled in later) scenType = 'MAF', \ numFollowers = int(tup[2]), \ numAggregators = int(tup[3]), \ aggPerFollower = int(tup[4]), \ policy = tup[0], \ numFacts = int(tup[5]), \ ruleScenario = tup[1], \ #valRange = config.getint('scenarioMAF', 'valRange'), \ valRange = tup[5], \ numExtraCols = config.getint('scenarioMAF', 'numExtraCols'), \ numHosts = config.getint('scenarioMAF', 'numHosts'), \ hosts = config.get('scenarioMAF', 'hosts').split(' '), \ numPeersPerHost = config.getint('scenarioMAF', 'numPeersPerHost') ) scenarioList.append(scenario) if scenType == 'PA': policyList = config.get('scenarioPA', 'policy').split(' ') networkFileList = config.get('scenarioPA', 'networkFile').split(' ') numFactsList = config.get('scenarioPA', 'numFacts').split(' ') for tup in itertools.product(policyList, networkFileList, numFactsList): print tup numf = int(tup[1].split('-')[1][1:])-3 numh = config.getint('scenarioPA', 'numHosts') scenario = models.Scenario( \ scenType = 'PA', \ numFollowers = numf, \ numAggregators = 0, \ aggPerFollower = 0, \ policy = tup[0], \ numFacts = int(tup[2]), \ ruleScenario = 'PA', \ valRange = int(tup[2]), \ numExtraCols = 0, \ numHosts = numh, \ hosts = config.get('scenarioPA', 'hosts').split(' '), \ numPeersPerHost = numf/(numh-1)+1, \ networkFile = tup[1] ) scenarioList.append(scenario) print '*** Checking / creating %i scenarios...' % len(scenarioList) # get scenario IDs after matching or creating scenarios scenIDList = matchOrCreateScenario(scenarioList, rootPath) driver.localSVNCommit( os.path.join(rootPath, 'webdamlog-exp') ) # start on executions... # set-valued execution parameters (space delimited in config file) accessCList = config.get('execution', 'accessControl').split(' ') for scenID in scenIDList: # run all the executions, for each scenID for run in range( config.getint('execution', 'numRuns') ): print 'Running executions for scenID %i' % scenID for tup in accessCList: print 'the string is:', tup mode = int(tup,2) print 'mode is *****', mode execID = execution.executeScenario( rootPath, scenID, scenType, mode, \ config.getfloat('execution', 'timeToRun'), config.getfloat('execution', 'masterDelay') ) print '*** Finished run %i of execution %i.' % (run, execID) sleep(30) print '*** Done with executions.'