def main(argv=None): ''' Executes the HPCC test suite on fout m1.xlarge instances. Total cores = 16 Total memory = 60GB HPCC N = sqrt(60000000 / 8) * .8 = 2190 ''' conf = FileConfiguration("/home/willmore/test.conf") ec2ConnFactory = EC2ConnectionFactory(conf.awsCred.access_key_id, conf.awsCred.secret_access_key, StdOutLogger()) masterRole = "master" slaveRole = "slave" masterOutDir = "./results/master/" slaveOutDir = "./results/slave/" sshCred = SSHCred('hpcc-user', '/home/willmore/.ssh/id_rsa_nopw') numCores = 16 doneFile = "/tmp/done.txt" startCmd = "mpirun -np %d -hostfile /tmp/d2c.context hpcc > experiment.out 2>&1; date > %s" % (numCores, doneFile) instanceType = InstanceType("m1.xlarge", 4, 7500, 1500, []) cloudHppcAMI = AMI(amiId="ami-295e685d") collectdSrc = "/var/lib/collectd/rrd/cloud-hpcc" deployment = Deployment("cloud_hpcc", ec2ConnFactory, roles=[ Role(masterRole, "master", ami=cloudHppcAMI, count=1, instanceType=instanceType, contextCred=sshCred, startActions=[UploadAction(source="./input/hpccinf.txt", destination="/home/hpcc-user/hpccinf.txt", sshCred=sshCred), AsyncAction(command=startCmd, sshCred=sshCred)], finishedChecks=[FileExistsFinishedCheck(fileName=doneFile, sshCred=sshCred)], dataCollectors=[DataCollector(source="/home/hpcc-user", destination=masterOutDir + "home_dir", sshCred=sshCred), DataCollector(source=collectdSrc, destination=masterOutDir + "collectd_stats", sshCred=sshCred)]), Role(slaveRole, "slave", ami=cloudHppcAMI, count=3, instanceType=instanceType, contextCred=sshCred, dataCollectors=[DataCollector(source="/home/hpcc-user", destination=slaveOutDir + "home_dir", sshCred=sshCred), DataCollector(source=collectdSrc, destination=slaveOutDir + "collectd_stats", sshCred=sshCred)]) ]) class Listener: def notify(self, event): print "Deployment state changed to: " + event.newState deployment.addAnyStateChangeListener(Listener()) deployment.run()
def main(argv=None): conf = TestConfig("/home/willmore/test.conf") ec2ConnFactory = EC2ConnectionFactory(conf.awsCred.access_key_id, conf.awsCred.secret_access_key, StdOutLogger()) masterRole = "master" slaveRole = "slave" masterOutDir = "./results/master/" slaveOutDir = "./results/slave/" sshCred = SSHCred('dirac', '/home/willmore/dirac.id_rsa') numHosts = 32 script = "/home/dirac/amAu111BF4/34.py" doneFile = "/tmp/done.txt" startCmd = "mpirun -np %d -hostfile /tmp/d2c.context gpaw-python %s > experiment.out 2>&1; date > %s" % (numHosts, script, doneFile) instanceType = InstanceType("m1.xlarge", 2, 7500, 850, []) fixSSHCmd = "echo \"Host *\n StrictHostKeyChecking no\" > .ssh/config" deployment = Deployment("34_py", ec2ConnFactory, roles=[ Role(masterRole, "master", ami=AMI(amiId="ami-1396a167"), count=1, instanceType=instanceType, contextCred=sshCred, startActions=[Action(command=fixSSHCmd, sshCred=sshCred), AsyncAction(command=startCmd, sshCred=sshCred)], finishedChecks=[FileExistsFinishedCheck(fileName=doneFile, sshCred=sshCred)], dataCollectors=[DataCollector(source="/home/dirac", destination=masterOutDir + "home_dir", sshCred=sshCred), DataCollector(source="/opt/collectd/var/lib/collectd", destination=masterOutDir + "collectd_stats", sshCred=sshCred)]), Role(slaveRole, "slave", ami=AMI(amiId="ami-1396a167"), count=7, instanceType=instanceType, contextCred=sshCred, dataCollectors=[DataCollector(source="/home/dirac", destination=slaveOutDir + "home_dir", sshCred=sshCred), DataCollector(source="/opt/collectd/var/lib/collectd", destination=slaveOutDir + "collectd_stats", sshCred=sshCred)]) ]) class Listener: def notify(self, event): print "Deployment state changed to: " + event.newState deployment.addAnyStateChangeListener(Listener()) deployment.run()