def main(argv=None):
'''
Executes the HPCC test suite on fout m1.xlarge instances.
Total cores = 16
Total memory = 60GB
HPCC N = sqrt(60000000 / 8) * .8 = 2190
'''
conf = FileConfiguration("/home/willmore/test.conf")
ec2ConnFactory = EC2ConnectionFactory(conf.awsCred.access_key_id,
conf.awsCred.secret_access_key,
StdOutLogger())
masterRole = "master"
slaveRole = "slave"
masterOutDir = "./results/master/"
slaveOutDir = "./results/slave/"
sshCred = SSHCred('hpcc-user', '/home/willmore/.ssh/id_rsa_nopw')
numCores = 16
doneFile = "/tmp/done.txt"
startCmd = "mpirun -np %d -hostfile /tmp/d2c.context hpcc > experiment.out 2>&1; date > %s" % (numCores, doneFile)
instanceType = InstanceType("m1.xlarge", 4, 7500, 1500, [])
cloudHppcAMI = AMI(amiId="ami-295e685d")
collectdSrc = "/var/lib/collectd/rrd/cloud-hpcc"
deployment = Deployment("cloud_hpcc",
ec2ConnFactory,
roles=[
Role(masterRole, "master",
ami=cloudHppcAMI,
count=1,
instanceType=instanceType,
contextCred=sshCred,
startActions=[UploadAction(source="./input/hpccinf.txt",
destination="/home/hpcc-user/hpccinf.txt",
sshCred=sshCred),
AsyncAction(command=startCmd,
sshCred=sshCred)],
finishedChecks=[FileExistsFinishedCheck(fileName=doneFile,
sshCred=sshCred)],
dataCollectors=[DataCollector(source="/home/hpcc-user",
destination=masterOutDir + "home_dir",
sshCred=sshCred),
DataCollector(source=collectdSrc,
destination=masterOutDir + "collectd_stats",
sshCred=sshCred)]),
Role(slaveRole, "slave",
ami=cloudHppcAMI,
count=3,
instanceType=instanceType,
contextCred=sshCred,
dataCollectors=[DataCollector(source="/home/hpcc-user",
destination=slaveOutDir + "home_dir",
sshCred=sshCred),
DataCollector(source=collectdSrc,
destination=slaveOutDir + "collectd_stats",
sshCred=sshCred)])
])
class Listener:
def notify(self, event):
print "Deployment state changed to: " + event.newState
deployment.addAnyStateChangeListener(Listener())
deployment.run()
def main(argv=None):
conf = TestConfig("/home/willmore/test.conf")
ec2ConnFactory = EC2ConnectionFactory(conf.awsCred.access_key_id,
conf.awsCred.secret_access_key,
StdOutLogger())
masterRole = "master"
slaveRole = "slave"
masterOutDir = "./results/master/"
slaveOutDir = "./results/slave/"
sshCred = SSHCred('dirac', '/home/willmore/dirac.id_rsa')
numHosts = 32
script = "/home/dirac/amAu111BF4/34.py"
doneFile = "/tmp/done.txt"
startCmd = "mpirun -np %d -hostfile /tmp/d2c.context gpaw-python %s > experiment.out 2>&1; date > %s" % (numHosts, script, doneFile)
instanceType = InstanceType("m1.xlarge", 2, 7500, 850, [])
fixSSHCmd = "echo \"Host *\n StrictHostKeyChecking no\" > .ssh/config"
deployment = Deployment("34_py",
ec2ConnFactory,
roles=[
Role(masterRole, "master",
ami=AMI(amiId="ami-1396a167"),
count=1,
instanceType=instanceType,
contextCred=sshCred,
startActions=[Action(command=fixSSHCmd,
sshCred=sshCred),
AsyncAction(command=startCmd,
sshCred=sshCred)],
finishedChecks=[FileExistsFinishedCheck(fileName=doneFile,
sshCred=sshCred)],
dataCollectors=[DataCollector(source="/home/dirac",
destination=masterOutDir + "home_dir",
sshCred=sshCred),
DataCollector(source="/opt/collectd/var/lib/collectd",
destination=masterOutDir + "collectd_stats",
sshCred=sshCred)]),
Role(slaveRole, "slave",
ami=AMI(amiId="ami-1396a167"),
count=7,
instanceType=instanceType,
contextCred=sshCred,
dataCollectors=[DataCollector(source="/home/dirac",
destination=slaveOutDir + "home_dir",
sshCred=sshCred),
DataCollector(source="/opt/collectd/var/lib/collectd",
destination=slaveOutDir + "collectd_stats",
sshCred=sshCred)])
])
class Listener:
def notify(self, event):
print "Deployment state changed to: " + event.newState
deployment.addAnyStateChangeListener(Listener())
deployment.run()
