def main(argv=None): ''' End-to-End integration test of deployment lifecycle. Uses one micro, EBS-backed instance, ''' SQLITE_FILE = "%s/.d2c_test/deploy_test.sqlite" % os.path.expanduser('~') if os.path.exists(SQLITE_FILE): print "Deleting existing DB" os.unlink(SQLITE_FILE) dao = DAO(SQLITE_FILE) conf = TestConfig("/home/willmore/test.conf") dao.saveConfiguration(conf) ec2ConnFactory = EC2ConnectionFactory(conf.awsCred.access_key_id, conf.awsCred.secret_access_key, StdOutLogger()) #credStore = CredStore(dao) testDir = "/tmp/d2c/%s/" % random.randint(0, 1000) sshCred = SSHCred('dirac', '/home/willmore/dirac.id_rsa') deployment = Deployment("dummyDep", ec2ConnFactory, roles=[ Role("dummyDep", "loner", ami=AMI(amiId="ami-1396a167"), count=1, instanceType=InstanceType.M1_LARGE, startActions=[Action(command="echo howdy > /tmp/howdy.txt", sshCred=sshCred)], finishedChecks=[FileExistsFinishedCheck(fileName="/tmp/howdy.txt", sshCred=sshCred)], #stopActions=[Action(command="sudo service collectd stop", # sshCred=sshCred)], dataCollectors=[DataCollector(source="/tmp/howdy.txt", destination=testDir + "howdy.txt", sshCred=sshCred), DataCollector(source="/opt/collectd/var/lib/collectd", destination=testDir + "collectd_stats", sshCred=sshCred)])]) class Listener: def notify(self, event): print "Deployment state changed to: " + event.newState deployment.addAnyStateChangeListener(Listener()) thread = Thread(target=deployment.run) thread.start() thread.join()
def addDeployment(self, event): roles = self.wizard.roleWizard.getPanel("ROLES").roleList.getItems() deployment = Deployment(self.newName, roles=roles, awsCred=self.dao.getAWSCred("mainKey")) deployment.setCloud(self.cloud) self.dao.save(deployment) wx.CallAfter(Publisher().sendMessage, "DEPLOYMENT CREATED", {"deployment": deployment}) self.wizard.container.showPanel("COMPLETION")
def main(argv=None): conf = TestConfig("/home/willmore/test.conf") StdOutLogger().write("test123") ec2ConnFactory = EC2ConnectionFactory(conf.awsCred.access_key_id, conf.awsCred.secret_access_key, StdOutLogger()) masterOutDir = "./master/" sshCred = SSHCred('dirac', '/home/willmore/dirac.id_rsa') numHosts = 4 script = "/home/dirac/amAu111BF4/22.py" doneFile = "/tmp/done.txt" startCmd = "nohup mpirun -np %d -hostfile /tmp/d2c.context gpaw-python %s &> experiment.out || true && date > %s &" % (numHosts, script, doneFile) instanceType = InstanceType("m1.xlarge", 2, 7500, 850, []) deployment = Deployment("22_py", ec2ConnFactory, roles=[ Role("22_py", "master", ami=AMI(amiId="ami-1396a167"), count=1, instanceType=instanceType, contextCred=sshCred, startActions=[Action(command=startCmd, sshCred=sshCred)], finishedChecks=[FileExistsFinishedCheck(fileName=doneFile, sshCred=sshCred)], dataCollectors=[DataCollector(source="/home/dirac", destination=masterOutDir + "home_dir", sshCred=sshCred), DataCollector(source="/opt/collectd/var/lib/collectd", destination=masterOutDir + "collectd_stats", sshCred=sshCred)])]) class Listener: def notify(self, event): print "Deployment state changed to: " + event.newState deployment.addAnyStateChangeListener(Listener()) thread = Thread(target=deployment.run) thread.start() thread.join()
def setUp(self): dName = "Dummy" ami = AMI("ami-123", "foobar.vdi") dao = mock(DAO) conf = mock(Configuration) cred = mock(EC2Cred) conf.ec2Cred = cred when(dao).getEC2Cred().thenReturn(cred) when(dao).getConfiguration().thenReturn(conf) ''' self.deployment = Deployment(dName, roles = [Role(dName, "loner", ami, 2, dao=dao, startActions=[DummyAction(), DummyAction()], finishedChecks=[DummyFinishedTest(), DummyFinishedTest()]), Role(dName, "loner2", ami, 2, dao=dao, startActions=[DummyAction(), DummyAction()], finishedChecks=[DummyFinishedTest(), DummyFinishedTest()], dataCollectors=[Mock(DataCollector)])]) ''' roles = [] for _ in range(3): role = mock(Role) when(role).checkFinished().thenReturn(True) self.deployment = Deployment(dName, roles=roles)
def main(argv=None): ''' Executes the HPCC test suite on fout m1.xlarge instances. Total cores = 16 Total memory = 60GB HPCC N = sqrt(60000000 / 8) * .8 = 2190 ''' conf = FileConfiguration("/home/willmore/test.conf") ec2ConnFactory = EC2ConnectionFactory(conf.awsCred.access_key_id, conf.awsCred.secret_access_key, StdOutLogger()) masterRole = "master" slaveRole = "slave" masterOutDir = "./results/master/" slaveOutDir = "./results/slave/" sshCred = SSHCred('hpcc-user', '/home/willmore/.ssh/id_rsa_nopw') numCores = 16 doneFile = "/tmp/done.txt" startCmd = "mpirun -np %d -hostfile /tmp/d2c.context hpcc > experiment.out 2>&1; date > %s" % (numCores, doneFile) instanceType = InstanceType("m1.xlarge", 4, 7500, 1500, []) cloudHppcAMI = AMI(amiId="ami-295e685d") collectdSrc = "/var/lib/collectd/rrd/cloud-hpcc" deployment = Deployment("cloud_hpcc", ec2ConnFactory, roles=[ Role(masterRole, "master", ami=cloudHppcAMI, count=1, instanceType=instanceType, contextCred=sshCred, startActions=[UploadAction(source="./input/hpccinf.txt", destination="/home/hpcc-user/hpccinf.txt", sshCred=sshCred), AsyncAction(command=startCmd, sshCred=sshCred)], finishedChecks=[FileExistsFinishedCheck(fileName=doneFile, sshCred=sshCred)], dataCollectors=[DataCollector(source="/home/hpcc-user", destination=masterOutDir + "home_dir", sshCred=sshCred), DataCollector(source=collectdSrc, destination=masterOutDir + "collectd_stats", sshCred=sshCred)]), Role(slaveRole, "slave", ami=cloudHppcAMI, count=3, instanceType=instanceType, contextCred=sshCred, dataCollectors=[DataCollector(source="/home/hpcc-user", destination=slaveOutDir + "home_dir", sshCred=sshCred), DataCollector(source=collectdSrc, destination=slaveOutDir + "collectd_stats", sshCred=sshCred)]) ]) class Listener: def notify(self, event): print "Deployment state changed to: " + event.newState deployment.addAnyStateChangeListener(Listener()) deployment.run()
def main(argv=None): conf = TestConfig("/home/willmore/test.conf") ec2ConnFactory = EC2ConnectionFactory(conf.awsCred.access_key_id, conf.awsCred.secret_access_key, StdOutLogger()) masterRole = "master" slaveRole = "slave" masterOutDir = "./results/master/" slaveOutDir = "./results/slave/" sshCred = SSHCred('dirac', '/home/willmore/dirac.id_rsa') numHosts = 32 script = "/home/dirac/amAu111BF4/34.py" doneFile = "/tmp/done.txt" startCmd = "mpirun -np %d -hostfile /tmp/d2c.context gpaw-python %s > experiment.out 2>&1; date > %s" % (numHosts, script, doneFile) instanceType = InstanceType("m1.xlarge", 2, 7500, 850, []) fixSSHCmd = "echo \"Host *\n StrictHostKeyChecking no\" > .ssh/config" deployment = Deployment("34_py", ec2ConnFactory, roles=[ Role(masterRole, "master", ami=AMI(amiId="ami-1396a167"), count=1, instanceType=instanceType, contextCred=sshCred, startActions=[Action(command=fixSSHCmd, sshCred=sshCred), AsyncAction(command=startCmd, sshCred=sshCred)], finishedChecks=[FileExistsFinishedCheck(fileName=doneFile, sshCred=sshCred)], dataCollectors=[DataCollector(source="/home/dirac", destination=masterOutDir + "home_dir", sshCred=sshCred), DataCollector(source="/opt/collectd/var/lib/collectd", destination=masterOutDir + "collectd_stats", sshCred=sshCred)]), Role(slaveRole, "slave", ami=AMI(amiId="ami-1396a167"), count=7, instanceType=instanceType, contextCred=sshCred, dataCollectors=[DataCollector(source="/home/dirac", destination=slaveOutDir + "home_dir", sshCred=sshCred), DataCollector(source="/opt/collectd/var/lib/collectd", destination=slaveOutDir + "collectd_stats", sshCred=sshCred)]) ]) class Listener: def notify(self, event): print "Deployment state changed to: " + event.newState deployment.addAnyStateChangeListener(Listener()) deployment.run()
class DeploymentTest(unittest.TestCase): def setUp(self): dName = "Dummy" ami = AMI("ami-123", "foobar.vdi") dao = mock(DAO) conf = mock(Configuration) cred = mock(EC2Cred) conf.ec2Cred = cred when(dao).getEC2Cred().thenReturn(cred) when(dao).getConfiguration().thenReturn(conf) ''' self.deployment = Deployment(dName, roles = [Role(dName, "loner", ami, 2, dao=dao, startActions=[DummyAction(), DummyAction()], finishedChecks=[DummyFinishedTest(), DummyFinishedTest()]), Role(dName, "loner2", ami, 2, dao=dao, startActions=[DummyAction(), DummyAction()], finishedChecks=[DummyFinishedTest(), DummyFinishedTest()], dataCollectors=[Mock(DataCollector)])]) ''' roles = [] for _ in range(3): role = mock(Role) when(role).checkFinished().thenReturn(True) self.deployment = Deployment(dName, roles=roles) def tearDown(self): if hasattr(self, 'mon'): self.mon.stop() def assertStartActionsCalled(self, deployment): for role in deployment.roles: self.assertTrue(role.executeStartCommands.call_count, 1) def assertStartActionsNotCalled(self, deployment): for role in deployment.roles: self.assertFalse(role.executeStartCommands.called) def assertDataCollectorsCalled(self, deployment): for role in deployment.roles: self.assertTrue(role.collectData.call_count, 1) def testLifecycle(self): ''' Test listeners are properly notified when deployment goes to running state. ''' pollRate = 2 connFactory = DummyConnFactory() self.deployment.cloud = connFactory listeners = {} for state in (DeploymentState.INSTANCES_LAUNCHED, DeploymentState.ROLES_STARTED, DeploymentState.JOB_COMPLETED, DeploymentState.COLLECTING_DATA, DeploymentState.DATA_COLLECTED, DeploymentState.SHUTTING_DOWN, DeploymentState.COMPLETED): l = mock() self.deployment.addStateChangeListener(state, l) listeners[state] = l self.deployment.setPollRate(pollRate) thread = Thread(target=self.deployment.run) thread.start() time.sleep(2 * pollRate) #Manually set mock instances to running connFactory.setState('running') thread.join(30) for (s,l) in listeners.iteritems(): print "State is " + s verify(l).notify(any()) self.assertStartActionsCalled(self.deployment) self.assertDataCollectorsCalled(self.deployment) def testReAttachInstancesLaunched(self): ''' Simulate re-attaching to an already started deployment, which is at stage INSTANCES_LAUNCHED. Assert that the lifecycle properly continues after re-attaching. ''' connFactory = DummyConnFactory() self.deployment.cloud = connFactory for role in self.deployment.roles: reservationId = 'r-%s' % role.getName() role.setReservationId(reservationId) connFactory.conn.reservations[reservationId] = DummyReservation(count=role.getCount(), id=reservationId) connFactory.setState('running') pollRate = 2 hits = {} class Listener: def __init__(self, state): self.state = state def notify(self, evt): hits[self.state] = True for state in (DeploymentState.INSTANCES_LAUNCHED, DeploymentState.ROLES_STARTED, DeploymentState.JOB_COMPLETED, DeploymentState.COLLECTING_DATA, DeploymentState.DATA_COLLECTED, DeploymentState.SHUTTING_DOWN, DeploymentState.COMPLETED): self.deployment.addStateChangeListener(state, Listener(state)) #MicroMock(notify=lambda evt:hits.__setitem__(state, True))) self.deployment.state = DeploymentState.INSTANCES_LAUNCHED self.deployment.setPollRate(pollRate) thread = Thread(target=self.deployment.run) thread.start() time.sleep(2 * pollRate) thread.join(15) self.assertFalse(hits.has_key(DeploymentState.INSTANCES_LAUNCHED)) #Should only hit new stages after this one self.assertTrue(hits.has_key(DeploymentState.ROLES_STARTED)) self.assertTrue(hits.has_key(DeploymentState.JOB_COMPLETED)) self.assertTrue(hits.has_key(DeploymentState.COLLECTING_DATA)) self.assertTrue(hits.has_key(DeploymentState.DATA_COLLECTED)) self.assertTrue(hits.has_key(DeploymentState.SHUTTING_DOWN)) self.assertTrue(hits.has_key(DeploymentState.COMPLETED)) self.assertStartActionsCalled(self.deployment) def testReAttachRoleStarted(self): ''' Simulate re-attaching to an already started deployment, which is at stage ROLES_STARTED. Assert that the lifecycle properly continues after re-attaching. ''' cloud = DummyConnFactory() self.deployment.cloud = cloud for role in self.deployment.roles: reservationId = 'r-%s' % role.getName() cloud.conn.reservations[reservationId] = mock(Reservation) cloud.setState('running') pollRate = 2 hits = {} class Listener: def __init__(self, state): self.state = state def notify(self, evt): hits[self.state] = True for state in (DeploymentState.INSTANCES_LAUNCHED, DeploymentState.ROLES_STARTED, DeploymentState.JOB_COMPLETED, DeploymentState.COLLECTING_DATA, DeploymentState.DATA_COLLECTED, DeploymentState.SHUTTING_DOWN, DeploymentState.COMPLETED): self.deployment.addStateChangeListener(state, Listener(state)) #MicroMock(notify=lambda evt:hits.__setitem__(state, True))) self.deployment.state = DeploymentState.ROLES_STARTED self.deployment.setPollRate(pollRate) thread = Thread(target=self.deployment.run) thread.start() time.sleep(2 * pollRate) thread.join(15) self.assertFalse(hits.has_key(DeploymentState.INSTANCES_LAUNCHED)) #Should only hit new stages after this one self.assertFalse(hits.has_key(DeploymentState.ROLES_STARTED)) self.assertTrue(hits.has_key(DeploymentState.JOB_COMPLETED)) self.assertTrue(hits.has_key(DeploymentState.COLLECTING_DATA)) self.assertTrue(hits.has_key(DeploymentState.DATA_COLLECTED)) self.assertTrue(hits.has_key(DeploymentState.SHUTTING_DOWN)) self.assertTrue(hits.has_key(DeploymentState.COMPLETED)) self.assertStartActionsNotCalled(self.deployment)
def main(argv=None): ''' End-to-End integration test of deployment lifecycle. Uses one micro, EBS-backed instance, ''' SQLITE_FILE = "%s/.d2c_test/deploy_test.sqlite" % os.path.expanduser('~') if os.path.exists(SQLITE_FILE): print "Deleting existing DB" os.unlink(SQLITE_FILE) dao = DAO(SQLITE_FILE) conf = TestConfig("/home/willmore/test.conf") dao.saveConfiguration(conf) ec2ConnFactory = EC2ConnectionFactory(conf.awsCred.access_key_id, conf.awsCred.secret_access_key, StdOutLogger()) credStore = CredStore(dao) ec2Cred = credStore.getDefaultEC2Cred() testDir = "/tmp/d2c/%s" % random.randint(0, 1000) sshCred = SSHCred('ec2-user', '/home/willmore/cert/cloudeco.pem') deployment = Deployment("dummyDep", ec2ConnFactory, roles=[ Role("dummyDep", "loner", ami=AMI(amiId="ami-47cefa33"), count=1, launchCred=ec2Cred, instanceType=InstanceType.T1_MICRO, startActions=[Action(command="echo howdy > /tmp/howdy.txt", sshCred=sshCred)], finishedChecks=[FileExistsFinishedCheck(fileName="/tmp/howdy.txt", sshCred=sshCred)], stopActions=[Action(command="echo adios > /tmp/adios.txt", sshCred=sshCred)], dataCollectors=[DataCollector(source="/tmp/howdy.txt", destination=testDir, sshCred=sshCred), DataCollector(source="/tmp/adios.txt", destination=testDir, sshCred=sshCred), DataCollector(source="/tmp/d2c.context", destination=testDir, sshCred=sshCred)])]) class Listener: def notify(self, event): print "Deployment state changed to: " + event.newState deployment.addAnyStateChangeListener(Listener()) thread = Thread(target=deployment.run) thread.start() thread.join() print "howdy.txt fetch = %s" % str(os.path.exists(testDir + "/howdy.txt")) print "adios.txt fetch = %s" % str(os.path.exists(testDir + "/adios.txt")) print "d2c.context fetch = %s" % str(os.path.exists(testDir + "/d2c.context"))