class Application(object): def __init__(self, s3, manifestPath, localWorkingDir): self.manifestPath = manifestPath self.manifest = Manifest(manifestPath) self.s3interface = S3Interface(s3, self.manifest.GetBucketName(), localWorkingDir) metafac = InstanceMetadataFactory(self.manifest) self.instanceManager = InstanceManager(self.s3interface, self.manifest, metafac) self.manifestKey = "/".join([self.manifest.GetS3KeyPrefix(), "manifest.json"]) def downloadS3Document(self, documentName): logging.info("downloading specified document '{0}' from s3 bucket {1}" .format(documentName, self.s3interface.bucketName)) filteredDocs = list( self.manifest.GetS3Documents( filter = {"Name": documentName})) if len(filteredDocs) == 0: raise ValueError("specified document {0} not found".format(documentName)) elif filteredDocs[0]["Direction"] != "AWSToLocal": raise ValueError("specified document not marked AWSToLocal") elif len(filteredDocs)> 1: raise ValueError("manifest error") doc = filteredDocs[0] self.s3interface.downloadCompressed(self.manifest.GetS3KeyPrefix(), documentName, os.path.abspath(doc["LocalPath"])) def downloadS3Documents(self): logging.info("downloading files from s3 bucket {0}".format(self.s3interface.bucketName)) for doc in self.manifest.GetS3Documents(filter = {"Direction": "AWSToLocal"}): self.s3interface.downloadCompressed(self.manifest.GetS3KeyPrefix(), doc["Name"], os.path.abspath(doc["LocalPath"])) logging.info("downloading finished") def downloadLogs(self, outputdir): logging.info("downloading instance logs s3 bucket {0}".format(self.s3interface.bucketName)) for j in self.manifest.GetJobs(): self.instanceManager.downloadInstanceLog(j["Id"], outputdir) def uploadS3Documents(self): logging.info("uploading files to s3 bucket {0}".format(self.s3interface.bucketName)) logging.info("uploading manifest {0} to {1}".format(self.manifestPath, self.manifestKey)) self.s3interface.uploadFile(self.manifestPath, self.manifestKey) for doc in self.manifest.GetS3Documents(filter = {"Direction": "LocalToAWS"}): self.s3interface.uploadCompressed(self.manifest.GetS3KeyPrefix(), doc["Name"], os.path.abspath(doc["LocalPath"])) logging.info("uploading finished") def runInstances(self, ec2, instanceConfig): ec2interface = EC2Interface(ec2, instanceConfig["BootStrapperConfig"]["WorkingDirectory"], self.manifest, self.manifestKey, self.instanceManager, instanceConfig["BootStrapperConfig"]["PythonPath"], instanceConfig["BootStrapperConfig"]["BootStrapScriptPath"], instanceConfig["BootStrapperConfig"]["LineBreak"], instanceConfig["BootStrapperConfig"]["BootstrapCommands"]) ec2interface.launchInstances(instanceConfig["EC2Config"]["InstanceConfig"]) logging.info("ec2 launch finished")
def test_GetS3DocumentsReturnsFilteredDocuments(self): m = Manifest( self.writeTestJsonFile({ "ProjectName": "projectname", "BucketName": "bucket", "Documents": [{ "Name": "document1", "Direction": "LocalToAWS", "LocalPath": ".", "AWSInstancePath": "awsinstancepath" }, { "Name": "document2", "Direction": "AWSToLocal", "LocalPath": ".", "AWSInstancePath": "awsinstancepath" }, { "Name": "document3", "Direction": "LocalToAWS", "LocalPath": ".", "AWSInstancePath": "awsinstancepath" }], "InstanceJobs": [] })) result = list(m.GetS3Documents(filter={"Direction": "LocalToAWS"})) self.assertEqual(result[0]["Name"], "document1") self.assertEqual(result[1]["Name"], "document3") result = list( m.GetS3Documents(filter={ "Direction": "LocalToAWS", "Name": "document3" })) self.assertEqual(result[0]["Name"], "document3")
def test_GetS3DocumentsReturnsExpectedValue(self): m = Manifest( self.writeTestJsonFile({ "ProjectName": "projectname", "BucketName": "bucket", "Documents": [ { "Name": "document1", "Direction": "LocalToAWS", "LocalPath": ".", "AWSInstancePath": "awsinstancepath" }, { "Name": "document2", "Direction": "AWSToLocal", "LocalPath": ".", "AWSInstancePath": "awsinstancepath1" }, { "Name": "document3", "Direction": "LocalToAWS", "LocalPath": ".", "AWSInstancePath": "awsinstancepath2" }, ], "InstanceJobs": [] })) result = list(m.GetS3Documents()) self.assertEqual(result[0]["Name"], "document1") self.assertEqual(result[0]["Direction"], "LocalToAWS") self.assertEqual(result[0]["LocalPath"], ".") self.assertEqual(result[0]["AWSInstancePath"], "awsinstancepath") self.assertEqual(result[1]["Name"], "document2") self.assertEqual(result[1]["Direction"], "AWSToLocal") self.assertEqual(result[1]["LocalPath"], ".") self.assertEqual(result[1]["AWSInstancePath"], "awsinstancepath1") self.assertEqual(result[2]["Name"], "document3") self.assertEqual(result[2]["Direction"], "LocalToAWS") self.assertEqual(result[2]["LocalPath"], ".") self.assertEqual(result[2]["AWSInstancePath"], "awsinstancepath2")
def test_GetS3DocumentsThrowsErrorWithBadDirectionParameterInJson(self): m = Manifest( self.writeTestJsonFile({ "ProjectName": "testProject", "BucketName": "bucket", "Documents": [{ "Name": "document1", "Direction": "InvalidDirection", "LocalPath": ".", "AWSInstancePath": "awsinstancepath" }], "InstanceJobs": [] })) with self.assertRaises(ValueError) as context: m.GetS3Documents()
def test_GetS3DocumentsThrowsErrorWithBadFilter(self): m = Manifest( self.writeTestJsonFile({ "ProjectName": "projectname", "BucketName": "bucket", "Documents": [ { "Name": "document1", "Direction": "LocalToAWS", "LocalPath": ".", "AWSInstancePath": "awsinstancepath" }, ], "InstanceJobs": [] })) self.assertRaises( KeyError, lambda: list(m.GetS3Documents(filter={"nonmatchingkey": "value"})))