class Application(object):
def __init__(self, s3, manifestPath, localWorkingDir):
self.manifestPath = manifestPath
self.manifest = Manifest(manifestPath)
self.s3interface = S3Interface(s3, self.manifest.GetBucketName(), localWorkingDir)
metafac = InstanceMetadataFactory(self.manifest)
self.instanceManager = InstanceManager(self.s3interface, self.manifest, metafac)
self.manifestKey = "/".join([self.manifest.GetS3KeyPrefix(), "manifest.json"])
def downloadS3Document(self, documentName):
logging.info("downloading specified document '{0}' from s3 bucket {1}"
.format(documentName, self.s3interface.bucketName))
filteredDocs = list(
self.manifest.GetS3Documents(
filter = {"Name": documentName}))
if len(filteredDocs) == 0:
raise ValueError("specified document {0} not found".format(documentName))
elif filteredDocs[0]["Direction"] != "AWSToLocal":
raise ValueError("specified document not marked AWSToLocal")
elif len(filteredDocs)> 1:
raise ValueError("manifest error")
doc = filteredDocs[0]
self.s3interface.downloadCompressed(self.manifest.GetS3KeyPrefix(), documentName,
os.path.abspath(doc["LocalPath"]))
def downloadS3Documents(self):
logging.info("downloading files from s3 bucket {0}".format(self.s3interface.bucketName))
for doc in self.manifest.GetS3Documents(filter = {"Direction": "AWSToLocal"}):
self.s3interface.downloadCompressed(self.manifest.GetS3KeyPrefix(), doc["Name"],
os.path.abspath(doc["LocalPath"]))
logging.info("downloading finished")
def downloadLogs(self, outputdir):
logging.info("downloading instance logs s3 bucket {0}".format(self.s3interface.bucketName))
for j in self.manifest.GetJobs():
self.instanceManager.downloadInstanceLog(j["Id"], outputdir)
def uploadS3Documents(self):
logging.info("uploading files to s3 bucket {0}".format(self.s3interface.bucketName))
logging.info("uploading manifest {0} to {1}".format(self.manifestPath, self.manifestKey))
self.s3interface.uploadFile(self.manifestPath, self.manifestKey)
for doc in self.manifest.GetS3Documents(filter = {"Direction": "LocalToAWS"}):
self.s3interface.uploadCompressed(self.manifest.GetS3KeyPrefix(), doc["Name"],
os.path.abspath(doc["LocalPath"]))
logging.info("uploading finished")
def runInstances(self, ec2, instanceConfig):
ec2interface = EC2Interface(ec2, instanceConfig["BootStrapperConfig"]["WorkingDirectory"],
self.manifest, self.manifestKey, self.instanceManager,
instanceConfig["BootStrapperConfig"]["PythonPath"],
instanceConfig["BootStrapperConfig"]["BootStrapScriptPath"],
instanceConfig["BootStrapperConfig"]["LineBreak"],
instanceConfig["BootStrapperConfig"]["BootstrapCommands"])
ec2interface.launchInstances(instanceConfig["EC2Config"]["InstanceConfig"])
logging.info("ec2 launch finished")
def test_GetS3DocumentsReturnsFilteredDocuments(self):
m = Manifest(
self.writeTestJsonFile({
"ProjectName":
"projectname",
"BucketName":
"bucket",
"Documents": [{
"Name": "document1",
"Direction": "LocalToAWS",
"LocalPath": ".",
"AWSInstancePath": "awsinstancepath"
}, {
"Name": "document2",
"Direction": "AWSToLocal",
"LocalPath": ".",
"AWSInstancePath": "awsinstancepath"
}, {
"Name": "document3",
"Direction": "LocalToAWS",
"LocalPath": ".",
"AWSInstancePath": "awsinstancepath"
}],
"InstanceJobs": []
}))
result = list(m.GetS3Documents(filter={"Direction": "LocalToAWS"}))
self.assertEqual(result[0]["Name"], "document1")
self.assertEqual(result[1]["Name"], "document3")
result = list(
m.GetS3Documents(filter={
"Direction": "LocalToAWS",
"Name": "document3"
}))
self.assertEqual(result[0]["Name"], "document3")
def test_GetS3DocumentsReturnsExpectedValue(self):
m = Manifest(
self.writeTestJsonFile({
"ProjectName":
"projectname",
"BucketName":
"bucket",
"Documents": [
{
"Name": "document1",
"Direction": "LocalToAWS",
"LocalPath": ".",
"AWSInstancePath": "awsinstancepath"
},
{
"Name": "document2",
"Direction": "AWSToLocal",
"LocalPath": ".",
"AWSInstancePath": "awsinstancepath1"
},
{
"Name": "document3",
"Direction": "LocalToAWS",
"LocalPath": ".",
"AWSInstancePath": "awsinstancepath2"
},
],
"InstanceJobs": []
}))
result = list(m.GetS3Documents())
self.assertEqual(result[0]["Name"], "document1")
self.assertEqual(result[0]["Direction"], "LocalToAWS")
self.assertEqual(result[0]["LocalPath"], ".")
self.assertEqual(result[0]["AWSInstancePath"], "awsinstancepath")
self.assertEqual(result[1]["Name"], "document2")
self.assertEqual(result[1]["Direction"], "AWSToLocal")
self.assertEqual(result[1]["LocalPath"], ".")
self.assertEqual(result[1]["AWSInstancePath"], "awsinstancepath1")
self.assertEqual(result[2]["Name"], "document3")
self.assertEqual(result[2]["Direction"], "LocalToAWS")
self.assertEqual(result[2]["LocalPath"], ".")
self.assertEqual(result[2]["AWSInstancePath"], "awsinstancepath2")
def test_GetS3DocumentsThrowsErrorWithBadDirectionParameterInJson(self):
m = Manifest(
self.writeTestJsonFile({
"ProjectName":
"testProject",
"BucketName":
"bucket",
"Documents": [{
"Name": "document1",
"Direction": "InvalidDirection",
"LocalPath": ".",
"AWSInstancePath": "awsinstancepath"
}],
"InstanceJobs": []
}))
with self.assertRaises(ValueError) as context:
m.GetS3Documents()
def test_GetS3DocumentsThrowsErrorWithBadFilter(self):
m = Manifest(
self.writeTestJsonFile({
"ProjectName":
"projectname",
"BucketName":
"bucket",
"Documents": [
{
"Name": "document1",
"Direction": "LocalToAWS",
"LocalPath": ".",
"AWSInstancePath": "awsinstancepath"
},
],
"InstanceJobs": []
}))
self.assertRaises(
KeyError,
lambda: list(m.GetS3Documents(filter={"nonmatchingkey": "value"})))
