def test_downloadMetaData(self):
mockManifest = Mock(spec=Manifest)
mockManifest.GetS3KeyPrefix.side_effect = lambda: ""
mockS3Interface = Mock(spec=S3Interface)
mockS3Interface.localTempDir = "tests"
mockInstanceMetadataFactory = Mock(spec=InstanceMetadataFactory)
inst = InstanceManager(mockS3Interface, mockManifest,
mockInstanceMetadataFactory)
mockInstanceMetadataFactory.FromJson.side_effect = \
lambda tmpfile : -9999
instId = 10
tmpfile = inst.GetMetaFileTempPath(instId)
with open(tmpfile, 'w') as tmp:
tmp.write("nothing")
res = inst.downloadMetaData(instId)
mockInstanceMetadataFactory.FromJson.assert_called_once_with(tmpfile)
self.assertTrue(res == -9999)
self.assertFalse(os.path.exists(tmpfile))
mockManifest.GetS3KeyPrefix.has_call()
mockS3Interface.downloadFile.has_call()
self.assertFalse(os.path.exists(tmpfile))
def main():
"""to be run on by each instance as a startup command"""
import argparse, sys
import boto3
#from powershell_s3 import powershell_s3
from s3interface import S3Interface
from manifest import Manifest
from instancemanager import InstanceManager
from instancemetadatafactory import InstanceMetadataFactory
from loghelper import LogHelper
parser = argparse.ArgumentParser(
description="AWS Instance bootstrapper" +
"Loads manifest which contains data and commands to run on this instance,"
+ "downloads data from S3, runs commands, and uploads results to S3")
parser.add_argument("--bucketName",
help="the name of the S3 bucket to work with",
required=True)
parser.add_argument(
"--manifestKey",
help="the key pointing to the manifest file in the s3 bucket",
required=True)
parser.add_argument(
"--instanceId",
help="the id of this instance as defined in the manifest file",
required=True)
parser.add_argument(
"--localWorkingDir",
help=
"a directory to store working files, it will be created if it does not exist on the instance",
required=True)
try:
#boto3.set_stream_logger(name='botocore')
args = vars(parser.parse_args())
bootstrapper = None
bucketName = args["bucketName"]
manifestKey = args["manifestKey"]
instanceId = int(args["instanceId"])
localWorkingDir = args["localWorkingDir"]
if not os.path.exists(localWorkingDir):
os.makedirs(localWorkingDir)
logPath = LogHelper.instanceLogPath(localWorkingDir, instanceId)
LogHelper.start_logging(logPath)
logging.info("startup")
logging.info("creating boto3 s3 resource")
s3 = boto3.resource('s3')
logging.info("creating S3Interface")
s3interface = S3Interface(s3, bucketName, localWorkingDir)
localManifestPath = os.path.join(localWorkingDir, "manifest.json")
logging.info("downloading manifest from S3")
s3interface.downloadFile(manifestKey, localManifestPath)
manifest = Manifest(localManifestPath)
metafac = InstanceMetadataFactory(manifest)
instancemanager = InstanceManager(s3interface, manifest, metafac)
metadata = instancemanager.downloadMetaData(instanceId)
bootstrapper = AWSInstanceBootStrapper(instanceId, manifest,
s3interface, instancemanager,
metadata)
bootstrapper.DownloadS3Documents()
bootstrapper.RunCommands()
bootstrapper.UploadS3Documents()
except Exception as ex:
logging.exception("error in bootstrapper")
if bootstrapper is not None:
bootstrapper.UploadStatus()
sys.exit(1)