def __init__(self, s3, manifestPath, localWorkingDir):
self.manifestPath = manifestPath
self.manifest = Manifest(manifestPath)
self.s3interface = S3Interface(s3, self.manifest.GetBucketName(), localWorkingDir)
metafac = InstanceMetadataFactory(self.manifest)
self.instanceManager = InstanceManager(self.s3interface, self.manifest, metafac)
self.manifestKey = "/".join([self.manifest.GetS3KeyPrefix(), "manifest.json"])
def test_publishInstance(self):
mockManifest = Mock(spec=Manifest)
mockS3Interface = Mock(spec=S3Interface)
mockInstanceMetadataFactory = Mock(spec=InstanceMetadataFactory)
mockInstanceMetadataFactory.InitializeMetadata.side_effect = \
lambda i, a : "metadata"
mockManifest.GetS3KeyPrefix.side_effect = lambda: "the prefix"
mockS3Interface.localTempDir = "tests"
inst = InstanceManager(mockS3Interface, mockManifest,
mockInstanceMetadataFactory)
instId = 100
tmpfile = inst.GetMetaFileTempPath(instId)
with open(tmpfile, 'w') as tmp:
tmp.write("nothing")
inst.publishInstance(instId, 1000)
mockManifest.GetS3KeyPrefix.assert_any_call()
mockS3Interface.uploadFile.assert_called_with(
tmpfile, "the prefix/instances/100/metadata.json", True)
mockInstanceMetadataFactory.InitializeMetadata.assert_any_call(
instId, 1000)
mockInstanceMetadataFactory.ToJson.assert_any_call("metadata", tmpfile)
self.assertFalse(os.path.exists(tmpfile))
class Application(object):
def __init__(self, s3, manifestPath, localWorkingDir):
self.manifestPath = manifestPath
self.manifest = Manifest(manifestPath)
self.s3interface = S3Interface(s3, self.manifest.GetBucketName(), localWorkingDir)
metafac = InstanceMetadataFactory(self.manifest)
self.instanceManager = InstanceManager(self.s3interface, self.manifest, metafac)
self.manifestKey = "/".join([self.manifest.GetS3KeyPrefix(), "manifest.json"])
def downloadS3Document(self, documentName):
logging.info("downloading specified document '{0}' from s3 bucket {1}"
.format(documentName, self.s3interface.bucketName))
filteredDocs = list(
self.manifest.GetS3Documents(
filter = {"Name": documentName}))
if len(filteredDocs) == 0:
raise ValueError("specified document {0} not found".format(documentName))
elif filteredDocs[0]["Direction"] != "AWSToLocal":
raise ValueError("specified document not marked AWSToLocal")
elif len(filteredDocs)> 1:
raise ValueError("manifest error")
doc = filteredDocs[0]
self.s3interface.downloadCompressed(self.manifest.GetS3KeyPrefix(), documentName,
os.path.abspath(doc["LocalPath"]))
def downloadS3Documents(self):
logging.info("downloading files from s3 bucket {0}".format(self.s3interface.bucketName))
for doc in self.manifest.GetS3Documents(filter = {"Direction": "AWSToLocal"}):
self.s3interface.downloadCompressed(self.manifest.GetS3KeyPrefix(), doc["Name"],
os.path.abspath(doc["LocalPath"]))
logging.info("downloading finished")
def downloadLogs(self, outputdir):
logging.info("downloading instance logs s3 bucket {0}".format(self.s3interface.bucketName))
for j in self.manifest.GetJobs():
self.instanceManager.downloadInstanceLog(j["Id"], outputdir)
def uploadS3Documents(self):
logging.info("uploading files to s3 bucket {0}".format(self.s3interface.bucketName))
logging.info("uploading manifest {0} to {1}".format(self.manifestPath, self.manifestKey))
self.s3interface.uploadFile(self.manifestPath, self.manifestKey)
for doc in self.manifest.GetS3Documents(filter = {"Direction": "LocalToAWS"}):
self.s3interface.uploadCompressed(self.manifest.GetS3KeyPrefix(), doc["Name"],
os.path.abspath(doc["LocalPath"]))
logging.info("uploading finished")
def runInstances(self, ec2, instanceConfig):
ec2interface = EC2Interface(ec2, instanceConfig["BootStrapperConfig"]["WorkingDirectory"],
self.manifest, self.manifestKey, self.instanceManager,
instanceConfig["BootStrapperConfig"]["PythonPath"],
instanceConfig["BootStrapperConfig"]["BootStrapScriptPath"],
instanceConfig["BootStrapperConfig"]["LineBreak"],
instanceConfig["BootStrapperConfig"]["BootstrapCommands"])
ec2interface.launchInstances(instanceConfig["EC2Config"]["InstanceConfig"])
logging.info("ec2 launch finished")
def __init__(self, *args):
logging.debug('MainWindow creation')
QMainWindow.__init__(self, *args)
self.ui = Ui_MainWindow()
logging.debug('MainWindow ui setup')
self.ui.setupUi(self)
self.instanceManager = InstanceManager(InstanceManager.getStoredKey())
self.readConfig()
self.setupModels()
self.ui.treeView.activated.connect(self.updatePasswordsModel)
self.ui.tableView.activated.connect(self.udpateDetailTabs)
self.ui.actionQuit.triggered.connect(self.closeReally)
self.ui.actionPreferences.triggered.connect(self.openPreferences)
self.ui.actionRefresh.triggered.connect(self.refreshInstances)
# Init QSystemTrayIcon
icon = QIcon.fromTheme('dialog-passwords',
QIcon(':/icons/dialog-password.svg'))
self.trayIcon = QSystemTrayIcon(icon, self)
systrayMenu = QMenu(self)
systrayMenu.addAction(self.ui.actionRefresh)
systrayMenu.addAction(self.ui.actionQuit)
self.trayIcon.setContextMenu(systrayMenu)
self.trayIcon.show()
self.trayIcon.activated.connect(self.handleSystray)
def test_GetKeyPrefix(self):
mockManifest = Mock(spec=Manifest)
mockS3Interface = Mock(spec=S3Interface)
mockInstanceMetadataFactory = Mock(spec=InstanceMetadataFactory)
inst = InstanceManager(mockS3Interface, mockManifest,
mockInstanceMetadataFactory)
mockManifest.GetS3KeyPrefix.side_effect = lambda: "the prefix"
self.assertTrue(inst.GetKeyPrefix(1000) == "the prefix/instances/1000")
mockManifest.GetS3KeyPrefix.assert_any_call()
def test_GetMetaFileTempPath(self):
mockManifest = Mock(spec=Manifest)
mockS3Interface = Mock(spec=S3Interface)
mockS3Interface.localTempDir = "temp123"
mockInstanceMetadataFactory = Mock(spec=InstanceMetadataFactory)
inst = InstanceManager(mockS3Interface, mockManifest,
mockInstanceMetadataFactory)
result = inst.GetMetaFileTempPath(10)
self.assertEqual(result,
os.path.join("temp123", "instance_metadata10.json"))
def test_downloadInstanceData(self):
mockManifest = Mock(spec=Manifest)
mockManifest.GetS3KeyPrefix.side_effect = lambda: "the prefix"
mockS3Interface = Mock(spec=S3Interface)
mockInstanceMetadataFactory = Mock(spec=InstanceMetadataFactory)
inst = InstanceManager(mockS3Interface, mockManifest,
mockInstanceMetadataFactory)
inst.uploadInstanceData(10000, "docname", "localPath")
mockS3Interface.uploadFile.assert_has_calls(
[call("localPath", "the prefix/instances/10000/docname", True)])
def addInstance(self, data):
tab = data[0]
instance = data[1]
tabCount = tab.rowCount()
tab.setRowCount(tabCount+1)
for i in xrange(0, len(self.configAccountTabFields)):
instance_manager = InstanceManager(instance, ro = True)
item = AMITableWidgetItem(instance_manager)
item.setText(instance_manager.get(self.configAccountTabFields[i][0]))
tab.setItem(tabCount, i, item)
tab.resizeColumnsToContents()
def test_constructor(self):
mockManifest = Mock(spec=Manifest)
mockS3Interface = Mock(spec=S3Interface)
mockInstanceMetaFac = Mock(spec=InstanceMetadataFactory)
inst = InstanceManager(mockS3Interface, mockManifest,
mockInstanceMetaFac)
self.assertTrue(inst.manifest == mockManifest)
self.assertTrue(inst.s3Interface == mockS3Interface)
self.assertTrue(inst.instanceMetadataFactory == mockInstanceMetaFac)
def test_uploadMetaData(self):
mockManifest = Mock(spec=Manifest)
mockManifest.GetS3KeyPrefix.side_effect = lambda: ""
mockS3Interface = Mock(spec=S3Interface)
mockS3Interface.localTempDir = "tests"
mockInstanceMetadataFactory = Mock(spec=InstanceMetadataFactory)
mockMetadata = Mock(spec=InstanceMetadata)
instId = 17
mockMetadata.Get.side_effect = lambda key: {"Id": instId}[key]
inst = InstanceManager(mockS3Interface, mockManifest,
mockInstanceMetadataFactory)
tmpfile = inst.GetMetaFileTempPath(instId)
with open(tmpfile, 'w') as tmp:
tmp.write("nothing")
inst.uploadMetaData(mockMetadata)
self.assertFalse(os.path.exists(tmpfile))
mockS3Interface.uploadFile.has_call()
mockInstanceMetadataFactory.ToJson.assert_any_call(
mockMetadata, tmpfile)
def test_downloadMetaData(self):
mockManifest = Mock(spec=Manifest)
mockManifest.GetS3KeyPrefix.side_effect = lambda: ""
mockS3Interface = Mock(spec=S3Interface)
mockS3Interface.localTempDir = "tests"
mockInstanceMetadataFactory = Mock(spec=InstanceMetadataFactory)
inst = InstanceManager(mockS3Interface, mockManifest,
mockInstanceMetadataFactory)
mockInstanceMetadataFactory.FromJson.side_effect = \
lambda tmpfile : -9999
instId = 10
tmpfile = inst.GetMetaFileTempPath(instId)
with open(tmpfile, 'w') as tmp:
tmp.write("nothing")
res = inst.downloadMetaData(instId)
mockInstanceMetadataFactory.FromJson.assert_called_once_with(tmpfile)
self.assertTrue(res == -9999)
self.assertFalse(os.path.exists(tmpfile))
mockManifest.GetS3KeyPrefix.has_call()
mockS3Interface.downloadFile.has_call()
self.assertFalse(os.path.exists(tmpfile))
def main():
"""to be run on by each instance as a startup command"""
import argparse, sys
import boto3
#from powershell_s3 import powershell_s3
from s3interface import S3Interface
from manifest import Manifest
from instancemanager import InstanceManager
from instancemetadatafactory import InstanceMetadataFactory
from loghelper import LogHelper
parser = argparse.ArgumentParser(
description="AWS Instance bootstrapper" +
"Loads manifest which contains data and commands to run on this instance,"
+ "downloads data from S3, runs commands, and uploads results to S3")
parser.add_argument("--bucketName",
help="the name of the S3 bucket to work with",
required=True)
parser.add_argument(
"--manifestKey",
help="the key pointing to the manifest file in the s3 bucket",
required=True)
parser.add_argument(
"--instanceId",
help="the id of this instance as defined in the manifest file",
required=True)
parser.add_argument(
"--localWorkingDir",
help=
"a directory to store working files, it will be created if it does not exist on the instance",
required=True)
try:
#boto3.set_stream_logger(name='botocore')
args = vars(parser.parse_args())
bootstrapper = None
bucketName = args["bucketName"]
manifestKey = args["manifestKey"]
instanceId = int(args["instanceId"])
localWorkingDir = args["localWorkingDir"]
if not os.path.exists(localWorkingDir):
os.makedirs(localWorkingDir)
logPath = LogHelper.instanceLogPath(localWorkingDir, instanceId)
LogHelper.start_logging(logPath)
logging.info("startup")
logging.info("creating boto3 s3 resource")
s3 = boto3.resource('s3')
logging.info("creating S3Interface")
s3interface = S3Interface(s3, bucketName, localWorkingDir)
localManifestPath = os.path.join(localWorkingDir, "manifest.json")
logging.info("downloading manifest from S3")
s3interface.downloadFile(manifestKey, localManifestPath)
manifest = Manifest(localManifestPath)
metafac = InstanceMetadataFactory(manifest)
instancemanager = InstanceManager(s3interface, manifest, metafac)
metadata = instancemanager.downloadMetaData(instanceId)
bootstrapper = AWSInstanceBootStrapper(instanceId, manifest,
s3interface, instancemanager,
metadata)
bootstrapper.DownloadS3Documents()
bootstrapper.RunCommands()
bootstrapper.UploadS3Documents()
except Exception as ex:
logging.exception("error in bootstrapper")
if bootstrapper is not None:
bootstrapper.UploadStatus()
sys.exit(1)
class MainWindow(QMainWindow):
def __init__(self, *args):
logging.debug('MainWindow creation')
QMainWindow.__init__(self, *args)
self.ui = Ui_MainWindow()
logging.debug('MainWindow ui setup')
self.ui.setupUi(self)
self.instanceManager = InstanceManager(InstanceManager.getStoredKey())
self.readConfig()
self.setupModels()
self.ui.treeView.activated.connect(self.updatePasswordsModel)
self.ui.tableView.activated.connect(self.udpateDetailTabs)
self.ui.actionQuit.triggered.connect(self.closeReally)
self.ui.actionPreferences.triggered.connect(self.openPreferences)
self.ui.actionRefresh.triggered.connect(self.refreshInstances)
# Init QSystemTrayIcon
icon = QIcon.fromTheme('dialog-passwords',
QIcon(':/icons/dialog-password.svg'))
self.trayIcon = QSystemTrayIcon(icon, self)
systrayMenu = QMenu(self)
systrayMenu.addAction(self.ui.actionRefresh)
systrayMenu.addAction(self.ui.actionQuit)
self.trayIcon.setContextMenu(systrayMenu)
self.trayIcon.show()
self.trayIcon.activated.connect(self.handleSystray)
def closeEvent(self, event):
""" Close to systray
"""
event.ignore()
self.hide()
def closeReally(self):
""" Close by Quit action
"""
exit(0)
def handleSystray(self):
if self.isVisible():
self.hide()
else:
self.show()
@Slot(QModelIndex)
def updatePasswordsModel(self, index):
logging.debug('MainWindow.updatePasswordsModel called')
passwords = self.treeModel.getPasswords(index)
self.listModel.setPasswords(passwords)
def udpateDetailTabs(self, index):
logging.debug('MainWindow.udpateDetailTabs called')
password = self.listModel.getPassword(index.row())
self.ui.detailTabWidget.setVisible(True)
self.ui.detailTabWidget.setCurrentIndex(0)
if 'notes' in password:
self.ui.notesBrowser.setText(password['notes'])
self.ui.tabNotes.setEnabled(True)
else:
self.ui.tabNotes.setEnabled(False)
if 'revisions' in password:
for i in password['revisions']:
for key, val in i.items():
self.ui.revisionsBrowser.append(key + ': ' + str(val))
self.ui.revisionsBrowser.append('\n')
self.ui.tabRevisions.setEnabled(True)
else:
self.ui.tabRevisions.setEnabled(False)
self.ui.detailsBrowser.clear()
for key, val in password.items():
if key not in ['notes', 'revisions']:
self.ui.detailsBrowser.append(key + ': ' + str(val))
def readConfig(self):
self.ncInstances = []
self.instanceManager.readConfig()
for i, val in self.instanceManager.getInstances().items():
logging.debug('reading: ' + i)
instance = NextcloudInstance(description=val['name'],
url=val['url'],
username=val['username'],
password=val['password'],
verifySSL=val['verifySSL'])
self.ncInstances.append(instance)
def setupModels(self):
self.ui.treeView.blockSignals(True)
self.ui.detailTabWidget.setVisible(False)
self.treeModel = NCTreeModel(self.ncInstances, self)
self.listModel = PasswordsModel(self)
self.ui.treeView.setModel(self.treeModel)
self.ui.treeView.blockSignals(False)
self.ui.tableView.setModel(self.listModel)
def openPreferences(self):
dia = PreferencesDialog(self.instanceManager.getInstances())
if (dia.exec_()):
self.instanceManager.replaceInstances(dia.getInstances())
self.instanceManager.writeConfig()
self.refreshInstances()
def refreshInstances(self):
self.readConfig()
self.setupModels()