def runInstances(credClient,
ami,
key,
iType,
groups,
availZone,
bidPrice,
minInstances,
maxInstances,
userData):
def _runInstances(num):
if bidPrice:
return credClient.runSpotInstances(bidPrice=bidPrice,
ami=ami,
key=key,
instanceType=iType,
groups=groups,
availabilityZone=availZone,
numInstances=num,
userData=userData)
else:
return credClient.runInstances(ami=ami,
key=key,
instanceType=iType,
groups=groups,
availabilityZone=availZone,
numInstances=num,
userData=userData)
instances = []
@defer.inlineCallbacks
def _startInstances():
startedInstances = yield _runInstances(maxInstances - len(instances))
instances.extend(startedInstances)
if len(instances) < minInstances:
raise InstanceStartError('Wanted %d instances got %d' %
(maxInstances - len(instances),
len(startedInstances)))
try:
yield defer_utils.tryUntil(RUN_INSTANCE_TRIES,
_startInstances,
onFailure=defer_utils.sleep(30))
except Exception, err:
## If we got an exception then terminate any instances
## that were started and reraise exception.
## The last thing we want is to leak instances
##
## This is not completely safe! We should probably
## raise an exception with the started instances in it
## and let the caller decide what to do with them
log.err('Error starting instances')
log.err(err)
defer_utils.mapSerial(lambda iChunk :
credClient.terminateInstances(iChunk),
func.chunk(5, instances))
def runInstances(credClient, ami, key, iType, groups, availZone, bidPrice,
minInstances, maxInstances, userData):
def _runInstances(num):
if bidPrice:
return credClient.runSpotInstances(bidPrice=bidPrice,
ami=ami,
key=key,
instanceType=iType,
groups=groups,
availabilityZone=availZone,
numInstances=num,
userData=userData)
else:
return credClient.runInstances(ami=ami,
key=key,
instanceType=iType,
groups=groups,
availabilityZone=availZone,
numInstances=num,
userData=userData)
instances = []
@defer.inlineCallbacks
def _startInstances():
startedInstances = yield _runInstances(maxInstances - len(instances))
instances.extend(startedInstances)
if len(instances) < minInstances:
raise InstanceStartError(
'Wanted %d instances got %d' %
(maxInstances - len(instances), len(startedInstances)))
try:
yield defer_utils.tryUntil(RUN_INSTANCE_TRIES,
_startInstances,
onFailure=defer_utils.sleep(30))
except Exception, err:
## If we got an exception then terminate any instances
## that were started and reraise exception.
## The last thing we want is to leak instances
##
## This is not completely safe! We should probably
## raise an exception with the started instances in it
## and let the caller decide what to do with them
log.err('Error starting instances')
log.err(err)
defer_utils.mapSerial(
lambda iChunk: credClient.terminateInstances(iChunk),
func.chunk(5, instances))
def _makeDirsOnCluster(cluster, dirNames):
"""
Creates a series of directories on a cluster
"""
@defer.inlineCallbacks
def _createDir(d):
yield ssh.runProcessSSH(cluster['master']['public_dns'],
'mkdir -p ' + d,
None,
log.err,
cluster['config']['ssh.user'],
cluster['config']['ssh.options'])
try:
if cluster['master']['instance_type'] is not None:
yield ssh.runProcessSSH(cluster['master']['public_dns'],
'chown -R %s %s' % (cluster['config']['vappio.user'],
d),
None,
log.err,
cluster['config']['ssh.user'],
cluster['config']['ssh.options'])
except commands.Error:
pass
return defer_utils.mapSerial(_createDir, dirNames)
def _deleteEmptyDirs(files):
def _rmDir(d):
return commands.runProcess(
['rmdir', d], stderrf=log.err).addErrback(lambda _: None)
dirs = set([os.path.dirname(f) for f in files])
return defer_utils.mapSerial(_rmDir, dirs)
def downloadMapSerial(urls):
"""
Uses mapSerial to download all urls in serial
"""
getPage = lambda url: http.getPage(url, connectionTimeout=30, timeout=30
).addCallback(lambda content:
(url, content))
d = defer_utils.mapSerial(getPage, urls)
d.addCallback(dict)
return d
def terminateCluster(credClient, persistManager, clusterName, userName):
cluster = yield persistManager.loadCluster(clusterName, userName)
yield defer_utils.mapSerial(
lambda instances: credClient.terminateInstances(instances),
func.chunk(5, cluster.execNodes + cluster.dataNodes))
if cluster.master:
yield credClient.terminateInstances([cluster.master])
defer.returnValue(cluster.setState(cluster.TERMINATED))
def terminateCluster(credClient, persistManager, clusterName, userName):
cluster = yield persistManager.loadCluster(clusterName, userName)
yield defer_utils.mapSerial(
lambda instances: credClient.terminateInstances(instances), func.chunk(5, cluster.execNodes + cluster.dataNodes)
)
if cluster.master:
yield credClient.terminateInstances([cluster.master])
defer.returnValue(cluster.setState(cluster.TERMINATED))
def downloadMapSerial(urls):
"""
Uses mapSerial to download all urls in serial
"""
getPage = lambda url : http.getPage(url,
connectionTimeout=30,
timeout=30
).addCallback(lambda content : (url, content))
d = defer_utils.mapSerial(getPage, urls)
d.addCallback(dict)
return d
def terminateInstancesByAttribute(persistManager, credClient, clusterName,
userName, byAttribute, attributeValues):
cluster = yield persistManager.loadCluster(clusterName, userName)
instances = [
i for i in cluster.execNodes + cluster.dataNodes
if i[byAttribute] in attributeValues
]
yield defer_utils.mapSerial(credClient.terminateInstances,
func.chunk(5, instances))
defer.returnValue(
cluster.removeExecNodes(instances).removeDataNodes(instances))
def terminateInstancesByAttribute(persistManager,
credClient,
clusterName,
userName,
byAttribute,
attributeValues):
cluster = yield persistManager.loadCluster(clusterName, userName)
instances = [i
for i in cluster.execNodes + cluster.dataNodes
if i[byAttribute] in attributeValues]
yield defer_utils.mapSerial(credClient.terminateInstances,
func.chunk(5, instances))
defer.returnValue(cluster.removeExecNodes(instances).removeDataNodes(instances))
def removeTerminatedCluster(persistManager, credClient, clusterName, userName):
yield defer_utils.sleep(REMOVE_CLUSTER_TIMEOUT)()
cluster = yield persistManager.loadCluster(clusterName, userName)
if cluster.state == cluster.TERMINATED:
# Another check to make sure the instances have
# really been terminated
instances = ([cluster.master] + cluster.execNodes + cluster.dataNodes)
instances = yield credClient.updateInstances(instances)
undeadInstances = [i for i in instances if i['state'] != 'terminated']
if undeadInstances:
yield defer_utils.mapSerial(
lambda instances: credClient.terminateInstances(instances),
func.chunk(5, undeadInstances))
yield persistManager.removeCluster(clusterName, userName)
def removeTerminatedCluster(persistManager, credClient, clusterName, userName):
yield defer_utils.sleep(REMOVE_CLUSTER_TIMEOUT)()
cluster = yield persistManager.loadCluster(clusterName, userName)
if cluster.state == cluster.TERMINATED:
# Another check to make sure the instances have
# really been terminated
instances = [cluster.master] + cluster.execNodes + cluster.dataNodes
instances = yield credClient.updateInstances(instances)
undeadInstances = [i for i in instances if i["state"] != "terminated"]
if undeadInstances:
yield defer_utils.mapSerial(
lambda instances: credClient.terminateInstances(instances), func.chunk(5, undeadInstances)
)
yield persistManager.removeCluster(clusterName, userName)
def _makeDirsOnCluster(cluster, dirNames):
"""
Creates a series of directories on a cluster
"""
@defer.inlineCallbacks
def _createDir(d):
yield ssh.runProcessSSH(cluster['master']['public_dns'],
'mkdir -p ' + d, None, log.err,
cluster['config']['ssh.user'],
cluster['config']['ssh.options'])
try:
if cluster['master']['instance_type'] is not None:
yield ssh.runProcessSSH(
cluster['master']['public_dns'],
'chown -R %s %s' % (cluster['config']['vappio.user'], d),
None, log.err, cluster['config']['ssh.user'],
cluster['config']['ssh.options'])
except commands.Error:
pass
return defer_utils.mapSerial(_createDir, dirNames)
try:
self.cache.pop(credential)
except Exception, e:
log.err(e)
return defer.succeed(None)
@defer_utils.timeIt
@defer.inlineCallbacks
def refreshInstances(self):
@defer.inlineCallbacks
def _refresh(credDict):
credInstance = credDict['cred_instance']
try:
instances = yield credInstance.listInstances()
credDict['instances'] = instances
except Exception, e:
log.err(e)
yield defer_utils.mapSerial(_refresh, self.cache.values())
self.state.refreshInstancesDelayed = reactor.callLater(
REFRESH_INTERVAL, self.refreshInstances)
def getCredential(self, credName):
return self.cache[credName]
def getAllCredentials(self):
return self.cache
def _deleteEmptyDirs(files):
def _rmDir(d):
return commands.runProcess(["rmdir", d], stderrf=log.err).addErrback(lambda _: None)
dirs = set([os.path.dirname(f) for f in files])
return defer_utils.mapSerial(_rmDir, dirs)
def deleteCredentialFromCache(self, credential):
try:
self.cache.pop(credential)
except Exception, e:
log.err(e)
return defer.succeed(None)
@defer_utils.timeIt
@defer.inlineCallbacks
def refreshInstances(self):
@defer.inlineCallbacks
def _refresh(credDict):
credInstance = credDict["cred_instance"]
try:
instances = yield credInstance.listInstances()
credDict["instances"] = instances
except Exception, e:
log.err(e)
yield defer_utils.mapSerial(_refresh, self.cache.values())
self.state.refreshInstancesDelayed = reactor.callLater(REFRESH_INTERVAL, self.refreshInstances)
def getCredential(self, credName):
return self.cache[credName]
def getAllCredentials(self):
return self.cache