def _handleTerminateInstances(request):
yield tasks_tx.updateTask(request.body['task_name'],
lambda t: t.setState(tasks_tx.task.TASK_RUNNING))
persistManager = request.state.persistManager
cluster = yield persistManager.loadCluster(request.body['cluster_name'],
request.body['user_name'])
credClient = cred_client.CredentialClient(cluster.credName, request.mq,
request.state.conf)
if request.body['cluster_name'] != 'local':
try:
remoteTaskName = yield clusters_client_www.terminateInstances(
cluster.master['public_dns'], 'local',
request.body['user_name'], request.body['by_attribute'],
request.body['attribute_values'])
localTask = yield tasks_tx.loadTask(request.body['task_name'])
yield tasks_tx.blockOnTaskAndForward('localhost',
request.body['cluster_name'],
remoteTaskName, localTask)
except:
yield terminateInstancesByAttribute(
persistManager, credClient, request.body['cluster_name'],
request.body['user_name'], request.body['by_attribute'],
request.body['attribute_values'])
else:
cl = yield terminateInstancesByAttribute(
persistManager, credClient, 'local', None,
request.body['by_attribute'], request.body['attribute_values'])
yield tasks_tx.updateTask(request.body['task_name'],
lambda t: t.progress())
defer.returnValue(request)
def getOrphans(credentialName, state):
@defer.inlineCallbacks
def _sshAble(instance):
sshUser = state.clustersState.machineConf('ssh.user')
sshOptions = state.clustersState.machineConf('ssh.options')
try:
yield ssh.runProcessSSH(instance['public_dns'], 'echo hello', None,
None, sshUser, sshOptions)
except:
defer.returnValue(False)
defer.returnValue(True)
credClient = cred_client.CredentialClient(credentialName, state.mq,
state.clustersState.conf)
knownInstances = yield getKnownInstances(credentialName)
instances = yield credClient.listInstances()
instances = [
i for i in instances if i['instance_id'] not in knownInstances
]
sshAble = yield defer_utils.mapPar(_sshAble, instances, parallel=5)
ret = set(
[i['instance_id'] for canSsh, i in zip(sshAble, instances) if canSsh])
defer.returnValue(ret)
def terminateRemoteCluster(request):
persistManager = request.state.persistManager
cluster = yield persistManager.loadCluster(request.body['cluster_name'],
request.body['user_name'])
authToken = auth_token.generateToken(
cluster.config('cluster.cluster_public_key'))
credClient = cred_client.CredentialClient(cluster.credName, request.mq,
request.state.conf)
try:
if cluster.master:
wwwTerminateCluster = clusters_client_www.terminateCluster
remoteTaskName = yield wwwTerminateCluster(
cluster.master['public_dns'], 'local', None, authToken)
localTask = yield tasks_tx.loadTask(request.body['task_name'])
yield tasks_tx.blockOnTaskAndForward('localhost',
request.body['cluster_name'],
remoteTaskName, localTask)
except errors.RemoteError, err:
# If the error is not an auth token one then kill it
# otherwise it means we think we own a cluster that
# we don't
#
# In this case another part of the system is in charge
# of forgetting about the clusters we shouldn't know
if err.name != 'igs.utils.auth_token.AuthTokenError':
log.err(err)
yield terminateCluster(credClient, persistManager,
request.body['cluster_name'],
request.body['user_name'])
else:
raise
def _handleStartCluster(request):
persistManager = request.state.persistManager
yield tasks_tx.updateTask(request.body['task_name'],
lambda t: t.setState(tasks_tx.task.TASK_RUNNING))
cluster = yield persistManager.loadCluster(request.body['cluster_name'],
request.body['user_name'])
cluster = cluster.update(startTask=request.body['task_name'])
credClient = cred_client.CredentialClient(cluster.credName, request.mq,
request.state.conf)
try:
cluster = yield instance_flow.startMaster(request.state, credClient,
request.body['task_name'],
cluster)
except Exception, err:
log.err('STARTCLUSETER: Failed')
log.err(err)
cluster = yield request.state.persistManager.loadCluster(
request.body['cluster_name'], request.body['user_name'])
cluster = cluster.setState(cluster.FAILED)
yield defer_utils.sleep(120)()
yield request.state.persistManager.removeCluster(
request.body['cluster_name'], request.body['user_name'])
raise err
def _handleTerminateCluster(request):
# Start task running
yield tasks_tx.updateTask(request.body['task_name'],
lambda t: t.setState(tasks_tx.task.TASK_RUNNING))
persistManager = request.state.persistManager
credClient = cred_client.CredentialClient('local', request.mq,
request.state.conf)
if request.body['cluster_name'] != 'local':
cluster = yield terminateRemoteCluster(request)
yield persistManager.saveCluster(cluster)
removeTerminatedCluster(persistManager, credClient,
request.body['cluster_name'],
request.body['user_name'])
else:
if ('auth_token' in request.body
and auth_token.validateToken(request.body['auth_token'])):
yield terminateCluster(credClient, persistManager, 'local',
request.body['user_name'])
removeTerminatedCluster(persistManager, credClient,
request.body['cluster_name'],
request.body['user_name'])
else:
raise auth_token.AuthTokenError()
yield tasks_tx.updateTask(request.body['task_name'],
lambda t: t.progress())
defer.returnValue(request)
def t_credential(state, value, _params):
"""
Ensure a credential exists
"""
credClient = cred_client.CredentialClient(value, state.mq, state.conf)
try:
yield credClient.listInstances()
defer.returnValue(value)
except Exception:
raise InvalidPipelineValue('"%s" is not a valid credential' %
str(value))
def _handleAddInstances(request):
yield tasks_tx.updateTask(request.body['task_name'],
lambda t: t.setState(tasks_tx.task.TASK_RUNNING))
cluster = yield request.state.persistManager.loadCluster(
request.body['cluster'], request.body['user_name'])
credClient = cred_client.CredentialClient(cluster.credName, request.mq,
request.state.conf)
cType = yield credClient.getCType()
if cType != 'local':
if request.body['num_exec'] > 0:
yield instance_flow.startExecs(
request.state, credClient, request.body['task_name'],
request.body['num_exec'],
request.body.get('exec_instance_type', None), cluster)
defer.returnValue(request)
def _handleImportCluster(request):
"""Imports a VM found on a remote host."""
persistManager = request.state.persistManager
yield tasks_tx.updateTask(request.body['task_name'],
lambda t: t.setState(tasks_tx.task.TASK_RUNNING))
cluster = yield request.state.persistManager.loadCluster(
request.body['dst_cluster'], request.body['user_name'])
cluster = cluster.update(startTask=request.body['task_name'])
credClient = cred_client.CredentialClient(cluster.credName, request.mq,
request.state.conf)
try:
cluster = yield instance_flow.importCluster(
request.state, credClient, request.body['task_name'],
request.body['host'], request.body['src_cluster'], cluster)
except Exception, err:
if isinstance(err, auth_token.AuthTokenError):
log.err('IMPORTCLUSTER: Authorization failed')
else:
log.err('IMPORTCLUSTER: Failed')
log.err(err)
cluster = yield request.state.persistManager.loadCluster(
request.body['dst_cluster'], request.body['user_name'])
log.msg('DEBUG importcluster.py: cluster -', cluster)
cluster = cluster.setState(cluster.FAILED)
yield defer_utils.sleep(120)()
log.msg('DEBUG importcluster.py: About to remove cluster')
yield request.state.persistManager.removeCluster(
request.body['dst_cluster'], request.body['user_name'])
raise err
def refreshInstances(mq, state):
"""Refreshes the local clusters instances (exec + data nodes).
Any instances that have been terminated, are unresponsive, or disappeared
since our last refresh are considered terminated and will be removed from
the local clusters instance list.
"""
persistManager = state.persistManager
updatedExecNodes = []
updatedDataNodes = []
try:
cluster = yield persistManager.loadCluster('local', None)
credClient = cred_client.CredentialClient(cluster.credName, mq,
state.conf)
# Want to make sure we only are doing this on running or unresponsive
# clusters; terminated clusters will be cleaned up elsewhere
if cluster.state in [cluster.RUNNING, cluster.UNRESPONSIVE]:
clExecNodes = cluster.execNodes
clDataNodes = cluster.dataNodes
instances = yield credClient.updateInstances(clExecNodes +
clDataNodes)
updatedExecNodes.extend([x for x in instances if x in clExecNodes])
updatedDataNodes.extend([x for x in instances if x in clDataNodes])
cluster = cluster.updateExecNodes(updatedExecNodes)
cluster = cluster.updateDataNodes(updatedDataNodes)
yield persistManager.saveCluster(cluster)
except Exception as err:
log.msg('INSTANCES REFRESH: Error')
log.err(err)
reactor.callLater(INSTANCE_REFRESH_FREQUENCY, refreshInstances, mq, state)
def terminateInstances(credentialName, instances, state):
credClient = cred_client.CredentialClient(credentialName, state.mq,
state.clustersState.conf)
yield credClient.terminateInstances(instances)
from twisted.application import service
from twisted.python import log
from igs.utils import config
from vappio_tx.mq import client
from vappio_tx.internal_client import credentials
conf = config.configFromStream(open('/mnt/vappio-conf/vappio_apps.conf'))
application = service.Application('test')
s1 = client.makeService(conf)
s1.setServiceParent(application)
cc = credentials.CredentialClient('diag', s1.mqFactory, conf)
d = cc.listInstances()
def _terminate(instances):
print 'Num instances:', len(instances)
instances = instances[:3]
print 'Shutting down: ', instances
return cc.terminateInstances(instances)
def _print(foo):
print 'Foo:', foo
def loadLocalCluster(mq, state):
"""
If local cluster is not present, load it
"""
def _credential():
if os.path.exists('/tmp/cred-info'):
cert, pkey, ctype, metadata = open('/tmp/cred-info').read().split(
'\t')
return {
'name':
'local',
'desc':
'Local credential',
'ctype':
ctype,
'cert':
open(cert).read(),
'pkey':
open(pkey).read(),
'metadata':
metadata
and dict([v.split('=', 1) for v in metadata.split(',')]) or {},
'conf':
config.configFromStream(open('/tmp/machine.conf'), lazy=True)
}
else:
return {
'name': 'local',
'desc': 'Local credential',
'ctype': 'local',
'cert': None,
'pkey': None,
'metadata': {},
'conf': config.configFromMap({})
}
try:
cluster = yield state.persistManager.loadCluster('local', None)
baseConf = config.configFromStream(open('/tmp/machine.conf'),
base=config.configFromEnv())
conf = config.configFromMap(
{
'config_loaded': True,
'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'
},
base=baseConf)
if (cluster.credName == 'local' and conf('MASTER_IP') not in [
cluster.master['public_dns'], cluster.master['private_dns']
]):
master = dict(instance_id='local',
ami_id=None,
public_dns=conf('MASTER_IP'),
private_dns=conf('MASTER_IP'),
state='running',
key=None,
index=None,
instance_type=None,
launch=None,
availability_zone=None,
monitor=None,
spot_request_id=None,
bid_price=None)
cluster = cluster.setMaster(master).update(config=conf)
yield state.persistManager.saveCluster(cluster)
defer.returnValue(cluster)
except persist.ClusterNotFoundError:
credential = _credential()
credTaskName = yield cred_client.saveCredential(
credential['name'], credential['desc'], credential['ctype'],
credential['cert'], credential['pkey'], credential['metadata'],
credential['conf'])
## Wait for credential to be added.
## TODO: Should handle failure here
yield tasks_tx.blockOnTask('localhost', 'local', credTaskName)
credClient = cred_client.CredentialClient('local', mq, state.conf)
## If it isn't a local ctype then we need to wait for
## the credential to come alive
if credential['ctype'] != 'local':
instances = yield credClient.listInstances()
else:
instances = []
baseConf = config.configFromStream(open('/tmp/machine.conf'),
base=config.configFromEnv())
conf = config.configFromMap(
{
'config_loaded': True,
'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'
},
base=baseConf)
cluster = persist.Cluster('local', None, 'local', conf)
startTaskName = yield tasks_tx.createTaskAndSave('startCluster', 1)
yield tasks_tx.updateTask(
startTaskName,
lambda t: t.setState(tasks_tx.task.TASK_COMPLETED).progress())
cluster = cluster.update(startTask=startTaskName)
masterIp = cluster.config('MASTER_IP')
masterIdx = func.find(
lambda i: masterIp in [i['public_dns'], i['private_dns']],
instances)
if masterIdx is not None:
master = instances[masterIdx]
else:
master = dict(instance_id='local',
ami_id=None,
public_dns=masterIp,
private_dns=masterIp,
state='running',
key=None,
index=None,
instance_type=None,
launch=None,
availability_zone=None,
monitor=None,
spot_request_id=None,
bid_price=None)
cluster = cluster.setMaster(master)
cluster = cluster.setState(cluster.RUNNING)
yield state.persistManager.saveCluster(cluster)
defer.returnValue(cluster)