def _instanceParse(lines):
instanceSet = set([
i.instanceId for i in instances
if i.instanceId and not i.spotRequestId
])
spotRequestSet = set(
[i.spotRequestId for i in instances if i.spotRequestId])
spotInstanceSet = set()
spotInstances = []
for line in lines:
instance = parseInstanceLine(line)
if instance and (instance.instanceId
and instance.instanceId in instanceSet
or instance.spotRequestId
and instance.spotRequestId in spotRequestSet):
if not instance.instanceId and instance.spotRequestId:
retInst.append(instance)
elif instance.instanceId and instance.spotRequestId:
spotInstanceSet.add(instance.instanceId)
instanceSet.add(instance.instanceId)
spotInstances.append(instance)
elif instance.instanceId in spotInstanceSet:
idx = functional.find(
lambda i: i.instanceId == instance.instanceId,
spotInstances)
t = spotInstances[idx]
instance.spotRequestId = t.spotRequestId
instance.bidPrice = t.bidPrice
retInst.append(instance)
elif instance.instanceId in instanceSet:
retInst.append(instance)
return retInst
def _instanceParse(lines):
instanceSet = set([i.instanceId for i in instances if i.instanceId and not i.spotRequestId])
spotRequestSet = set([i.spotRequestId for i in instances if i.spotRequestId])
spotInstanceSet = set()
spotInstances = []
for line in lines:
instance = parseInstanceLine(line)
if instance and (instance.instanceId and instance.instanceId in instanceSet or
instance.spotRequestId and instance.spotRequestId in spotRequestSet):
if not instance.instanceId and instance.spotRequestId:
retInst.append(instance)
elif instance.instanceId and instance.spotRequestId:
spotInstanceSet.add(instance.instanceId)
instanceSet.add(instance.instanceId)
spotInstances.append(instance)
elif instance.instanceId in spotInstanceSet:
idx = functional.find(lambda i: i.instanceId == instance.instanceId, spotInstances)
t = spotInstances[idx]
instance.spotRequestId = t.spotRequestId
instance.bidPrice = t.bidPrice
retInst.append(instance)
elif instance.instanceId in instanceSet:
retInst.append(instance)
return retInst
def _instanceParse(line):
instance = parseInstanceLine(line)
if instance and (instance.instanceId and instance.instanceId
in instanceSet or instance.spotRequestId
and instance.spotRequestId in spotRequestSet):
if not instance.instanceId and instance.spotRequestId:
retInst.append(instance)
elif instance.instanceId and instance.spotRequestId:
spotInstanceSet.add(instance.instanceId)
instanceSet.add(instance.instanceId)
spotInstances.append(instance)
elif instance.instanceId in spotInstanceSet:
idx = functional.find(
lambda i: i.instanceId == instance.instanceId,
spotInstances)
t = spotInstances[idx]
instance.spotRequestId = t.spotRequestId
instance.bidPrice = t.bidPrice
retInst.append(instance)
elif instance.instanceId in instanceSet:
retInst.append(instance)
def _instanceParse(line):
instance = parseInstanceLine(line)
if instance and (
instance.instanceId
and instance.instanceId in instanceSet
or instance.spotRequestId
and instance.spotRequestId in spotRequestSet
):
if not instance.instanceId and instance.spotRequestId:
retInst.append(instance)
elif instance.instanceId and instance.spotRequestId:
spotInstanceSet.add(instance.instanceId)
instanceSet.add(instance.instanceId)
spotInstances.append(instance)
elif instance.instanceId in spotInstanceSet:
idx = functional.find(lambda i: i.instanceId == instance.instanceId, spotInstances)
t = spotInstances[idx]
instance.spotRequestId = t.spotRequestId
instance.bidPrice = t.bidPrice
retInst.append(instance)
elif instance.instanceId in instanceSet:
retInst.append(instance)
def loadLocalCluster(mq, state):
"""
If local cluster is not present, load it
"""
def _credential():
if os.path.exists('/tmp/cred-info'):
cert, pkey, ctype, metadata = open('/tmp/cred-info').read().split('\t')
return {'name': 'local',
'desc': 'Local credential',
'ctype': ctype,
'cert': open(cert).read(),
'pkey': open(pkey).read(),
'metadata': metadata and dict([v.split('=', 1)
for v in metadata.split(',')]) or {},
'conf': config.configFromStream(open('/tmp/machine.conf'), lazy=True)}
else:
return {'name': 'local',
'desc': 'Local credential',
'ctype': 'local',
'cert': None,
'pkey': None,
'metadata': {},
'conf': config.configFromMap({})}
try:
cluster = yield state.persistManager.loadCluster('local', None)
baseConf = config.configFromStream(open('/tmp/machine.conf'),
base=config.configFromEnv())
conf = config.configFromMap({'config_loaded': True,
'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'},
base=baseConf)
if (cluster.credName == 'local' and
conf('MASTER_IP') not in [cluster.master['public_dns'],
cluster.master['private_dns']]):
master = dict(instance_id='local',
ami_id=None,
public_dns=conf('MASTER_IP'),
private_dns=conf('MASTER_IP'),
state='running',
key=None,
index=None,
instance_type=None,
launch=None,
availability_zone=None,
monitor=None,
spot_request_id=None,
bid_price=None)
cluster = cluster.setMaster(master).update(config=conf)
yield state.persistManager.saveCluster(cluster)
defer.returnValue(cluster)
except persist.ClusterNotFoundError:
credential = _credential()
credTaskName = yield cred_client.saveCredential(credential['name'],
credential['desc'],
credential['ctype'],
credential['cert'],
credential['pkey'],
credential['metadata'],
credential['conf'])
## Wait for credential to be added.
## TODO: Should handle failure here
yield tasks_tx.blockOnTask('localhost',
'local',
credTaskName)
credClient = cred_client.CredentialClient('local',
mq,
state.conf)
## If it isn't a local ctype then we need to wait for
## the credential to come alive
if credential['ctype'] != 'local':
instances = yield credClient.listInstances()
else:
instances = []
baseConf = config.configFromStream(open('/tmp/machine.conf'),
base=config.configFromEnv())
conf = config.configFromMap({'config_loaded': True,
'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'},
base=baseConf)
cluster = persist.Cluster('local',
None,
'local',
conf)
startTaskName = yield tasks_tx.createTaskAndSave('startCluster', 1)
yield tasks_tx.updateTask(startTaskName,
lambda t : t.setState(tasks_tx.task.TASK_COMPLETED).progress())
cluster = cluster.update(startTask=startTaskName)
masterIp = cluster.config('MASTER_IP')
masterIdx = func.find(lambda i : masterIp in [i['public_dns'], i['private_dns']],
instances)
if masterIdx is not None:
master = instances[masterIdx]
else:
master = dict(instance_id='local',
ami_id=None,
public_dns=masterIp,
private_dns=masterIp,
state='running',
key=None,
index=None,
instance_type=None,
launch=None,
availability_zone=None,
monitor=None,
spot_request_id=None,
bid_price=None)
cluster = cluster.setMaster(master)
cluster = cluster.setState(cluster.RUNNING)
yield state.persistManager.saveCluster(cluster)
defer.returnValue(cluster)
def loadLocalCluster(mq, state):
"""
If local cluster is not present, load it
"""
def _credential():
if os.path.exists('/tmp/cred-info'):
cert, pkey, ctype, metadata = open('/tmp/cred-info').read().split(
'\t')
return {
'name':
'local',
'desc':
'Local credential',
'ctype':
ctype,
'cert':
open(cert).read(),
'pkey':
open(pkey).read(),
'metadata':
metadata
and dict([v.split('=', 1) for v in metadata.split(',')]) or {},
'conf':
config.configFromStream(open('/tmp/machine.conf'), lazy=True)
}
else:
return {
'name': 'local',
'desc': 'Local credential',
'ctype': 'local',
'cert': None,
'pkey': None,
'metadata': {},
'conf': config.configFromMap({})
}
try:
cluster = yield state.persistManager.loadCluster('local', None)
baseConf = config.configFromStream(open('/tmp/machine.conf'),
base=config.configFromEnv())
conf = config.configFromMap(
{
'config_loaded': True,
'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'
},
base=baseConf)
if (cluster.credName == 'local' and conf('MASTER_IP') not in [
cluster.master['public_dns'], cluster.master['private_dns']
]):
master = dict(instance_id='local',
ami_id=None,
public_dns=conf('MASTER_IP'),
private_dns=conf('MASTER_IP'),
state='running',
key=None,
index=None,
instance_type=None,
launch=None,
availability_zone=None,
monitor=None,
spot_request_id=None,
bid_price=None)
cluster = cluster.setMaster(master).update(config=conf)
yield state.persistManager.saveCluster(cluster)
defer.returnValue(cluster)
except persist.ClusterNotFoundError:
credential = _credential()
credTaskName = yield cred_client.saveCredential(
credential['name'], credential['desc'], credential['ctype'],
credential['cert'], credential['pkey'], credential['metadata'],
credential['conf'])
## Wait for credential to be added.
## TODO: Should handle failure here
yield tasks_tx.blockOnTask('localhost', 'local', credTaskName)
credClient = cred_client.CredentialClient('local', mq, state.conf)
## If it isn't a local ctype then we need to wait for
## the credential to come alive
if credential['ctype'] != 'local':
instances = yield credClient.listInstances()
else:
instances = []
baseConf = config.configFromStream(open('/tmp/machine.conf'),
base=config.configFromEnv())
conf = config.configFromMap(
{
'config_loaded': True,
'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'
},
base=baseConf)
cluster = persist.Cluster('local', None, 'local', conf)
startTaskName = yield tasks_tx.createTaskAndSave('startCluster', 1)
yield tasks_tx.updateTask(
startTaskName,
lambda t: t.setState(tasks_tx.task.TASK_COMPLETED).progress())
cluster = cluster.update(startTask=startTaskName)
masterIp = cluster.config('MASTER_IP')
masterIdx = func.find(
lambda i: masterIp in [i['public_dns'], i['private_dns']],
instances)
if masterIdx is not None:
master = instances[masterIdx]
else:
master = dict(instance_id='local',
ami_id=None,
public_dns=masterIp,
private_dns=masterIp,
state='running',
key=None,
index=None,
instance_type=None,
launch=None,
availability_zone=None,
monitor=None,
spot_request_id=None,
bid_price=None)
cluster = cluster.setMaster(master)
cluster = cluster.setState(cluster.RUNNING)
yield state.persistManager.saveCluster(cluster)
defer.returnValue(cluster)
