def _instanceParse(lines): instanceSet = set([ i.instanceId for i in instances if i.instanceId and not i.spotRequestId ]) spotRequestSet = set( [i.spotRequestId for i in instances if i.spotRequestId]) spotInstanceSet = set() spotInstances = [] for line in lines: instance = parseInstanceLine(line) if instance and (instance.instanceId and instance.instanceId in instanceSet or instance.spotRequestId and instance.spotRequestId in spotRequestSet): if not instance.instanceId and instance.spotRequestId: retInst.append(instance) elif instance.instanceId and instance.spotRequestId: spotInstanceSet.add(instance.instanceId) instanceSet.add(instance.instanceId) spotInstances.append(instance) elif instance.instanceId in spotInstanceSet: idx = functional.find( lambda i: i.instanceId == instance.instanceId, spotInstances) t = spotInstances[idx] instance.spotRequestId = t.spotRequestId instance.bidPrice = t.bidPrice retInst.append(instance) elif instance.instanceId in instanceSet: retInst.append(instance) return retInst
def _instanceParse(lines): instanceSet = set([i.instanceId for i in instances if i.instanceId and not i.spotRequestId]) spotRequestSet = set([i.spotRequestId for i in instances if i.spotRequestId]) spotInstanceSet = set() spotInstances = [] for line in lines: instance = parseInstanceLine(line) if instance and (instance.instanceId and instance.instanceId in instanceSet or instance.spotRequestId and instance.spotRequestId in spotRequestSet): if not instance.instanceId and instance.spotRequestId: retInst.append(instance) elif instance.instanceId and instance.spotRequestId: spotInstanceSet.add(instance.instanceId) instanceSet.add(instance.instanceId) spotInstances.append(instance) elif instance.instanceId in spotInstanceSet: idx = functional.find(lambda i: i.instanceId == instance.instanceId, spotInstances) t = spotInstances[idx] instance.spotRequestId = t.spotRequestId instance.bidPrice = t.bidPrice retInst.append(instance) elif instance.instanceId in instanceSet: retInst.append(instance) return retInst
def _instanceParse(line): instance = parseInstanceLine(line) if instance and (instance.instanceId and instance.instanceId in instanceSet or instance.spotRequestId and instance.spotRequestId in spotRequestSet): if not instance.instanceId and instance.spotRequestId: retInst.append(instance) elif instance.instanceId and instance.spotRequestId: spotInstanceSet.add(instance.instanceId) instanceSet.add(instance.instanceId) spotInstances.append(instance) elif instance.instanceId in spotInstanceSet: idx = functional.find( lambda i: i.instanceId == instance.instanceId, spotInstances) t = spotInstances[idx] instance.spotRequestId = t.spotRequestId instance.bidPrice = t.bidPrice retInst.append(instance) elif instance.instanceId in instanceSet: retInst.append(instance)
def _instanceParse(line): instance = parseInstanceLine(line) if instance and ( instance.instanceId and instance.instanceId in instanceSet or instance.spotRequestId and instance.spotRequestId in spotRequestSet ): if not instance.instanceId and instance.spotRequestId: retInst.append(instance) elif instance.instanceId and instance.spotRequestId: spotInstanceSet.add(instance.instanceId) instanceSet.add(instance.instanceId) spotInstances.append(instance) elif instance.instanceId in spotInstanceSet: idx = functional.find(lambda i: i.instanceId == instance.instanceId, spotInstances) t = spotInstances[idx] instance.spotRequestId = t.spotRequestId instance.bidPrice = t.bidPrice retInst.append(instance) elif instance.instanceId in instanceSet: retInst.append(instance)
def loadLocalCluster(mq, state): """ If local cluster is not present, load it """ def _credential(): if os.path.exists('/tmp/cred-info'): cert, pkey, ctype, metadata = open('/tmp/cred-info').read().split('\t') return {'name': 'local', 'desc': 'Local credential', 'ctype': ctype, 'cert': open(cert).read(), 'pkey': open(pkey).read(), 'metadata': metadata and dict([v.split('=', 1) for v in metadata.split(',')]) or {}, 'conf': config.configFromStream(open('/tmp/machine.conf'), lazy=True)} else: return {'name': 'local', 'desc': 'Local credential', 'ctype': 'local', 'cert': None, 'pkey': None, 'metadata': {}, 'conf': config.configFromMap({})} try: cluster = yield state.persistManager.loadCluster('local', None) baseConf = config.configFromStream(open('/tmp/machine.conf'), base=config.configFromEnv()) conf = config.configFromMap({'config_loaded': True, 'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'}, base=baseConf) if (cluster.credName == 'local' and conf('MASTER_IP') not in [cluster.master['public_dns'], cluster.master['private_dns']]): master = dict(instance_id='local', ami_id=None, public_dns=conf('MASTER_IP'), private_dns=conf('MASTER_IP'), state='running', key=None, index=None, instance_type=None, launch=None, availability_zone=None, monitor=None, spot_request_id=None, bid_price=None) cluster = cluster.setMaster(master).update(config=conf) yield state.persistManager.saveCluster(cluster) defer.returnValue(cluster) except persist.ClusterNotFoundError: credential = _credential() credTaskName = yield cred_client.saveCredential(credential['name'], credential['desc'], credential['ctype'], credential['cert'], credential['pkey'], credential['metadata'], credential['conf']) ## Wait for credential to be added. ## TODO: Should handle failure here yield tasks_tx.blockOnTask('localhost', 'local', credTaskName) credClient = cred_client.CredentialClient('local', mq, state.conf) ## If it isn't a local ctype then we need to wait for ## the credential to come alive if credential['ctype'] != 'local': instances = yield credClient.listInstances() else: instances = [] baseConf = config.configFromStream(open('/tmp/machine.conf'), base=config.configFromEnv()) conf = config.configFromMap({'config_loaded': True, 'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'}, base=baseConf) cluster = persist.Cluster('local', None, 'local', conf) startTaskName = yield tasks_tx.createTaskAndSave('startCluster', 1) yield tasks_tx.updateTask(startTaskName, lambda t : t.setState(tasks_tx.task.TASK_COMPLETED).progress()) cluster = cluster.update(startTask=startTaskName) masterIp = cluster.config('MASTER_IP') masterIdx = func.find(lambda i : masterIp in [i['public_dns'], i['private_dns']], instances) if masterIdx is not None: master = instances[masterIdx] else: master = dict(instance_id='local', ami_id=None, public_dns=masterIp, private_dns=masterIp, state='running', key=None, index=None, instance_type=None, launch=None, availability_zone=None, monitor=None, spot_request_id=None, bid_price=None) cluster = cluster.setMaster(master) cluster = cluster.setState(cluster.RUNNING) yield state.persistManager.saveCluster(cluster) defer.returnValue(cluster)
def loadLocalCluster(mq, state): """ If local cluster is not present, load it """ def _credential(): if os.path.exists('/tmp/cred-info'): cert, pkey, ctype, metadata = open('/tmp/cred-info').read().split( '\t') return { 'name': 'local', 'desc': 'Local credential', 'ctype': ctype, 'cert': open(cert).read(), 'pkey': open(pkey).read(), 'metadata': metadata and dict([v.split('=', 1) for v in metadata.split(',')]) or {}, 'conf': config.configFromStream(open('/tmp/machine.conf'), lazy=True) } else: return { 'name': 'local', 'desc': 'Local credential', 'ctype': 'local', 'cert': None, 'pkey': None, 'metadata': {}, 'conf': config.configFromMap({}) } try: cluster = yield state.persistManager.loadCluster('local', None) baseConf = config.configFromStream(open('/tmp/machine.conf'), base=config.configFromEnv()) conf = config.configFromMap( { 'config_loaded': True, 'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub' }, base=baseConf) if (cluster.credName == 'local' and conf('MASTER_IP') not in [ cluster.master['public_dns'], cluster.master['private_dns'] ]): master = dict(instance_id='local', ami_id=None, public_dns=conf('MASTER_IP'), private_dns=conf('MASTER_IP'), state='running', key=None, index=None, instance_type=None, launch=None, availability_zone=None, monitor=None, spot_request_id=None, bid_price=None) cluster = cluster.setMaster(master).update(config=conf) yield state.persistManager.saveCluster(cluster) defer.returnValue(cluster) except persist.ClusterNotFoundError: credential = _credential() credTaskName = yield cred_client.saveCredential( credential['name'], credential['desc'], credential['ctype'], credential['cert'], credential['pkey'], credential['metadata'], credential['conf']) ## Wait for credential to be added. ## TODO: Should handle failure here yield tasks_tx.blockOnTask('localhost', 'local', credTaskName) credClient = cred_client.CredentialClient('local', mq, state.conf) ## If it isn't a local ctype then we need to wait for ## the credential to come alive if credential['ctype'] != 'local': instances = yield credClient.listInstances() else: instances = [] baseConf = config.configFromStream(open('/tmp/machine.conf'), base=config.configFromEnv()) conf = config.configFromMap( { 'config_loaded': True, 'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub' }, base=baseConf) cluster = persist.Cluster('local', None, 'local', conf) startTaskName = yield tasks_tx.createTaskAndSave('startCluster', 1) yield tasks_tx.updateTask( startTaskName, lambda t: t.setState(tasks_tx.task.TASK_COMPLETED).progress()) cluster = cluster.update(startTask=startTaskName) masterIp = cluster.config('MASTER_IP') masterIdx = func.find( lambda i: masterIp in [i['public_dns'], i['private_dns']], instances) if masterIdx is not None: master = instances[masterIdx] else: master = dict(instance_id='local', ami_id=None, public_dns=masterIp, private_dns=masterIp, state='running', key=None, index=None, instance_type=None, launch=None, availability_zone=None, monitor=None, spot_request_id=None, bid_price=None) cluster = cluster.setMaster(master) cluster = cluster.setState(cluster.RUNNING) yield state.persistManager.saveCluster(cluster) defer.returnValue(cluster)