def _blockOnTask(taskName, cluster='local'):
taskState = yield tasks.blockOnTask('localhost', cluster, taskName)
if taskState != tasks.task.TASK_COMPLETED:
raise TaskError(taskName)
defer.returnValue(taskState)
def _blockOnTask(taskName, cluster='local'):
taskState = yield tasks.blockOnTask('localhost',
cluster,
taskName)
if taskState != tasks.task.TASK_COMPLETED:
raise TaskError(taskName)
defer.returnValue(taskState)
def _run(state, batchState):
if 'state' not in batchState:
_log(batchState,
'First time running, creating pipeline state information')
batchState['pipeline_config'] = yield _applyActions(
state.innerPipelineConfig(), batchState['actions'])
batchState['pipeline_state'] = PRESTART_STATE
# We need to create a fake, local, pipeline for metrics to work
batchState['pipeline_name'] = pipeline_misc.checksumInput(
batchState['pipeline_config'])
batchState['pipeline_config']['pipeline.PIPELINE_NAME'] = batchState[
'pipeline_name']
batchState['pipeline_config'][
'pipeline.PIPELINE_WRAPPER_NAME'] = batchState['pipeline_name']
_log(batchState, 'Pipeline named ' + batchState['pipeline_name'])
pipeline = yield pipelines_client.createPipeline(
host='localhost',
clusterName='local',
userName='******',
pipelineName=batchState['pipeline_name'],
protocol='clovr_wrapper',
queue='pipeline.q',
config=batchState['pipeline_config'],
parentPipeline=state.parentPipeline())
batchState['clovr_wrapper_task_name'] = pipeline['task_name']
_log(
batchState,
'Setting number of tasks to 6 (number in a standard clovr_wrapper)'
)
yield _updateTask(batchState,
lambda t: t.update(completedTasks=0, numTasks=6))
state.updateBatchState()
else:
_log(batchState, 'Pipeline run before, loading pipeline information')
pipeline = yield pipelines_client.pipelineList(
'localhost',
'local',
'guest',
batchState['pipeline_name'],
detail=True)
batchState['state'] = RUNNING_STATE
yield _updateTask(batchState,
lambda t: t.setState(tasks.task.TASK_RUNNING))
pipelineConfigFile = os.path.join(TMP_DIR, 'pipeline_configs',
global_state.make_ref() + '.conf')
_log(batchState, 'Creating ergatis configuration')
_writeErgatisConfig(batchState['pipeline_config'], pipelineConfigFile)
if batchState['pipeline_state'] == PRESTART_STATE:
_log(batchState, 'Pipeline is in PRESTART state')
yield state.prerunQueue.addWithDeferred(
workflow_runner.run, state.workflowConfig(),
batchState['pipeline_config']['pipeline.PRESTART_TEMPLATE_XML'],
pipelineConfigFile, TMP_DIR)
yield _updateTask(
batchState, lambda t: t.addMessage(
tasks.task.MSG_SILENT, 'Completed prestart').progress())
batchState['pipeline_state'] = STARTING_STATE
state.updateBatchState()
if batchState['pipeline_state'] == STARTING_STATE:
_log(batchState, 'Pipeline is in STARTING state')
clusterTask = yield clusters_client.startCluster(
'localhost',
batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest',
int(batchState['pipeline_config']['cluster.EXEC_NODES']), 0,
batchState['pipeline_config']['cluster.CLUSTER_CREDENTIAL'], {
'cluster.master_type':
batchState['pipeline_config']['cluster.MASTER_INSTANCE_TYPE'],
'cluster.master_bid_price':
batchState['pipeline_config']['cluster.MASTER_BID_PRICE'],
'cluster.exec_type':
batchState['pipeline_config']['cluster.EXEC_INSTANCE_TYPE'],
'cluster.exec_bid_price':
batchState['pipeline_config']['cluster.EXEC_BID_PRICE']
})
taskState = yield tasks.blockOnTask('localhost', 'local', clusterTask)
if taskState != tasks.task.TASK_COMPLETED:
raise TaskError(clusterTask)
yield _updateTask(
batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT,
'Completed start').progress())
batchState['pipeline_state'] = PRERUN_STATE
state.updateBatchState()
if batchState['pipeline_state'] == PRERUN_STATE:
_log(batchState, 'Pipeline is in PRERUN state')
yield state.prerunQueue.addWithDeferred(
workflow_runner.run, state.workflowConfig(),
batchState['pipeline_config']['pipeline.PRERUN_TEMPLATE_XML'],
pipelineConfigFile, TMP_DIR)
yield _updateTask(
batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT,
'Completed prerun').progress())
batchState['pipeline_state'] = RUN_PIPELINE_STATE
state.updateBatchState()
if batchState['pipeline_state'] == RUN_PIPELINE_STATE:
_log(batchState, 'Pipeline is in RUN_PIPELINE state')
pipeline = yield pipelines_client.runPipeline(
host='localhost',
clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'],
userName='******',
parentPipeline=batchState['pipeline_name'],
bareRun=True,
queue=state.innerPipelineQueue(),
config=batchState['pipeline_config'],
overwrite=True)
batchState['pipeline_task'] = pipeline['task_name']
yield _updateTask(
batchState, lambda t: t.addMessage(
tasks.task.MSG_SILENT, 'Completed run pipeline').progress())
batchState['pipeline_state'] = RUNNING_PIPELINE_STATE
state.updateBatchState()
if batchState['pipeline_state'] == RUNNING_PIPELINE_STATE:
_log(batchState, 'Pipeline is in RUNNING_PIPELINE state')
_monitorPipeline(batchState)
yield _waitForPipeline(batchState)
yield _updateTask(
batchState,
lambda t: t.addMessage(tasks.task.MSG_SILENT,
'Completed running pipeline').progress())
batchState['pipeline_state'] = POSTRUN_STATE
state.updateBatchState()
if batchState['pipeline_state'] == POSTRUN_STATE:
_log(batchState, 'Pipeline is in POSTRUN state')
yield state.postrunQueue.addWithDeferred(
workflow_runner.run, state.workflowConfig(),
batchState['pipeline_config']['pipeline.POSTRUN_TEMPLATE_XML'],
pipelineConfigFile, TMP_DIR)
yield _updateTask(
batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT,
'Completed postrun').progress())
batchState['pipeline_state'] = COMPLETED_STATE
batchState['state'] = COMPLETED_STATE
state.updateBatchState()
yield _updateTask(batchState,
lambda t: t.setState(tasks.task.TASK_COMPLETED))
_log(batchState, 'Pipeline finished successfully')
def loadLocalCluster(mq, state):
"""
If local cluster is not present, load it
"""
def _credential():
if os.path.exists('/tmp/cred-info'):
cert, pkey, ctype, metadata = open('/tmp/cred-info').read().split('\t')
return {'name': 'local',
'desc': 'Local credential',
'ctype': ctype,
'cert': open(cert).read(),
'pkey': open(pkey).read(),
'metadata': metadata and dict([v.split('=', 1)
for v in metadata.split(',')]) or {},
'conf': config.configFromStream(open('/tmp/machine.conf'), lazy=True)}
else:
return {'name': 'local',
'desc': 'Local credential',
'ctype': 'local',
'cert': None,
'pkey': None,
'metadata': {},
'conf': config.configFromMap({})}
try:
cluster = yield state.persistManager.loadCluster('local', None)
baseConf = config.configFromStream(open('/tmp/machine.conf'),
base=config.configFromEnv())
conf = config.configFromMap({'config_loaded': True,
'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'},
base=baseConf)
if (cluster.credName == 'local' and
conf('MASTER_IP') not in [cluster.master['public_dns'],
cluster.master['private_dns']]):
master = dict(instance_id='local',
ami_id=None,
public_dns=conf('MASTER_IP'),
private_dns=conf('MASTER_IP'),
state='running',
key=None,
index=None,
instance_type=None,
launch=None,
availability_zone=None,
monitor=None,
spot_request_id=None,
bid_price=None)
cluster = cluster.setMaster(master).update(config=conf)
yield state.persistManager.saveCluster(cluster)
defer.returnValue(cluster)
except persist.ClusterNotFoundError:
credential = _credential()
credTaskName = yield cred_client.saveCredential(credential['name'],
credential['desc'],
credential['ctype'],
credential['cert'],
credential['pkey'],
credential['metadata'],
credential['conf'])
## Wait for credential to be added.
## TODO: Should handle failure here
yield tasks_tx.blockOnTask('localhost',
'local',
credTaskName)
credClient = cred_client.CredentialClient('local',
mq,
state.conf)
## If it isn't a local ctype then we need to wait for
## the credential to come alive
if credential['ctype'] != 'local':
instances = yield credClient.listInstances()
else:
instances = []
baseConf = config.configFromStream(open('/tmp/machine.conf'),
base=config.configFromEnv())
conf = config.configFromMap({'config_loaded': True,
'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'},
base=baseConf)
cluster = persist.Cluster('local',
None,
'local',
conf)
startTaskName = yield tasks_tx.createTaskAndSave('startCluster', 1)
yield tasks_tx.updateTask(startTaskName,
lambda t : t.setState(tasks_tx.task.TASK_COMPLETED).progress())
cluster = cluster.update(startTask=startTaskName)
masterIp = cluster.config('MASTER_IP')
masterIdx = func.find(lambda i : masterIp in [i['public_dns'], i['private_dns']],
instances)
if masterIdx is not None:
master = instances[masterIdx]
else:
master = dict(instance_id='local',
ami_id=None,
public_dns=masterIp,
private_dns=masterIp,
state='running',
key=None,
index=None,
instance_type=None,
launch=None,
availability_zone=None,
monitor=None,
spot_request_id=None,
bid_price=None)
cluster = cluster.setMaster(master)
cluster = cluster.setState(cluster.RUNNING)
yield state.persistManager.saveCluster(cluster)
defer.returnValue(cluster)
def loadLocalCluster(mq, state):
"""
If local cluster is not present, load it
"""
def _credential():
if os.path.exists('/tmp/cred-info'):
cert, pkey, ctype, metadata = open('/tmp/cred-info').read().split(
'\t')
return {
'name':
'local',
'desc':
'Local credential',
'ctype':
ctype,
'cert':
open(cert).read(),
'pkey':
open(pkey).read(),
'metadata':
metadata
and dict([v.split('=', 1) for v in metadata.split(',')]) or {},
'conf':
config.configFromStream(open('/tmp/machine.conf'), lazy=True)
}
else:
return {
'name': 'local',
'desc': 'Local credential',
'ctype': 'local',
'cert': None,
'pkey': None,
'metadata': {},
'conf': config.configFromMap({})
}
try:
cluster = yield state.persistManager.loadCluster('local', None)
baseConf = config.configFromStream(open('/tmp/machine.conf'),
base=config.configFromEnv())
conf = config.configFromMap(
{
'config_loaded': True,
'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'
},
base=baseConf)
if (cluster.credName == 'local' and conf('MASTER_IP') not in [
cluster.master['public_dns'], cluster.master['private_dns']
]):
master = dict(instance_id='local',
ami_id=None,
public_dns=conf('MASTER_IP'),
private_dns=conf('MASTER_IP'),
state='running',
key=None,
index=None,
instance_type=None,
launch=None,
availability_zone=None,
monitor=None,
spot_request_id=None,
bid_price=None)
cluster = cluster.setMaster(master).update(config=conf)
yield state.persistManager.saveCluster(cluster)
defer.returnValue(cluster)
except persist.ClusterNotFoundError:
credential = _credential()
credTaskName = yield cred_client.saveCredential(
credential['name'], credential['desc'], credential['ctype'],
credential['cert'], credential['pkey'], credential['metadata'],
credential['conf'])
## Wait for credential to be added.
## TODO: Should handle failure here
yield tasks_tx.blockOnTask('localhost', 'local', credTaskName)
credClient = cred_client.CredentialClient('local', mq, state.conf)
## If it isn't a local ctype then we need to wait for
## the credential to come alive
if credential['ctype'] != 'local':
instances = yield credClient.listInstances()
else:
instances = []
baseConf = config.configFromStream(open('/tmp/machine.conf'),
base=config.configFromEnv())
conf = config.configFromMap(
{
'config_loaded': True,
'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'
},
base=baseConf)
cluster = persist.Cluster('local', None, 'local', conf)
startTaskName = yield tasks_tx.createTaskAndSave('startCluster', 1)
yield tasks_tx.updateTask(
startTaskName,
lambda t: t.setState(tasks_tx.task.TASK_COMPLETED).progress())
cluster = cluster.update(startTask=startTaskName)
masterIp = cluster.config('MASTER_IP')
masterIdx = func.find(
lambda i: masterIp in [i['public_dns'], i['private_dns']],
instances)
if masterIdx is not None:
master = instances[masterIdx]
else:
master = dict(instance_id='local',
ami_id=None,
public_dns=masterIp,
private_dns=masterIp,
state='running',
key=None,
index=None,
instance_type=None,
launch=None,
availability_zone=None,
monitor=None,
spot_request_id=None,
bid_price=None)
cluster = cluster.setMaster(master)
cluster = cluster.setState(cluster.RUNNING)
yield state.persistManager.saveCluster(cluster)
defer.returnValue(cluster)
def _run(state, batchState):
if 'state' not in batchState:
_log(batchState, 'First time running, creating pipeline state information')
batchState['pipeline_config'] = yield _applyActions(state.innerPipelineConfig(),
batchState['actions'])
batchState['pipeline_state'] = PRESTART_STATE
# We need to create a fake, local, pipeline for metrics to work
batchState['pipeline_name'] = pipeline_misc.checksumInput(batchState['pipeline_config'])
batchState['pipeline_config']['pipeline.PIPELINE_NAME'] = batchState['pipeline_name']
batchState['pipeline_config']['pipeline.PIPELINE_WRAPPER_NAME'] = batchState['pipeline_name']
_log(batchState, 'Pipeline named ' + batchState['pipeline_name'])
pipeline = yield pipelines_client.createPipeline(host='localhost',
clusterName='local',
userName='******',
pipelineName=batchState['pipeline_name'],
protocol='clovr_wrapper',
queue='pipeline.q',
config=batchState['pipeline_config'],
parentPipeline=state.parentPipeline())
batchState['clovr_wrapper_task_name'] = pipeline['task_name']
_log(batchState, 'Setting number of tasks to 6 (number in a standard clovr_wrapper)')
yield _updateTask(batchState,
lambda t : t.update(completedTasks=0,
numTasks=6))
state.updateBatchState()
else:
_log(batchState, 'Pipeline run before, loading pipeline information')
pipeline = yield pipelines_client.pipelineList('localhost',
'local',
'guest',
batchState['pipeline_name'],
detail=True)
batchState['state'] = RUNNING_STATE
yield _updateTask(batchState,
lambda t : t.setState(tasks.task.TASK_RUNNING))
pipelineConfigFile = os.path.join(TMP_DIR, 'pipeline_configs', global_state.make_ref() + '.conf')
_log(batchState, 'Creating ergatis configuration')
_writeErgatisConfig(batchState['pipeline_config'], pipelineConfigFile)
if batchState['pipeline_state'] == PRESTART_STATE:
_log(batchState, 'Pipeline is in PRESTART state')
yield state.prerunQueue.addWithDeferred(workflow_runner.run,
state.workflowConfig(),
batchState['pipeline_config']['pipeline.PRESTART_TEMPLATE_XML'],
pipelineConfigFile,
TMP_DIR)
yield _updateTask(batchState,
lambda t : t.addMessage(tasks.task.MSG_SILENT,
'Completed prestart'
).progress())
batchState['pipeline_state'] = STARTING_STATE
state.updateBatchState()
if batchState['pipeline_state'] == STARTING_STATE:
_log(batchState, 'Pipeline is in STARTING state')
clusterTask = yield clusters_client.startCluster(
'localhost',
batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest',
int(batchState['pipeline_config']['cluster.EXEC_NODES']),
0,
batchState['pipeline_config']['cluster.CLUSTER_CREDENTIAL'],
{'cluster.master_type': batchState['pipeline_config']['cluster.MASTER_INSTANCE_TYPE'],
'cluster.master_bid_price': batchState['pipeline_config']['cluster.MASTER_BID_PRICE'],
'cluster.exec_type': batchState['pipeline_config']['cluster.EXEC_INSTANCE_TYPE'],
'cluster.exec_bid_price': batchState['pipeline_config']['cluster.EXEC_BID_PRICE']})
taskState = yield tasks.blockOnTask('localhost',
'local',
clusterTask)
if taskState != tasks.task.TASK_COMPLETED:
raise TaskError(clusterTask)
yield _updateTask(batchState,
lambda t : t.addMessage(tasks.task.MSG_SILENT,
'Completed start'
).progress())
batchState['pipeline_state'] = PRERUN_STATE
state.updateBatchState()
if batchState['pipeline_state'] == PRERUN_STATE:
_log(batchState, 'Pipeline is in PRERUN state')
yield state.prerunQueue.addWithDeferred(workflow_runner.run,
state.workflowConfig(),
batchState['pipeline_config']['pipeline.PRERUN_TEMPLATE_XML'],
pipelineConfigFile,
TMP_DIR)
yield _updateTask(batchState,
lambda t : t.addMessage(tasks.task.MSG_SILENT,
'Completed prerun'
).progress())
batchState['pipeline_state'] = RUN_PIPELINE_STATE
state.updateBatchState()
if batchState['pipeline_state'] == RUN_PIPELINE_STATE:
_log(batchState, 'Pipeline is in RUN_PIPELINE state')
pipeline = yield pipelines_client.runPipeline(host='localhost',
clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'],
userName='******',
parentPipeline=batchState['pipeline_name'],
bareRun=True,
queue=state.innerPipelineQueue(),
config=batchState['pipeline_config'],
overwrite=True)
batchState['pipeline_task'] = pipeline['task_name']
yield _updateTask(batchState,
lambda t : t.addMessage(tasks.task.MSG_SILENT,
'Completed run pipeline'
).progress())
batchState['pipeline_state'] = RUNNING_PIPELINE_STATE
state.updateBatchState()
if batchState['pipeline_state'] == RUNNING_PIPELINE_STATE:
_log(batchState, 'Pipeline is in RUNNING_PIPELINE state')
_monitorPipeline(batchState)
yield _waitForPipeline(batchState)
yield _updateTask(batchState,
lambda t : t.addMessage(tasks.task.MSG_SILENT,
'Completed running pipeline'
).progress())
batchState['pipeline_state'] = POSTRUN_STATE
state.updateBatchState()
if batchState['pipeline_state'] == POSTRUN_STATE:
_log(batchState, 'Pipeline is in POSTRUN state')
yield state.postrunQueue.addWithDeferred(workflow_runner.run,
state.workflowConfig(),
batchState['pipeline_config']['pipeline.POSTRUN_TEMPLATE_XML'],
pipelineConfigFile,
TMP_DIR)
yield _updateTask(batchState,
lambda t : t.addMessage(tasks.task.MSG_SILENT,
'Completed postrun'
).progress())
batchState['pipeline_state'] = COMPLETED_STATE
batchState['state'] = COMPLETED_STATE
state.updateBatchState()
yield _updateTask(batchState,
lambda t : t.setState(tasks.task.TASK_COMPLETED))
_log(batchState, 'Pipeline finished successfully')