def _handleTerminateInstances(request):
yield tasks_tx.updateTask(request.body['task_name'],
lambda t: t.setState(tasks_tx.task.TASK_RUNNING))
persistManager = request.state.persistManager
cluster = yield persistManager.loadCluster(request.body['cluster_name'],
request.body['user_name'])
credClient = cred_client.CredentialClient(cluster.credName, request.mq,
request.state.conf)
if request.body['cluster_name'] != 'local':
try:
remoteTaskName = yield clusters_client_www.terminateInstances(
cluster.master['public_dns'], 'local',
request.body['user_name'], request.body['by_attribute'],
request.body['attribute_values'])
localTask = yield tasks_tx.loadTask(request.body['task_name'])
yield tasks_tx.blockOnTaskAndForward('localhost',
request.body['cluster_name'],
remoteTaskName, localTask)
except:
yield terminateInstancesByAttribute(
persistManager, credClient, request.body['cluster_name'],
request.body['user_name'], request.body['by_attribute'],
request.body['attribute_values'])
else:
cl = yield terminateInstancesByAttribute(
persistManager, credClient, 'local', None,
request.body['by_attribute'], request.body['attribute_values'])
yield tasks_tx.updateTask(request.body['task_name'],
lambda t: t.progress())
defer.returnValue(request)
def terminateRemoteCluster(request):
persistManager = request.state.persistManager
cluster = yield persistManager.loadCluster(request.body["cluster_name"], request.body["user_name"])
authToken = auth_token.generateToken(cluster.config("cluster.cluster_public_key"))
credClient = cred_client.CredentialClient(cluster.credName, request.mq, request.state.conf)
try:
if cluster.master:
wwwTerminateCluster = clusters_client_www.terminateCluster
remoteTaskName = yield wwwTerminateCluster(cluster.master["public_dns"], "local", None, authToken)
localTask = yield tasks_tx.loadTask(request.body["task_name"])
yield tasks_tx.blockOnTaskAndForward("localhost", request.body["cluster_name"], remoteTaskName, localTask)
except errors.RemoteError, err:
# If the error is not an auth token one then kill it
# otherwise it means we think we own a cluster that
# we don't
#
# In this case another part of the system is in charge
# of forgetting about the clusters we shouldn't know
if err.name != "igs.utils.auth_token.AuthTokenError":
log.err(err)
yield terminateCluster(credClient, persistManager, request.body["cluster_name"], request.body["user_name"])
else:
raise
def terminateRemoteCluster(request):
persistManager = request.state.persistManager
cluster = yield persistManager.loadCluster(request.body['cluster_name'],
request.body['user_name'])
authToken = auth_token.generateToken(
cluster.config('cluster.cluster_public_key'))
credClient = cred_client.CredentialClient(cluster.credName, request.mq,
request.state.conf)
try:
if cluster.master:
wwwTerminateCluster = clusters_client_www.terminateCluster
remoteTaskName = yield wwwTerminateCluster(
cluster.master['public_dns'], 'local', None, authToken)
localTask = yield tasks_tx.loadTask(request.body['task_name'])
yield tasks_tx.blockOnTaskAndForward('localhost',
request.body['cluster_name'],
remoteTaskName, localTask)
except errors.RemoteError, err:
# If the error is not an auth token one then kill it
# otherwise it means we think we own a cluster that
# we don't
#
# In this case another part of the system is in charge
# of forgetting about the clusters we shouldn't know
if err.name != 'igs.utils.auth_token.AuthTokenError':
log.err(err)
yield terminateCluster(credClient, persistManager,
request.body['cluster_name'],
request.body['user_name'])
else:
raise
def _tagToDictAndCache(self, aspect, tag):
if tag.taskName:
try:
t = yield tasks_tx.loadTask(tag.taskName)
state = t.state
except:
state = None
else:
state = None
d = {}
d.update({"files": tag.files, "metadata": tag.metadata})
d.update(
{
"tag_name": tag.tagName,
"file_count": len(tag.files),
"pipelines": [],
"task_name": tag.taskName,
"state": state,
"phantom": config.configToDict(tag.phantom) if tag.phantom else None,
}
)
yield self.cache.save(d)
self.changed(aspect, d)
def pipelineToDict(self, pipeline):
protocolConf = protocol_format.load(self.machineConf, pipeline.config("pipeline.PIPELINE_TEMPLATE"))
inputTagsList = [
pipeline.config(k).split(",")
for k, v in protocolConf
if v.get("type").split()[0] in ["dataset", "blastdb_dataset", "paired_dataset", "singleton_dataset"]
and pipeline.config(k)
]
inputTags = []
for i in inputTagsList:
inputTags.extend(i)
possibleOutputTags = set(
[
pipeline.pipelineName + "_" + t.strip()
for t in pipeline.config("output.TAGS_TO_DOWNLOAD", default="").split(",")
]
)
query = [{"tag_name": t} for t in possibleOutputTags]
tags = yield www_tags.loadTagsBy("localhost", "local", pipeline.userName, {"$or": query}, False)
tags = set([t["tag_name"] for t in tags])
outputTags = list(tags & possibleOutputTags)
pipelineTask = yield tasks_tx.loadTask(pipeline.taskName)
pipelineWrapper = pipeline_misc.determineWrapper(
self.machineConf, pipeline.config("pipeline.PIPELINE_TEMPLATE")
)
pipelineDict = {
"pipeline_id": pipeline.pipelineId,
"pipeline_name": pipeline.pipelineName,
"user_name": pipeline.userName,
"wrapper": pipeline.protocol == pipelineWrapper,
"protocol": pipeline.config("pipeline.PIPELINE_TEMPLATE"),
"checksum": pipeline.checksum,
"task_name": pipeline.taskName,
"queue": pipeline.queue,
"children": pipeline.children,
"state": pipelineTask.state,
"num_steps": pipelineTask.numTasks,
"num_complete": pipelineTask.completedTasks,
"input_tags": inputTags,
"output_tags": outputTags,
"pipeline_desc": pipeline.config("pipeline.PIPELINE_DESC", default=""),
"config": config.configToDict(pipeline.config, lazy=True),
}
defer.returnValue(pipelineDict)
def pipelineToDict(self, pipeline):
protocolConf = protocol_format.load(self.machineConf, pipeline.config('pipeline.PIPELINE_TEMPLATE'))
inputTagsList = [pipeline.config(k).split(',')
for k, v in protocolConf
if v.get('type').split()[0] in ['dataset',
'blastdb_dataset',
'paired_dataset',
'singleton_dataset'] and pipeline.config(k)]
inputTags = []
for i in inputTagsList:
inputTags.extend(i)
possibleOutputTags = set([pipeline.pipelineName + '_' + t.strip()
for t in pipeline.config('output.TAGS_TO_DOWNLOAD', default='').split(',')])
query = [{'tag_name': t} for t in possibleOutputTags]
tags = yield www_tags.loadTagsBy('localhost', 'local', pipeline.userName, {'$or': query}, False)
tags = set([t['tag_name'] for t in tags])
outputTags = list(tags & possibleOutputTags)
pipelineTask = yield tasks_tx.loadTask(pipeline.taskName)
pipelineWrapper = pipeline_misc.determineWrapper(self.machineConf,
pipeline.config('pipeline.PIPELINE_TEMPLATE'))
pipelineDict = {'pipeline_id': pipeline.pipelineId,
'pipeline_name': pipeline.pipelineName,
'user_name': pipeline.userName,
'wrapper': pipeline.protocol == pipelineWrapper,
'protocol': pipeline.config('pipeline.PIPELINE_TEMPLATE'),
'checksum': pipeline.checksum,
'task_name': pipeline.taskName,
'queue': pipeline.queue,
'children': pipeline.children,
'state': pipelineTask.state,
'num_steps': pipelineTask.numTasks,
'num_complete': pipelineTask.completedTasks,
'input_tags': inputTags,
'output_tags': outputTags,
'pipeline_desc': pipeline.config('pipeline.PIPELINE_DESC', default=''),
'config': config.configToDict(pipeline.config, lazy=True),
}
defer.returnValue(pipelineDict)
def _handleTerminateInstances(request):
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.setState(tasks_tx.task.TASK_RUNNING))
persistManager = request.state.persistManager
cluster = yield persistManager.loadCluster(request.body['cluster_name'],
request.body['user_name'])
credClient = cred_client.CredentialClient(cluster.credName,
request.mq,
request.state.conf)
if request.body['cluster_name'] != 'local':
try:
remoteTaskName = yield clusters_client_www.terminateInstances(
cluster.master['public_dns'],
'local',
request.body['user_name'],
request.body['by_attribute'],
request.body['attribute_values'])
localTask = yield tasks_tx.loadTask(request.body['task_name'])
yield tasks_tx.blockOnTaskAndForward('localhost',
request.body['cluster_name'],
remoteTaskName,
localTask)
except:
yield terminateInstancesByAttribute(persistManager,
credClient,
request.body['cluster_name'],
request.body['user_name'],
request.body['by_attribute'],
request.body['attribute_values'])
else:
cl = yield terminateInstancesByAttribute(persistManager,
credClient,
'local',
None,
request.body['by_attribute'],
request.body['attribute_values'])
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.progress())
defer.returnValue(request)
def _tagToDictAndCache(self, aspect, tag):
if tag.taskName:
try:
t = yield tasks_tx.loadTask(tag.taskName)
state = t.state
except:
state = None
else:
state = None
d = {}
d.update({'files': tag.files,
'metadata': tag.metadata})
d.update({'tag_name': tag.tagName,
'file_count': len(tag.files),
'pipelines': [],
'task_name': tag.taskName,
'state': state,
'phantom': config.configToDict(tag.phantom) if tag.phantom else None})
yield self.cache.save(d)
self.changed(aspect, d)
def _realizeUrls(request):
localTag = yield request.state.tagPersist.loadTag(request.body['tag_name'])
# If we have urls we create a fake phantom tag
fakePhantom = {'cluster.ALL.command':
'reliableDownloader.py -m 300 -t 20 -b ${base_dir} ' + ' '.join(localTag.metadata['urls'])}
taskName = yield www_tags.realizePhantom('localhost',
request.body['dst_cluster'],
request.body['user_name'],
localTag.tagName,
fakePhantom,
func.updateDict(localTag.metadata, {'urls_realized': True}))
localTask = yield tasks_tx.loadTask(request.body['task_name'])
endState, tsk = yield tasks_tx.blockOnTaskAndForward('localhost',
request.body['dst_cluster'],
taskName,
localTask)
if endState == tasks_tx.task.TASK_FAILED:
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.setState(tasks_tx.task.TASK_FAILED))
raise RealizePhantomError(request.body['tag_name'])
if request.body['dst_cluster'] == 'local':
yield tag_mq_data.tagData(request.state,
request.body['tag_name'],
request.body['task_name'],
files=localTag.files,
action=tag_mq_data.ACTION_APPEND,
metadata={},
recursive=False,
expand=False,
compressDir=None)
else:
localTask = yield www_tags.tagData('localhost',
request.body['dst_cluster'],
request.body['user_name'],
action=tag_mq_data.ACTION_APPEND,
tagName=localTag.tagName,
files=localTag.files,
metadata={},
recursive=False,
expand=False,
compressDir=None)
localTask = yield tasks_tx.loadTask(request.body['task_name'])
endState, tsk = yield tasks_tx.blockOnTaskAndForward('localhost',
request.body['dst_cluster'],
taskName,
localTask)
if endState == tasks_tx.task.TASK_FAILED:
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.setState(tasks_tx.task.TASK_FAILED))
raise RealizePhantomError(request.body['tag_name'])
# Load the tag up and return it so we can have the files it created
tag = yield www_tags.loadTag('localhost',
request.body['dst_cluster'],
request.body['user_name'],
request.body['tag_name'])
defer.returnValue(tag)
def _handleTransferTag(request):
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.setState(tasks_tx.task.TASK_RUNNING).update(numTasks=2))
srcTag = yield www_tags.loadTag('localhost',
request.body['src_cluster'],
request.body['user_name'],
request.body['tag_name'])
if not srcTag['phantom'] and (request.body['src_cluster'] != 'local' or request.body['dst_cluster'] != 'local'):
if request.body['src_cluster'] == 'local':
tag = yield _uploadTag(request)
elif request.body['dst_cluster'] == 'local':
tag = yield _downloadTag(request)
else:
raise NoLocalClusterError('Source cluster or destination cluster must be local')
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.progress())
if request.body.get('compress', False) or request.body.get('compress_dir', False):
defaultDir = '/mnt/output' if request.body['dst_cluster'] == 'local' else tag.metadata['tag_base_dir']
compressDir = request.body.get('compress_dir') if request.body.get('compress_dir', False) else defaultDir
else:
compressDir = None
if request.body['dst_cluster'] == 'local':
yield tag_mq_data.tagData(request.state,
request.body['tag_name'],
request.body['task_name'],
files=tag.files,
action=tag_mq_data.ACTION_OVERWRITE,
metadata=tag.metadata,
recursive=False,
expand=False,
compressDir=compressDir)
else:
newTag = yield www_tags.tagData('localhost',
request.body['dst_cluster'],
request.body['user_name'],
action=tag_mq_data.ACTION_OVERWRITE,
tagName=tag.tagName,
files=tag.files,
metadata=tag.metadata,
recursive=False,
expand=False,
compressDir=compressDir)
localTask = yield tasks_tx.loadTask(request.body['task_name'])
endState, tsk = yield tasks_tx.blockOnTaskAndForward('localhost',
request.body['dst_cluster'],
newTag['task_name'],
localTask)
if endState == tasks_tx.task.TASK_FAILED:
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.setState(tasks_tx.task.TASK_FAILED))
raise TransferTagError(request.body['tag_name'])
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.progress())
elif not srcTag['phantom'] and srcTag['metadata'].get('urls', []) and not srcTag['metadata'].get('urls_realized', False):
# It's a local to local but we have urls and haven't realized them
yield _realizeUrls(request)
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.progress(2))
elif srcTag['phantom']:
# Upload the depends file
srcClusters = yield www_clusters.listClusters('localhost',
{'cluster_name': request.body['src_cluster']},
request.body['user_name'])
srcCluster = srcClusters[0]
dstClusters = yield www_clusters.listClusters('localhost',
{'cluster_name': request.body['dst_cluster']},
request.body['user_name'])
dstCluster = dstClusters[0]
dependsOn = srcTag['phantom'].get('depends_on', '').split()
yield rsync.rsyncTo(dstCluster['master']['public_dns'],
'/',
'/',
dependsOn,
srcCluster['config']['rsync.options'],
srcCluster['config']['rsync.user'],
log=True)
taskName = yield www_tags.realizePhantom('localhost',
request.body['dst_cluster'],
request.body['user_name'],
srcTag['tag_name'],
srcTag['phantom'],
srcTag['metadata'])
localTask = yield tasks_tx.loadTask(request.body['task_name'])
endState, tsk = yield tasks_tx.blockOnTaskAndForward('localhost',
request.body['dst_cluster'],
taskName,
localTask)
if endState == tasks_tx.task.TASK_FAILED:
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.setState(tasks_tx.task.TASK_FAILED))
raise RealizePhantomError(request.body['tag_name'])
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.update(numTasks=1).progress())
else:
yield tag_mq_data.tagData(request.state,
request.body['tag_name'],
request.body['task_name'],
files=[],
action=tag_mq_data.ACTION_APPEND,
metadata={},
recursive=False,
expand=False,
compressDir='/mnt/output' if request.body.get('compress', False) else None)
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.progress(2))
defer.returnValue(request)
log.err('STARTCLUSETER: Failed')
log.err(err)
cluster = yield request.state.persistManager.loadCluster(
request.body['cluster_name'], request.body['user_name'])
cluster = cluster.setState(cluster.FAILED)
yield defer_utils.sleep(120)()
yield request.state.persistManager.removeCluster(
request.body['cluster_name'], request.body['user_name'])
raise err
if request.body['num_exec'] > 0 or request.body['num_data'] > 0:
addInstancesTaskName = yield clusters_client_www.addInstances(
'localhost', request.body['cluster_name'],
request.body['user_name'], request.body['num_exec'],
request.body['num_data'])
localTask = yield tasks_tx.loadTask(request.body['task_name'])
yield tasks_tx.blockOnTaskForward('localhost',
request.body['cluster_name'],
addInstancesTaskName, localTask)
defer.returnValue(request)
@defer.inlineCallbacks
def handleStartCluster(request):
ret = yield request.state.clusterLocks.run(
(request.body['cluster_name'], request.body['user_name']),
_handleStartCluster, request)
defer.returnValue(ret)
cluster = yield request.state.persistManager.loadCluster(request.body['cluster_name'],
request.body['user_name'])
cluster = cluster.setState(cluster.FAILED)
yield defer_utils.sleep(120)()
yield request.state.persistManager.removeCluster(request.body['cluster_name'],
request.body['user_name'])
raise err
if request.body['num_exec'] > 0 or request.body['num_data'] > 0:
addInstancesTaskName = yield clusters_client_www.addInstances('localhost',
request.body['cluster_name'],
request.body['user_name'],
request.body['num_exec'],
request.body['num_data'])
localTask = yield tasks_tx.loadTask(request.body['task_name'])
yield tasks_tx.blockOnTaskForward('localhost',
request.body['cluster_name'],
addInstancesTaskName,
localTask)
defer.returnValue(request)
@defer.inlineCallbacks
def handleStartCluster(request):
ret = yield request.state.clusterLocks.run((request.body['cluster_name'],
request.body['user_name']),
_handleStartCluster,
request)
defer.returnValue(ret)
def _realizeUrls(request):
localTag = yield request.state.tagPersist.loadTag(request.body['tag_name'])
# If we have urls we create a fake phantom tag
fakePhantom = {
'cluster.ALL.command':
'reliableDownloader.py -m 300 -t 20 -b ${base_dir} ' +
' '.join(localTag.metadata['urls'])
}
taskName = yield www_tags.realizePhantom(
'localhost', request.body['dst_cluster'], request.body['user_name'],
localTag.tagName, fakePhantom,
func.updateDict(localTag.metadata, {'urls_realized': True}))
localTask = yield tasks_tx.loadTask(request.body['task_name'])
endState, tsk = yield tasks_tx.blockOnTaskAndForward(
'localhost', request.body['dst_cluster'], taskName, localTask)
if endState == tasks_tx.task.TASK_FAILED:
yield tasks_tx.updateTask(
request.body['task_name'],
lambda t: t.setState(tasks_tx.task.TASK_FAILED))
raise RealizePhantomError(request.body['tag_name'])
if request.body['dst_cluster'] == 'local':
yield tag_mq_data.tagData(request.state,
request.body['tag_name'],
request.body['task_name'],
files=localTag.files,
action=tag_mq_data.ACTION_APPEND,
metadata={},
recursive=False,
expand=False,
compressDir=None)
else:
localTask = yield www_tags.tagData('localhost',
request.body['dst_cluster'],
request.body['user_name'],
action=tag_mq_data.ACTION_APPEND,
tagName=localTag.tagName,
files=localTag.files,
metadata={},
recursive=False,
expand=False,
compressDir=None)
localTask = yield tasks_tx.loadTask(request.body['task_name'])
endState, tsk = yield tasks_tx.blockOnTaskAndForward(
'localhost', request.body['dst_cluster'], taskName, localTask)
if endState == tasks_tx.task.TASK_FAILED:
yield tasks_tx.updateTask(
request.body['task_name'],
lambda t: t.setState(tasks_tx.task.TASK_FAILED))
raise RealizePhantomError(request.body['tag_name'])
# Load the tag up and return it so we can have the files it created
tag = yield www_tags.loadTag('localhost', request.body['dst_cluster'],
request.body['user_name'],
request.body['tag_name'])
defer.returnValue(tag)
def _handleTransferTag(request):
yield tasks_tx.updateTask(
request.body['task_name'],
lambda t: t.setState(tasks_tx.task.TASK_RUNNING).update(numTasks=2))
srcTag = yield www_tags.loadTag('localhost', request.body['src_cluster'],
request.body['user_name'],
request.body['tag_name'])
if not srcTag['phantom'] and (request.body['src_cluster'] != 'local'
or request.body['dst_cluster'] != 'local'):
if request.body['src_cluster'] == 'local':
tag = yield _uploadTag(request)
elif request.body['dst_cluster'] == 'local':
tag = yield _downloadTag(request)
else:
raise NoLocalClusterError(
'Source cluster or destination cluster must be local')
yield tasks_tx.updateTask(request.body['task_name'],
lambda t: t.progress())
if request.body.get('compress', False) or request.body.get(
'compress_dir', False):
defaultDir = '/mnt/output' if request.body[
'dst_cluster'] == 'local' else tag.metadata['tag_base_dir']
compressDir = request.body.get('compress_dir') if request.body.get(
'compress_dir', False) else defaultDir
else:
compressDir = None
if request.body['dst_cluster'] == 'local':
yield tag_mq_data.tagData(request.state,
request.body['tag_name'],
request.body['task_name'],
files=tag.files,
action=tag_mq_data.ACTION_OVERWRITE,
metadata=tag.metadata,
recursive=False,
expand=False,
compressDir=compressDir)
else:
newTag = yield www_tags.tagData(
'localhost',
request.body['dst_cluster'],
request.body['user_name'],
action=tag_mq_data.ACTION_OVERWRITE,
tagName=tag.tagName,
files=tag.files,
metadata=tag.metadata,
recursive=False,
expand=False,
compressDir=compressDir)
localTask = yield tasks_tx.loadTask(request.body['task_name'])
endState, tsk = yield tasks_tx.blockOnTaskAndForward(
'localhost', request.body['dst_cluster'], newTag['task_name'],
localTask)
if endState == tasks_tx.task.TASK_FAILED:
yield tasks_tx.updateTask(
request.body['task_name'],
lambda t: t.setState(tasks_tx.task.TASK_FAILED))
raise TransferTagError(request.body['tag_name'])
yield tasks_tx.updateTask(request.body['task_name'],
lambda t: t.progress())
elif not srcTag['phantom'] and srcTag['metadata'].get(
'urls', []) and not srcTag['metadata'].get('urls_realized', False):
# It's a local to local but we have urls and haven't realized them
yield _realizeUrls(request)
yield tasks_tx.updateTask(request.body['task_name'],
lambda t: t.progress(2))
elif srcTag['phantom']:
# Upload the depends file
srcClusters = yield www_clusters.listClusters(
'localhost', {'cluster_name': request.body['src_cluster']},
request.body['user_name'])
srcCluster = srcClusters[0]
dstClusters = yield www_clusters.listClusters(
'localhost', {'cluster_name': request.body['dst_cluster']},
request.body['user_name'])
dstCluster = dstClusters[0]
dependsOn = srcTag['phantom'].get('depends_on', '').split()
yield rsync.rsyncTo(dstCluster['master']['public_dns'],
'/',
'/',
dependsOn,
srcCluster['config']['rsync.options'],
srcCluster['config']['rsync.user'],
log=True)
taskName = yield www_tags.realizePhantom('localhost',
request.body['dst_cluster'],
request.body['user_name'],
srcTag['tag_name'],
srcTag['phantom'],
srcTag['metadata'])
localTask = yield tasks_tx.loadTask(request.body['task_name'])
endState, tsk = yield tasks_tx.blockOnTaskAndForward(
'localhost', request.body['dst_cluster'], taskName, localTask)
if endState == tasks_tx.task.TASK_FAILED:
yield tasks_tx.updateTask(
request.body['task_name'],
lambda t: t.setState(tasks_tx.task.TASK_FAILED))
raise RealizePhantomError(request.body['tag_name'])
yield tasks_tx.updateTask(request.body['task_name'],
lambda t: t.update(numTasks=1).progress())
else:
yield tag_mq_data.tagData(
request.state,
request.body['tag_name'],
request.body['task_name'],
files=[],
action=tag_mq_data.ACTION_APPEND,
metadata={},
recursive=False,
expand=False,
compressDir='/mnt/output'
if request.body.get('compress', False) else None)
yield tasks_tx.updateTask(request.body['task_name'],
lambda t: t.progress(2))
defer.returnValue(request)
