def _handleTerminateInstances(request):
yield tasks_tx.updateTask(request.body['task_name'],
lambda t: t.setState(tasks_tx.task.TASK_RUNNING))
persistManager = request.state.persistManager
cluster = yield persistManager.loadCluster(request.body['cluster_name'],
request.body['user_name'])
credClient = cred_client.CredentialClient(cluster.credName, request.mq,
request.state.conf)
if request.body['cluster_name'] != 'local':
try:
remoteTaskName = yield clusters_client_www.terminateInstances(
cluster.master['public_dns'], 'local',
request.body['user_name'], request.body['by_attribute'],
request.body['attribute_values'])
localTask = yield tasks_tx.loadTask(request.body['task_name'])
yield tasks_tx.blockOnTaskAndForward('localhost',
request.body['cluster_name'],
remoteTaskName, localTask)
except:
yield terminateInstancesByAttribute(
persistManager, credClient, request.body['cluster_name'],
request.body['user_name'], request.body['by_attribute'],
request.body['attribute_values'])
else:
cl = yield terminateInstancesByAttribute(
persistManager, credClient, 'local', None,
request.body['by_attribute'], request.body['attribute_values'])
yield tasks_tx.updateTask(request.body['task_name'],
lambda t: t.progress())
defer.returnValue(request)
def terminateRemoteCluster(request):
persistManager = request.state.persistManager
cluster = yield persistManager.loadCluster(request.body['cluster_name'],
request.body['user_name'])
authToken = auth_token.generateToken(
cluster.config('cluster.cluster_public_key'))
credClient = cred_client.CredentialClient(cluster.credName, request.mq,
request.state.conf)
try:
if cluster.master:
wwwTerminateCluster = clusters_client_www.terminateCluster
remoteTaskName = yield wwwTerminateCluster(
cluster.master['public_dns'], 'local', None, authToken)
localTask = yield tasks_tx.loadTask(request.body['task_name'])
yield tasks_tx.blockOnTaskAndForward('localhost',
request.body['cluster_name'],
remoteTaskName, localTask)
except errors.RemoteError, err:
# If the error is not an auth token one then kill it
# otherwise it means we think we own a cluster that
# we don't
#
# In this case another part of the system is in charge
# of forgetting about the clusters we shouldn't know
if err.name != 'igs.utils.auth_token.AuthTokenError':
log.err(err)
yield terminateCluster(credClient, persistManager,
request.body['cluster_name'],
request.body['user_name'])
else:
raise
def terminateRemoteCluster(request):
persistManager = request.state.persistManager
cluster = yield persistManager.loadCluster(request.body["cluster_name"], request.body["user_name"])
authToken = auth_token.generateToken(cluster.config("cluster.cluster_public_key"))
credClient = cred_client.CredentialClient(cluster.credName, request.mq, request.state.conf)
try:
if cluster.master:
wwwTerminateCluster = clusters_client_www.terminateCluster
remoteTaskName = yield wwwTerminateCluster(cluster.master["public_dns"], "local", None, authToken)
localTask = yield tasks_tx.loadTask(request.body["task_name"])
yield tasks_tx.blockOnTaskAndForward("localhost", request.body["cluster_name"], remoteTaskName, localTask)
except errors.RemoteError, err:
# If the error is not an auth token one then kill it
# otherwise it means we think we own a cluster that
# we don't
#
# In this case another part of the system is in charge
# of forgetting about the clusters we shouldn't know
if err.name != "igs.utils.auth_token.AuthTokenError":
log.err(err)
yield terminateCluster(credClient, persistManager, request.body["cluster_name"], request.body["user_name"])
else:
raise
def _handleTerminateInstances(request):
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.setState(tasks_tx.task.TASK_RUNNING))
persistManager = request.state.persistManager
cluster = yield persistManager.loadCluster(request.body['cluster_name'],
request.body['user_name'])
credClient = cred_client.CredentialClient(cluster.credName,
request.mq,
request.state.conf)
if request.body['cluster_name'] != 'local':
try:
remoteTaskName = yield clusters_client_www.terminateInstances(
cluster.master['public_dns'],
'local',
request.body['user_name'],
request.body['by_attribute'],
request.body['attribute_values'])
localTask = yield tasks_tx.loadTask(request.body['task_name'])
yield tasks_tx.blockOnTaskAndForward('localhost',
request.body['cluster_name'],
remoteTaskName,
localTask)
except:
yield terminateInstancesByAttribute(persistManager,
credClient,
request.body['cluster_name'],
request.body['user_name'],
request.body['by_attribute'],
request.body['attribute_values'])
else:
cl = yield terminateInstancesByAttribute(persistManager,
credClient,
'local',
None,
request.body['by_attribute'],
request.body['attribute_values'])
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.progress())
defer.returnValue(request)
def _realizeUrls(request):
localTag = yield request.state.tagPersist.loadTag(request.body['tag_name'])
# If we have urls we create a fake phantom tag
fakePhantom = {'cluster.ALL.command':
'reliableDownloader.py -m 300 -t 20 -b ${base_dir} ' + ' '.join(localTag.metadata['urls'])}
taskName = yield www_tags.realizePhantom('localhost',
request.body['dst_cluster'],
request.body['user_name'],
localTag.tagName,
fakePhantom,
func.updateDict(localTag.metadata, {'urls_realized': True}))
localTask = yield tasks_tx.loadTask(request.body['task_name'])
endState, tsk = yield tasks_tx.blockOnTaskAndForward('localhost',
request.body['dst_cluster'],
taskName,
localTask)
if endState == tasks_tx.task.TASK_FAILED:
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.setState(tasks_tx.task.TASK_FAILED))
raise RealizePhantomError(request.body['tag_name'])
if request.body['dst_cluster'] == 'local':
yield tag_mq_data.tagData(request.state,
request.body['tag_name'],
request.body['task_name'],
files=localTag.files,
action=tag_mq_data.ACTION_APPEND,
metadata={},
recursive=False,
expand=False,
compressDir=None)
else:
localTask = yield www_tags.tagData('localhost',
request.body['dst_cluster'],
request.body['user_name'],
action=tag_mq_data.ACTION_APPEND,
tagName=localTag.tagName,
files=localTag.files,
metadata={},
recursive=False,
expand=False,
compressDir=None)
localTask = yield tasks_tx.loadTask(request.body['task_name'])
endState, tsk = yield tasks_tx.blockOnTaskAndForward('localhost',
request.body['dst_cluster'],
taskName,
localTask)
if endState == tasks_tx.task.TASK_FAILED:
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.setState(tasks_tx.task.TASK_FAILED))
raise RealizePhantomError(request.body['tag_name'])
# Load the tag up and return it so we can have the files it created
tag = yield www_tags.loadTag('localhost',
request.body['dst_cluster'],
request.body['user_name'],
request.body['tag_name'])
defer.returnValue(tag)
def _handleTransferTag(request):
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.setState(tasks_tx.task.TASK_RUNNING).update(numTasks=2))
srcTag = yield www_tags.loadTag('localhost',
request.body['src_cluster'],
request.body['user_name'],
request.body['tag_name'])
if not srcTag['phantom'] and (request.body['src_cluster'] != 'local' or request.body['dst_cluster'] != 'local'):
if request.body['src_cluster'] == 'local':
tag = yield _uploadTag(request)
elif request.body['dst_cluster'] == 'local':
tag = yield _downloadTag(request)
else:
raise NoLocalClusterError('Source cluster or destination cluster must be local')
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.progress())
if request.body.get('compress', False) or request.body.get('compress_dir', False):
defaultDir = '/mnt/output' if request.body['dst_cluster'] == 'local' else tag.metadata['tag_base_dir']
compressDir = request.body.get('compress_dir') if request.body.get('compress_dir', False) else defaultDir
else:
compressDir = None
if request.body['dst_cluster'] == 'local':
yield tag_mq_data.tagData(request.state,
request.body['tag_name'],
request.body['task_name'],
files=tag.files,
action=tag_mq_data.ACTION_OVERWRITE,
metadata=tag.metadata,
recursive=False,
expand=False,
compressDir=compressDir)
else:
newTag = yield www_tags.tagData('localhost',
request.body['dst_cluster'],
request.body['user_name'],
action=tag_mq_data.ACTION_OVERWRITE,
tagName=tag.tagName,
files=tag.files,
metadata=tag.metadata,
recursive=False,
expand=False,
compressDir=compressDir)
localTask = yield tasks_tx.loadTask(request.body['task_name'])
endState, tsk = yield tasks_tx.blockOnTaskAndForward('localhost',
request.body['dst_cluster'],
newTag['task_name'],
localTask)
if endState == tasks_tx.task.TASK_FAILED:
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.setState(tasks_tx.task.TASK_FAILED))
raise TransferTagError(request.body['tag_name'])
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.progress())
elif not srcTag['phantom'] and srcTag['metadata'].get('urls', []) and not srcTag['metadata'].get('urls_realized', False):
# It's a local to local but we have urls and haven't realized them
yield _realizeUrls(request)
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.progress(2))
elif srcTag['phantom']:
# Upload the depends file
srcClusters = yield www_clusters.listClusters('localhost',
{'cluster_name': request.body['src_cluster']},
request.body['user_name'])
srcCluster = srcClusters[0]
dstClusters = yield www_clusters.listClusters('localhost',
{'cluster_name': request.body['dst_cluster']},
request.body['user_name'])
dstCluster = dstClusters[0]
dependsOn = srcTag['phantom'].get('depends_on', '').split()
yield rsync.rsyncTo(dstCluster['master']['public_dns'],
'/',
'/',
dependsOn,
srcCluster['config']['rsync.options'],
srcCluster['config']['rsync.user'],
log=True)
taskName = yield www_tags.realizePhantom('localhost',
request.body['dst_cluster'],
request.body['user_name'],
srcTag['tag_name'],
srcTag['phantom'],
srcTag['metadata'])
localTask = yield tasks_tx.loadTask(request.body['task_name'])
endState, tsk = yield tasks_tx.blockOnTaskAndForward('localhost',
request.body['dst_cluster'],
taskName,
localTask)
if endState == tasks_tx.task.TASK_FAILED:
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.setState(tasks_tx.task.TASK_FAILED))
raise RealizePhantomError(request.body['tag_name'])
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.update(numTasks=1).progress())
else:
yield tag_mq_data.tagData(request.state,
request.body['tag_name'],
request.body['task_name'],
files=[],
action=tag_mq_data.ACTION_APPEND,
metadata={},
recursive=False,
expand=False,
compressDir='/mnt/output' if request.body.get('compress', False) else None)
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.progress(2))
defer.returnValue(request)
def _realizeUrls(request):
localTag = yield request.state.tagPersist.loadTag(request.body['tag_name'])
# If we have urls we create a fake phantom tag
fakePhantom = {
'cluster.ALL.command':
'reliableDownloader.py -m 300 -t 20 -b ${base_dir} ' +
' '.join(localTag.metadata['urls'])
}
taskName = yield www_tags.realizePhantom(
'localhost', request.body['dst_cluster'], request.body['user_name'],
localTag.tagName, fakePhantom,
func.updateDict(localTag.metadata, {'urls_realized': True}))
localTask = yield tasks_tx.loadTask(request.body['task_name'])
endState, tsk = yield tasks_tx.blockOnTaskAndForward(
'localhost', request.body['dst_cluster'], taskName, localTask)
if endState == tasks_tx.task.TASK_FAILED:
yield tasks_tx.updateTask(
request.body['task_name'],
lambda t: t.setState(tasks_tx.task.TASK_FAILED))
raise RealizePhantomError(request.body['tag_name'])
if request.body['dst_cluster'] == 'local':
yield tag_mq_data.tagData(request.state,
request.body['tag_name'],
request.body['task_name'],
files=localTag.files,
action=tag_mq_data.ACTION_APPEND,
metadata={},
recursive=False,
expand=False,
compressDir=None)
else:
localTask = yield www_tags.tagData('localhost',
request.body['dst_cluster'],
request.body['user_name'],
action=tag_mq_data.ACTION_APPEND,
tagName=localTag.tagName,
files=localTag.files,
metadata={},
recursive=False,
expand=False,
compressDir=None)
localTask = yield tasks_tx.loadTask(request.body['task_name'])
endState, tsk = yield tasks_tx.blockOnTaskAndForward(
'localhost', request.body['dst_cluster'], taskName, localTask)
if endState == tasks_tx.task.TASK_FAILED:
yield tasks_tx.updateTask(
request.body['task_name'],
lambda t: t.setState(tasks_tx.task.TASK_FAILED))
raise RealizePhantomError(request.body['tag_name'])
# Load the tag up and return it so we can have the files it created
tag = yield www_tags.loadTag('localhost', request.body['dst_cluster'],
request.body['user_name'],
request.body['tag_name'])
defer.returnValue(tag)
def _handleTransferTag(request):
yield tasks_tx.updateTask(
request.body['task_name'],
lambda t: t.setState(tasks_tx.task.TASK_RUNNING).update(numTasks=2))
srcTag = yield www_tags.loadTag('localhost', request.body['src_cluster'],
request.body['user_name'],
request.body['tag_name'])
if not srcTag['phantom'] and (request.body['src_cluster'] != 'local'
or request.body['dst_cluster'] != 'local'):
if request.body['src_cluster'] == 'local':
tag = yield _uploadTag(request)
elif request.body['dst_cluster'] == 'local':
tag = yield _downloadTag(request)
else:
raise NoLocalClusterError(
'Source cluster or destination cluster must be local')
yield tasks_tx.updateTask(request.body['task_name'],
lambda t: t.progress())
if request.body.get('compress', False) or request.body.get(
'compress_dir', False):
defaultDir = '/mnt/output' if request.body[
'dst_cluster'] == 'local' else tag.metadata['tag_base_dir']
compressDir = request.body.get('compress_dir') if request.body.get(
'compress_dir', False) else defaultDir
else:
compressDir = None
if request.body['dst_cluster'] == 'local':
yield tag_mq_data.tagData(request.state,
request.body['tag_name'],
request.body['task_name'],
files=tag.files,
action=tag_mq_data.ACTION_OVERWRITE,
metadata=tag.metadata,
recursive=False,
expand=False,
compressDir=compressDir)
else:
newTag = yield www_tags.tagData(
'localhost',
request.body['dst_cluster'],
request.body['user_name'],
action=tag_mq_data.ACTION_OVERWRITE,
tagName=tag.tagName,
files=tag.files,
metadata=tag.metadata,
recursive=False,
expand=False,
compressDir=compressDir)
localTask = yield tasks_tx.loadTask(request.body['task_name'])
endState, tsk = yield tasks_tx.blockOnTaskAndForward(
'localhost', request.body['dst_cluster'], newTag['task_name'],
localTask)
if endState == tasks_tx.task.TASK_FAILED:
yield tasks_tx.updateTask(
request.body['task_name'],
lambda t: t.setState(tasks_tx.task.TASK_FAILED))
raise TransferTagError(request.body['tag_name'])
yield tasks_tx.updateTask(request.body['task_name'],
lambda t: t.progress())
elif not srcTag['phantom'] and srcTag['metadata'].get(
'urls', []) and not srcTag['metadata'].get('urls_realized', False):
# It's a local to local but we have urls and haven't realized them
yield _realizeUrls(request)
yield tasks_tx.updateTask(request.body['task_name'],
lambda t: t.progress(2))
elif srcTag['phantom']:
# Upload the depends file
srcClusters = yield www_clusters.listClusters(
'localhost', {'cluster_name': request.body['src_cluster']},
request.body['user_name'])
srcCluster = srcClusters[0]
dstClusters = yield www_clusters.listClusters(
'localhost', {'cluster_name': request.body['dst_cluster']},
request.body['user_name'])
dstCluster = dstClusters[0]
dependsOn = srcTag['phantom'].get('depends_on', '').split()
yield rsync.rsyncTo(dstCluster['master']['public_dns'],
'/',
'/',
dependsOn,
srcCluster['config']['rsync.options'],
srcCluster['config']['rsync.user'],
log=True)
taskName = yield www_tags.realizePhantom('localhost',
request.body['dst_cluster'],
request.body['user_name'],
srcTag['tag_name'],
srcTag['phantom'],
srcTag['metadata'])
localTask = yield tasks_tx.loadTask(request.body['task_name'])
endState, tsk = yield tasks_tx.blockOnTaskAndForward(
'localhost', request.body['dst_cluster'], taskName, localTask)
if endState == tasks_tx.task.TASK_FAILED:
yield tasks_tx.updateTask(
request.body['task_name'],
lambda t: t.setState(tasks_tx.task.TASK_FAILED))
raise RealizePhantomError(request.body['tag_name'])
yield tasks_tx.updateTask(request.body['task_name'],
lambda t: t.update(numTasks=1).progress())
else:
yield tag_mq_data.tagData(
request.state,
request.body['tag_name'],
request.body['task_name'],
files=[],
action=tag_mq_data.ACTION_APPEND,
metadata={},
recursive=False,
expand=False,
compressDir='/mnt/output'
if request.body.get('compress', False) else None)
yield tasks_tx.updateTask(request.body['task_name'],
lambda t: t.progress(2))
defer.returnValue(request)