def _downloadTag(request):
remoteTag = yield www_tags.loadTag('localhost',
request.body['src_cluster'],
request.body['user_name'],
request.body['tag_name'])
srcClusters = yield www_clusters.listClusters(
'localhost', {'cluster_name': request.body['src_cluster']},
request.body['user_name'])
srcCluster = srcClusters[0]
dstClusters = yield www_clusters.listClusters(
'localhost', {'cluster_name': request.body['dst_cluster']},
request.body['user_name'])
dstCluster = dstClusters[0]
dstTagPath = os.path.join(dstCluster['config']['dirs.upload_dir'],
remoteTag['tag_name'])
baseDirFiles, nonBaseDirFiles = _partitionFiles(
remoteTag['files'], remoteTag['metadata']['tag_base_dir'])
if baseDirFiles:
yield rsync.rsyncFrom(srcCluster['master']['public_dns'],
remoteTag['metadata']['tag_base_dir'],
dstTagPath,
baseDirFiles,
dstCluster['config']['rsync.options'],
dstCluster['config']['rsync.user'],
log=True)
if nonBaseDirFiles:
yield rsync.rsyncFrom(srcCluster['master']['public_dns'],
'/',
dstTagPath,
nonBaseDirFiles,
dstCluster['config']['rsync.options'],
dstCluster['config']['rsync.user'],
log=True)
remoteFiles = ([os.path.join(dstTagPath, f) for f in baseDirFiles] + [
os.path.join(dstTagPath, _makePathRelative(f)) for f in nonBaseDirFiles
])
defer.returnValue(
persist.Tag(tagName=remoteTag['tag_name'],
files=remoteFiles,
metadata=func.updateDict(remoteTag['metadata'],
{'tag_base_dir': dstTagPath}),
phantom=remoteTag['phantom'],
taskName=None))
def _downloadTag(request):
remoteTag = yield www_tags.loadTag('localhost',
request.body['src_cluster'],
request.body['user_name'],
request.body['tag_name'])
srcClusters = yield www_clusters.listClusters('localhost',
{'cluster_name': request.body['src_cluster']},
request.body['user_name'])
srcCluster = srcClusters[0]
dstClusters = yield www_clusters.listClusters('localhost',
{'cluster_name': request.body['dst_cluster']},
request.body['user_name'])
dstCluster = dstClusters[0]
dstTagPath = os.path.join(dstCluster['config']['dirs.upload_dir'], remoteTag['tag_name'])
baseDirFiles, nonBaseDirFiles = _partitionFiles(remoteTag['files'], remoteTag['metadata']['tag_base_dir'])
if baseDirFiles:
yield rsync.rsyncFrom(srcCluster['master']['public_dns'],
remoteTag['metadata']['tag_base_dir'],
dstTagPath,
baseDirFiles,
dstCluster['config']['rsync.options'],
dstCluster['config']['rsync.user'],
log=True)
if nonBaseDirFiles:
yield rsync.rsyncFrom(srcCluster['master']['public_dns'],
'/',
dstTagPath,
nonBaseDirFiles,
dstCluster['config']['rsync.options'],
dstCluster['config']['rsync.user'],
log=True)
remoteFiles = ([os.path.join(dstTagPath, f) for f in baseDirFiles] +
[os.path.join(dstTagPath, _makePathRelative(f)) for f in nonBaseDirFiles])
defer.returnValue(persist.Tag(tagName=remoteTag['tag_name'],
files=remoteFiles,
metadata=func.updateDict(remoteTag['metadata'],
{'tag_base_dir': dstTagPath}),
phantom=remoteTag['phantom'],
taskName=None))
def _parseInstances(importedClusters):
"""Parses out all instances associated with this cluster."""
instances = []
for cluster in importedClusters:
master = cluster.get('master')
if master and master.get('state') == 'running':
config = json.loads(cluster.get('config'))
srcCluster = config.get('general.src_cluster')
clusterKey = config.get('cluster.cluster_public_key')
authToken = auth_token.generateToken(clusterKey)
remoteClusters = yield clusters_www.listClusters(master.get('public_dns'),
{'cluster_name': srcCluster},
cluster.get('user_name'),
authToken)
remoteCluster = _formatLocalHostname(remoteClusters[0])
instances.extend([instanceFromDict(x) for x
in [remoteCluster.get('master'), remoteCluster.get('exec_nodes')]
if x and x.get('state') == 'running'])
defer.returnValue(instances)
def _sumChildrenPipelines(self, pl):
numSteps = 0
completedSteps = 0
messages = []
clusters = yield clusters_client.listClusters('localhost',
{},
pl.userName)
clusterNames = set([c['cluster_name']
for c in clusters])
for cl, remotePipelineName in pl.children:
try:
if cl in clusterNames:
remotePipelines = yield pipelines_www.pipelineList('localhost',
cl,
pl.userName,
remotePipelineName,
True)
remotePipeline = remotePipelines[0]
remoteTask = yield tasks_www.loadTask('localhost',
cl,
pl.userName,
remotePipeline['task_name'])
numSteps += remoteTask['numTasks']
completedSteps += remoteTask['completedTasks']
messages += remoteTask['messages']
except Exception, err:
log.err(err)
def _delayAutoshutdown(state, batchState):
_log(batchState, 'AUTOSHUTDOWN: Trying to touch autoshutdown file')
try:
clusters = yield clusters_client.listClusters('localhost',
{'cluster_name':
batchState['pipeline_config']['cluster.CLUSTER_NAME']},
'guest')
cluster = clusters[0]
if batchState.get('state', None) == COMPLETED_STATE:
_log(batchState, 'AUTOSHUTDOWN: Pipeline complete, done')
if batchState.get('state', None) != RUNNING_STATE:
_log(batchState, 'AUTOSHUTDOWN: Pipeline not running, calling later')
reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState)
elif cluster['state'] == 'running':
# We need the SSH options from the machine.conf, ugh I hate these OOB dependencies
conf = config.configFromStream(open('/tmp/machine.conf'))
_log(batchState, 'AUTOSHUTDOWN: Touching delayautoshutdown')
yield ssh.runProcessSSH(cluster['master']['public_dns'],
'touch /var/vappio/runtime/delayautoshutdown',
stdoutf=None,
stderrf=None,
sshUser=conf('ssh.user'),
sshFlags=conf('ssh.options'),
log=True)
_log(batchState, 'AUTOSHUTDOWN: Setting up next call')
reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState)
else:
_log(batchState, 'AUTOSHUTDOWN: Cluster not running, calling later')
reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState)
except:
_log(batchState, 'AUTOSHUTDOWN: Cluster does not exist, calling later')
reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState)
def _sumChildrenPipelines(self, pl):
numSteps = 0
completedSteps = 0
messages = []
clusters = yield clusters_client.listClusters('localhost', {},
pl.userName)
clusterNames = set([c['cluster_name'] for c in clusters])
for cl, remotePipelineName in pl.children:
try:
if cl in clusterNames:
remotePipelines = yield pipelines_www.pipelineList(
'localhost', cl, pl.userName, remotePipelineName, True)
remotePipeline = remotePipelines[0]
remoteTask = yield tasks_www.loadTask(
'localhost', cl, pl.userName,
remotePipeline['task_name'])
numSteps += remoteTask['numTasks']
completedSteps += remoteTask['completedTasks']
messages += remoteTask['messages']
except Exception, err:
log.err(err)
def _waitForClusterInfo(instance):
try:
yield clusters_client_www.listClusters(instance['public_dns'], {},
userName)
except:
defer.returnValue(False)
defer.returnValue(True)
def _waitForClusterInfo(instance):
try:
yield clusters_client_www.listClusters(instance['public_dns'],
{},
userName)
except:
defer.returnValue(False)
defer.returnValue(True)
def _waitForPopulatedMasterNode(instance):
retVal = False
try:
if instance.get('master'):
retVal = True
else:
cluster = yield clusters_client_www.listClusters(
instance.get('public_dns'), {'cluster_name': srcCluster},
instance.get('user_name'), authToken)
retVal = cluster.get('master') != None
except:
defer.returnValue(retVal)
defer.returnValue(retVal)
def _waitForPopulatedMasterNode(instance):
retVal = False
try:
if instance.get('master'):
retVal = True
else:
cluster = yield clusters_client_www.listClusters(instance.get('public_dns'),
{'cluster_name': srcCluster},
instance.get('user_name'),
authToken)
retVal = cluster.get('master') != None
except:
defer.returnValue(retVal)
defer.returnValue(retVal)
def handleWWWDeleteCredential(request):
clusters = yield clusters_client.listClusters('localhost',
{},
'guest')
for cluster in clusters:
if cluster['cred_name'] == request.body['credential_name']:
raise CredentialInUseError()
if not request.body.get('dry_run', False):
yield request.state.credentialPersist.deleteCredential(request.body['credential_name'])
queue.returnQueueSuccess(request.mq,
request.body['return_queue'],
True)
defer.returnValue(request)
def loadRemoteCluster(state, cl):
"""
Tries to load a cluster. Returns the cluster on success
otherwise throws an error.
If the cluster is not actually owned by us throws
auth_token.AuthTokenError
If it's unresponsive throws
errors.RemoteError
We also check for SSH being up and throw a RemoteError
if it is not responsive
"""
if cl.master:
authToken = auth_token.generateToken(
cl.config('cluster.cluster_public_key'))
try:
clusters = yield clusters_www_client.listClusters(
cl.master['public_dns'], {'cluster_name': 'local'},
None,
authToken,
timeout=10,
tries=3)
cluster = clusters[0]
yield ssh.runProcessSSH(cl.master['public_dns'],
'echo hello',
stdoutf=None,
stderrf=None,
sshUser=state.machineConf('ssh.user'),
sshFlags=state.machineConf('ssh.options'))
defer.returnValue(cluster)
except errors.RemoteError, err:
if err.name == 'igs.utils.auth_token.AuthTokenError':
raise auth_token.AuthTokenError()
else:
raise
except commands.ProgramRunError:
raise errors.RemoteError('SSH failed')
def loadRemoteCluster(state, cl):
"""
Tries to load a cluster. Returns the cluster on success
otherwise throws an error.
If the cluster is not actually owned by us throws
auth_token.AuthTokenError
If it's unresponsive throws
errors.RemoteError
We also check for SSH being up and throw a RemoteError
if it is not responsive
"""
if cl.master:
authToken = auth_token.generateToken(cl.config('cluster.cluster_public_key'))
try:
clusters = yield clusters_www_client.listClusters(cl.master['public_dns'],
{'cluster_name': 'local'},
None,
authToken,
timeout=10,
tries=3)
cluster = clusters[0]
yield ssh.runProcessSSH(cl.master['public_dns'],
'echo hello',
stdoutf=None,
stderrf=None,
sshUser=state.machineConf('ssh.user'),
sshFlags=state.machineConf('ssh.options'))
defer.returnValue(cluster)
except errors.RemoteError, err:
if err.name == 'igs.utils.auth_token.AuthTokenError':
raise auth_token.AuthTokenError()
else:
raise
except commands.ProgramRunError:
raise errors.RemoteError('SSH failed')
def _delayAutoshutdown(state, batchState):
_log(batchState, 'AUTOSHUTDOWN: Trying to touch autoshutdown file')
try:
clusters = yield clusters_client.listClusters('localhost', {
'cluster_name':
batchState['pipeline_config']['cluster.CLUSTER_NAME']
}, 'guest')
cluster = clusters[0]
if batchState.get('state', None) == COMPLETED_STATE:
_log(batchState, 'AUTOSHUTDOWN: Pipeline complete, done')
if batchState.get('state', None) != RUNNING_STATE:
_log(batchState,
'AUTOSHUTDOWN: Pipeline not running, calling later')
reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state,
batchState)
elif cluster['state'] == 'running':
# We need the SSH options from the machine.conf, ugh I hate these OOB dependencies
conf = config.configFromStream(open('/tmp/machine.conf'))
_log(batchState, 'AUTOSHUTDOWN: Touching delayautoshutdown')
yield ssh.runProcessSSH(
cluster['master']['public_dns'],
'touch /var/vappio/runtime/delayautoshutdown',
stdoutf=None,
stderrf=None,
sshUser=conf('ssh.user'),
sshFlags=conf('ssh.options'),
log=True)
_log(batchState, 'AUTOSHUTDOWN: Setting up next call')
reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state,
batchState)
else:
_log(batchState,
'AUTOSHUTDOWN: Cluster not running, calling later')
reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state,
batchState)
except:
_log(batchState, 'AUTOSHUTDOWN: Cluster does not exist, calling later')
reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state,
batchState)
def resume(state, pipeline):
# Restart childen first
for clusterName, child in pipeline.children:
if clusterName == 'local':
childPipeline = yield state.pipelinePersist.loadPipelineBy({'pipeline_name': child},
pipeline.userName)
yield resume(state, childPipeline)
else:
try:
clusters = yield clusters_www.listClusters('localhost',
{'cluster_name': clusterName},
'guest')
cluster = clusters[0]
yield pipeline_www.resumePipeline(cluster['master']['public_dns'],
'local',
pipeline.userName,
child)
except Exception, err:
log.err('Error trying to resume child')
log.err(err)
def handleRealizePhantom(request):
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.setState(tasks_tx.task.TASK_RUNNING).update(numTasks=1))
localClusters = yield www_clusters.listClusters('localhost',
{'cluster_name': 'local'},
request.body['user_name'])
localCluster = localClusters[0]
ctype = localCluster['config']['general.ctype']
dstTagPath = os.path.join(localCluster['config']['dirs.upload_dir'],
request.body['tag_name'])
metadata = func.updateDict(request.body['metadata'], {'tag_base_dir': dstTagPath})
yield commands.runProcess(['mkdir', '-p', dstTagPath])
try:
yield _realizePhantom(ctype, dstTagPath, request.body['phantom'])
except RunCommandError, err:
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.addMessage(tasks_tx.task.MSG_ERROR, str(err)))
raise err
def _uploadTag(request):
localTag = yield request.state.tagPersist.loadTag(request.body['tag_name'])
srcClusters = yield www_clusters.listClusters('localhost',
{'cluster_name': request.body['src_cluster']},
request.body['user_name'])
srcCluster = srcClusters[0]
dstClusters = yield www_clusters.listClusters('localhost',
{'cluster_name': request.body['dst_cluster']},
request.body['user_name'])
dstCluster = dstClusters[0]
# We want the trailing '/' so everyone knows it's a directory
dstTagPath = os.path.join(dstCluster['config']['dirs.upload_dir'], localTag.tagName) + '/'
# Change dstTagPath to use /mnt/nostaging instead of mnt/staging as the destination if the
# tag has a nostaging metadata value
#if localTag.metadata['nostaging']:
# dstTagPath = os.path.join(dstCluster['config']['dirs.nostaging_upload_dir'], localTag.tagName) + '/'
if request.body['dst_type'] == 'local' :
upload_dir = '/mnt/nostaging/'
if 'dirs.nostaging_upload_dir' in dstCluster['config']:
upload_dir = dstCluster['config']['dirs.nostaging_upload_dir']
yield _makeDirsOnCluster(dstCluster, [upload_dir])
dstTagPath = os.path.join(upload_dir, localTag.tagName) + '/'
baseDirFiles, nonBaseDirFiles = _partitionFiles(localTag.files, localTag.metadata['tag_base_dir'])
if baseDirFiles:
yield rsync.rsyncTo(dstCluster['master']['public_dns'],
localTag.metadata['tag_base_dir'],
dstTagPath,
baseDirFiles,
srcCluster['config']['rsync.options'],
srcCluster['config']['rsync.user'],
log=True)
if nonBaseDirFiles:
yield rsync.rsyncTo(dstCluster['master']['public_dns'],
'/',
dstTagPath,
nonBaseDirFiles,
srcCluster['config']['rsync.options'],
srcCluster['config']['rsync.user'],
log=True)
remoteFiles = ([os.path.join(dstTagPath, f) for f in baseDirFiles] +
[os.path.join(dstTagPath, _makePathRelative(f)) for f in nonBaseDirFiles])
metadata = localTag.metadata
if metadata.get('urls', []) and not metadata.get('urls_realized', False):
tag = yield _realizeUrls(request)
remoteFiles.extend(tag['files'])
metadata = func.updateDict(metadata,
{'urls_realized': True})
# If we are dealing with an adhoc cluster here there is a chance that
# we are attempting to transfer to a local VM's shared folder which does
# support chown/chmod so we will skip this step
if dstCluster['master']['instance_type'] is not None:
yield ssh.runProcessSSH(dstCluster['master']['public_dns'],
'chown -R %s %s' % (dstCluster['config']['vappio.user'],
dstTagPath),
None,
log.err,
srcCluster['config']['ssh.user'],
srcCluster['config']['ssh.options'])
defer.returnValue(persist.Tag(tagName=localTag.tagName,
files=remoteFiles,
metadata=func.updateDict(metadata,
{'tag_base_dir': dstTagPath}),
phantom=localTag.phantom,
taskName=None))
def loadCluster(host, clusterName, userName):
clusters = yield clusters_client.listClusters(host,
{'cluster_name': clusterName},
userName)
cluster = clusters[0]
defer.returnValue(cluster)
def listClusters(criteria, userName, timeout=30, tries=4):
return clusters.listClusters('localhost',
criteria,
userName,
timeout=timeout,
tries=tries)
def _uploadTag(request):
localTag = yield request.state.tagPersist.loadTag(request.body['tag_name'])
srcClusters = yield www_clusters.listClusters(
'localhost', {'cluster_name': request.body['src_cluster']},
request.body['user_name'])
srcCluster = srcClusters[0]
dstClusters = yield www_clusters.listClusters(
'localhost', {'cluster_name': request.body['dst_cluster']},
request.body['user_name'])
dstCluster = dstClusters[0]
# We want the trailing '/' so everyone knows it's a directory
dstTagPath = os.path.join(dstCluster['config']['dirs.upload_dir'],
localTag.tagName) + '/'
# Change dstTagPath to use /mnt/nostaging instead of mnt/staging as the destination if the
# tag has a nostaging metadata value
#if localTag.metadata['nostaging']:
# dstTagPath = os.path.join(dstCluster['config']['dirs.nostaging_upload_dir'], localTag.tagName) + '/'
if request.body['dst_type'] == 'local':
upload_dir = '/mnt/nostaging/'
if 'dirs.nostaging_upload_dir' in dstCluster['config']:
upload_dir = dstCluster['config']['dirs.nostaging_upload_dir']
yield _makeDirsOnCluster(dstCluster, [upload_dir])
dstTagPath = os.path.join(upload_dir, localTag.tagName) + '/'
baseDirFiles, nonBaseDirFiles = _partitionFiles(
localTag.files, localTag.metadata['tag_base_dir'])
if baseDirFiles:
yield rsync.rsyncTo(dstCluster['master']['public_dns'],
localTag.metadata['tag_base_dir'],
dstTagPath,
baseDirFiles,
srcCluster['config']['rsync.options'],
srcCluster['config']['rsync.user'],
log=True)
if nonBaseDirFiles:
yield rsync.rsyncTo(dstCluster['master']['public_dns'],
'/',
dstTagPath,
nonBaseDirFiles,
srcCluster['config']['rsync.options'],
srcCluster['config']['rsync.user'],
log=True)
remoteFiles = ([os.path.join(dstTagPath, f) for f in baseDirFiles] + [
os.path.join(dstTagPath, _makePathRelative(f)) for f in nonBaseDirFiles
])
metadata = localTag.metadata
if metadata.get('urls', []) and not metadata.get('urls_realized', False):
tag = yield _realizeUrls(request)
remoteFiles.extend(tag['files'])
metadata = func.updateDict(metadata, {'urls_realized': True})
# If we are dealing with an adhoc cluster here there is a chance that
# we are attempting to transfer to a local VM's shared folder which does
# support chown/chmod so we will skip this step
if dstCluster['master']['instance_type'] is not None:
yield ssh.runProcessSSH(
dstCluster['master']['public_dns'], 'chown -R %s %s' %
(dstCluster['config']['vappio.user'], dstTagPath), None, log.err,
srcCluster['config']['ssh.user'],
srcCluster['config']['ssh.options'])
defer.returnValue(
persist.Tag(tagName=localTag.tagName,
files=remoteFiles,
metadata=func.updateDict(metadata,
{'tag_base_dir': dstTagPath}),
phantom=localTag.phantom,
taskName=None))
def _handleTransferTag(request):
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.setState(tasks_tx.task.TASK_RUNNING).update(numTasks=2))
srcTag = yield www_tags.loadTag('localhost',
request.body['src_cluster'],
request.body['user_name'],
request.body['tag_name'])
if not srcTag['phantom'] and (request.body['src_cluster'] != 'local' or request.body['dst_cluster'] != 'local'):
if request.body['src_cluster'] == 'local':
tag = yield _uploadTag(request)
elif request.body['dst_cluster'] == 'local':
tag = yield _downloadTag(request)
else:
raise NoLocalClusterError('Source cluster or destination cluster must be local')
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.progress())
if request.body.get('compress', False) or request.body.get('compress_dir', False):
defaultDir = '/mnt/output' if request.body['dst_cluster'] == 'local' else tag.metadata['tag_base_dir']
compressDir = request.body.get('compress_dir') if request.body.get('compress_dir', False) else defaultDir
else:
compressDir = None
if request.body['dst_cluster'] == 'local':
yield tag_mq_data.tagData(request.state,
request.body['tag_name'],
request.body['task_name'],
files=tag.files,
action=tag_mq_data.ACTION_OVERWRITE,
metadata=tag.metadata,
recursive=False,
expand=False,
compressDir=compressDir)
else:
newTag = yield www_tags.tagData('localhost',
request.body['dst_cluster'],
request.body['user_name'],
action=tag_mq_data.ACTION_OVERWRITE,
tagName=tag.tagName,
files=tag.files,
metadata=tag.metadata,
recursive=False,
expand=False,
compressDir=compressDir)
localTask = yield tasks_tx.loadTask(request.body['task_name'])
endState, tsk = yield tasks_tx.blockOnTaskAndForward('localhost',
request.body['dst_cluster'],
newTag['task_name'],
localTask)
if endState == tasks_tx.task.TASK_FAILED:
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.setState(tasks_tx.task.TASK_FAILED))
raise TransferTagError(request.body['tag_name'])
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.progress())
elif not srcTag['phantom'] and srcTag['metadata'].get('urls', []) and not srcTag['metadata'].get('urls_realized', False):
# It's a local to local but we have urls and haven't realized them
yield _realizeUrls(request)
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.progress(2))
elif srcTag['phantom']:
# Upload the depends file
srcClusters = yield www_clusters.listClusters('localhost',
{'cluster_name': request.body['src_cluster']},
request.body['user_name'])
srcCluster = srcClusters[0]
dstClusters = yield www_clusters.listClusters('localhost',
{'cluster_name': request.body['dst_cluster']},
request.body['user_name'])
dstCluster = dstClusters[0]
dependsOn = srcTag['phantom'].get('depends_on', '').split()
yield rsync.rsyncTo(dstCluster['master']['public_dns'],
'/',
'/',
dependsOn,
srcCluster['config']['rsync.options'],
srcCluster['config']['rsync.user'],
log=True)
taskName = yield www_tags.realizePhantom('localhost',
request.body['dst_cluster'],
request.body['user_name'],
srcTag['tag_name'],
srcTag['phantom'],
srcTag['metadata'])
localTask = yield tasks_tx.loadTask(request.body['task_name'])
endState, tsk = yield tasks_tx.blockOnTaskAndForward('localhost',
request.body['dst_cluster'],
taskName,
localTask)
if endState == tasks_tx.task.TASK_FAILED:
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.setState(tasks_tx.task.TASK_FAILED))
raise RealizePhantomError(request.body['tag_name'])
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.update(numTasks=1).progress())
else:
yield tag_mq_data.tagData(request.state,
request.body['tag_name'],
request.body['task_name'],
files=[],
action=tag_mq_data.ACTION_APPEND,
metadata={},
recursive=False,
expand=False,
compressDir='/mnt/output' if request.body.get('compress', False) else None)
yield tasks_tx.updateTask(request.body['task_name'],
lambda t : t.progress(2))
defer.returnValue(request)
def loadCluster(host, clusterName, userName):
clusters = yield clusters_client.listClusters(
host, {'cluster_name': clusterName}, userName)
cluster = clusters[0]
defer.returnValue(cluster)
def listClusters(criteria, userName, timeout=30, tries=4):
return clusters.listClusters("localhost", criteria, userName, timeout=timeout, tries=tries)
def importCluster(state, credClient, taskName, remoteHost, srcCluster,
cluster):
"""Handles retrieving metadata from the remote host and running through
a battery of tests to ensure that the VM being imported is in a running
state and reachable.
"""
@defer.inlineCallbacks
def _saveCluster(instances):
instances = yield credClient.updateInstances(instances)
cl = yield state.persistManager.loadCluster(cluster.clusterName,
cluster.userName)
cl = cl.setMaster(instances[0])
yield state.persistManager.saveCluster(cl)
defer.returnValue(func.Record(succeeded=instances, failed=[]))
authToken = auth_token.generateToken(
cluster.config('cluster.cluster_public_key'))
remoteClusters = yield clusters_client_www.listClusters(
remoteHost, {'cluster_name': srcCluster}, cluster.userName, authToken)
remoteCluster = remoteClusters[0]
if remoteCluster.get('state') in ['terminated', 'failed']:
raise Error('Imported cluster in TERMINATED or FAILED state')
# If we are importing a local cluster the public and private DNS will
# not be valid hostnames that we can query. Need to set them to the
# remote host provided in the import-clusters call
if 'clovr-' in remoteCluster['master']['public_dns']:
remoteCluster['master']['public_dns'] = remoteHost
remoteCluster['master']['private_dns'] = remoteHost
# Sorta hacky but we need to check whether or not a master node is
# associated with the cluster being imported before proceeding
_instances = yield waitForInstances([remoteCluster], [
updateTask(taskName, 'Waiting for populated master node'),
waitForPopulatedMasterNode(srcCluster, authToken, WAIT_FOR_STATE_TRIES)
])
if not _instances.succeeded:
raise Error('Could not retrieve master node from imported cluster.')
baseConf = config.configFromMap(cluster.config.conf)
remoteClusterConf = config.configFromMap(
{'general.src_cluster': srcCluster}, base=baseConf)
cl = cluster.update(config=remoteClusterConf)
cl = cl.setMaster(remoteCluster.get('master'))
yield state.persistManager.saveCluster(cl)
log.msg('DEBUG importCluster: About to run tests on master node')
_instances = yield waitForInstances([remoteCluster.get('master')], [
updateTask(taskName, 'Waiting for master'),
waitForState(credClient, 'running', WAIT_FOR_STATE_TRIES),
_saveCluster,
waitForSSH(cluster.config('ssh.user'), cluster.config('ssh.options'),
WAIT_FOR_SSH_TRIES), _saveCluster,
updateTask(taskName, 'SSH up'),
updateTask(taskName, 'Master in running state')
])
if not _instances.succeeded:
raise Error('Failed to import cluster')
# TODO: Maybe implement another set of checks here on our exec nodes.
if remoteCluster.get('exec_nodes'):
cl = cl.addExecNodes(remoteCluster.get('exec_nodes'))
yield state.persistManager.saveCluster(cl)
cl = yield state.persistManager.loadCluster(cl.clusterName, cl.userName)
cl = cl.setState(cl.RUNNING)
yield state.persistManager.saveCluster(cl)
defer.returnValue(cl)
def importCluster(state, credClient, taskName, remoteHost, srcCluster, cluster):
"""Handles retrieving metadata from the remote host and running through
a battery of tests to ensure that the VM being imported is in a running
state and reachable.
"""
@defer.inlineCallbacks
def _saveCluster(instances):
instances = yield credClient.updateInstances(instances)
cl = yield state.persistManager.loadCluster(cluster.clusterName,
cluster.userName)
cl = cl.setMaster(instances[0])
yield state.persistManager.saveCluster(cl)
defer.returnValue(func.Record(succeeded=instances,
failed=[]))
authToken = auth_token.generateToken(cluster.config('cluster.cluster_public_key'))
remoteClusters = yield clusters_client_www.listClusters(remoteHost,
{'cluster_name': srcCluster},
cluster.userName,
authToken)
remoteCluster = remoteClusters[0]
if remoteCluster.get('state') in ['terminated', 'failed']:
raise Error('Imported cluster in TERMINATED or FAILED state')
# If we are importing a local cluster the public and private DNS will
# not be valid hostnames that we can query. Need to set them to the
# remote host provided in the import-clusters call
if 'clovr-' in remoteCluster['master']['public_dns']:
remoteCluster['master']['public_dns'] = remoteHost
remoteCluster['master']['private_dns'] = remoteHost
# Sorta hacky but we need to check whether or not a master node is
# associated with the cluster being imported before proceeding
_instances = yield waitForInstances([remoteCluster],
[updateTask(taskName,
'Waiting for populated master node'),
waitForPopulatedMasterNode(srcCluster,
authToken,
WAIT_FOR_STATE_TRIES)])
if not _instances.succeeded:
raise Error('Could not retrieve master node from imported cluster.')
baseConf = config.configFromMap(cluster.config.conf)
remoteClusterConf = config.configFromMap({'general.src_cluster': srcCluster},
base=baseConf)
cl = cluster.update(config=remoteClusterConf)
cl = cl.setMaster(remoteCluster.get('master'))
yield state.persistManager.saveCluster(cl)
log.msg('DEBUG importCluster: About to run tests on master node')
_instances = yield waitForInstances([remoteCluster.get('master')],
[updateTask(taskName,
'Waiting for master'),
waitForState(credClient,
'running',
WAIT_FOR_STATE_TRIES),
_saveCluster,
waitForSSH(cluster.config('ssh.user'),
cluster.config('ssh.options'),
WAIT_FOR_SSH_TRIES),
_saveCluster,
updateTask(taskName,
'SSH up'),
updateTask(taskName,
'Master in running state')])
if not _instances.succeeded:
raise Error('Failed to import cluster')
# TODO: Maybe implement another set of checks here on our exec nodes.
if remoteCluster.get('exec_nodes'):
cl = cl.addExecNodes(remoteCluster.get('exec_nodes'))
yield state.persistManager.saveCluster(cl)
cl = yield state.persistManager.loadCluster(cl.clusterName,
cl.userName)
cl = cl.setState(cl.RUNNING)
yield state.persistManager.saveCluster(cl)
defer.returnValue(cl)
def _handleTransferTag(request):
yield tasks_tx.updateTask(
request.body['task_name'],
lambda t: t.setState(tasks_tx.task.TASK_RUNNING).update(numTasks=2))
srcTag = yield www_tags.loadTag('localhost', request.body['src_cluster'],
request.body['user_name'],
request.body['tag_name'])
if not srcTag['phantom'] and (request.body['src_cluster'] != 'local'
or request.body['dst_cluster'] != 'local'):
if request.body['src_cluster'] == 'local':
tag = yield _uploadTag(request)
elif request.body['dst_cluster'] == 'local':
tag = yield _downloadTag(request)
else:
raise NoLocalClusterError(
'Source cluster or destination cluster must be local')
yield tasks_tx.updateTask(request.body['task_name'],
lambda t: t.progress())
if request.body.get('compress', False) or request.body.get(
'compress_dir', False):
defaultDir = '/mnt/output' if request.body[
'dst_cluster'] == 'local' else tag.metadata['tag_base_dir']
compressDir = request.body.get('compress_dir') if request.body.get(
'compress_dir', False) else defaultDir
else:
compressDir = None
if request.body['dst_cluster'] == 'local':
yield tag_mq_data.tagData(request.state,
request.body['tag_name'],
request.body['task_name'],
files=tag.files,
action=tag_mq_data.ACTION_OVERWRITE,
metadata=tag.metadata,
recursive=False,
expand=False,
compressDir=compressDir)
else:
newTag = yield www_tags.tagData(
'localhost',
request.body['dst_cluster'],
request.body['user_name'],
action=tag_mq_data.ACTION_OVERWRITE,
tagName=tag.tagName,
files=tag.files,
metadata=tag.metadata,
recursive=False,
expand=False,
compressDir=compressDir)
localTask = yield tasks_tx.loadTask(request.body['task_name'])
endState, tsk = yield tasks_tx.blockOnTaskAndForward(
'localhost', request.body['dst_cluster'], newTag['task_name'],
localTask)
if endState == tasks_tx.task.TASK_FAILED:
yield tasks_tx.updateTask(
request.body['task_name'],
lambda t: t.setState(tasks_tx.task.TASK_FAILED))
raise TransferTagError(request.body['tag_name'])
yield tasks_tx.updateTask(request.body['task_name'],
lambda t: t.progress())
elif not srcTag['phantom'] and srcTag['metadata'].get(
'urls', []) and not srcTag['metadata'].get('urls_realized', False):
# It's a local to local but we have urls and haven't realized them
yield _realizeUrls(request)
yield tasks_tx.updateTask(request.body['task_name'],
lambda t: t.progress(2))
elif srcTag['phantom']:
# Upload the depends file
srcClusters = yield www_clusters.listClusters(
'localhost', {'cluster_name': request.body['src_cluster']},
request.body['user_name'])
srcCluster = srcClusters[0]
dstClusters = yield www_clusters.listClusters(
'localhost', {'cluster_name': request.body['dst_cluster']},
request.body['user_name'])
dstCluster = dstClusters[0]
dependsOn = srcTag['phantom'].get('depends_on', '').split()
yield rsync.rsyncTo(dstCluster['master']['public_dns'],
'/',
'/',
dependsOn,
srcCluster['config']['rsync.options'],
srcCluster['config']['rsync.user'],
log=True)
taskName = yield www_tags.realizePhantom('localhost',
request.body['dst_cluster'],
request.body['user_name'],
srcTag['tag_name'],
srcTag['phantom'],
srcTag['metadata'])
localTask = yield tasks_tx.loadTask(request.body['task_name'])
endState, tsk = yield tasks_tx.blockOnTaskAndForward(
'localhost', request.body['dst_cluster'], taskName, localTask)
if endState == tasks_tx.task.TASK_FAILED:
yield tasks_tx.updateTask(
request.body['task_name'],
lambda t: t.setState(tasks_tx.task.TASK_FAILED))
raise RealizePhantomError(request.body['tag_name'])
yield tasks_tx.updateTask(request.body['task_name'],
lambda t: t.update(numTasks=1).progress())
else:
yield tag_mq_data.tagData(
request.state,
request.body['tag_name'],
request.body['task_name'],
files=[],
action=tag_mq_data.ACTION_APPEND,
metadata={},
recursive=False,
expand=False,
compressDir='/mnt/output'
if request.body.get('compress', False) else None)
yield tasks_tx.updateTask(request.body['task_name'],
lambda t: t.progress(2))
defer.returnValue(request)
