def _downloadTag(request): remoteTag = yield www_tags.loadTag('localhost', request.body['src_cluster'], request.body['user_name'], request.body['tag_name']) srcClusters = yield www_clusters.listClusters( 'localhost', {'cluster_name': request.body['src_cluster']}, request.body['user_name']) srcCluster = srcClusters[0] dstClusters = yield www_clusters.listClusters( 'localhost', {'cluster_name': request.body['dst_cluster']}, request.body['user_name']) dstCluster = dstClusters[0] dstTagPath = os.path.join(dstCluster['config']['dirs.upload_dir'], remoteTag['tag_name']) baseDirFiles, nonBaseDirFiles = _partitionFiles( remoteTag['files'], remoteTag['metadata']['tag_base_dir']) if baseDirFiles: yield rsync.rsyncFrom(srcCluster['master']['public_dns'], remoteTag['metadata']['tag_base_dir'], dstTagPath, baseDirFiles, dstCluster['config']['rsync.options'], dstCluster['config']['rsync.user'], log=True) if nonBaseDirFiles: yield rsync.rsyncFrom(srcCluster['master']['public_dns'], '/', dstTagPath, nonBaseDirFiles, dstCluster['config']['rsync.options'], dstCluster['config']['rsync.user'], log=True) remoteFiles = ([os.path.join(dstTagPath, f) for f in baseDirFiles] + [ os.path.join(dstTagPath, _makePathRelative(f)) for f in nonBaseDirFiles ]) defer.returnValue( persist.Tag(tagName=remoteTag['tag_name'], files=remoteFiles, metadata=func.updateDict(remoteTag['metadata'], {'tag_base_dir': dstTagPath}), phantom=remoteTag['phantom'], taskName=None))
def _downloadTag(request): remoteTag = yield www_tags.loadTag('localhost', request.body['src_cluster'], request.body['user_name'], request.body['tag_name']) srcClusters = yield www_clusters.listClusters('localhost', {'cluster_name': request.body['src_cluster']}, request.body['user_name']) srcCluster = srcClusters[0] dstClusters = yield www_clusters.listClusters('localhost', {'cluster_name': request.body['dst_cluster']}, request.body['user_name']) dstCluster = dstClusters[0] dstTagPath = os.path.join(dstCluster['config']['dirs.upload_dir'], remoteTag['tag_name']) baseDirFiles, nonBaseDirFiles = _partitionFiles(remoteTag['files'], remoteTag['metadata']['tag_base_dir']) if baseDirFiles: yield rsync.rsyncFrom(srcCluster['master']['public_dns'], remoteTag['metadata']['tag_base_dir'], dstTagPath, baseDirFiles, dstCluster['config']['rsync.options'], dstCluster['config']['rsync.user'], log=True) if nonBaseDirFiles: yield rsync.rsyncFrom(srcCluster['master']['public_dns'], '/', dstTagPath, nonBaseDirFiles, dstCluster['config']['rsync.options'], dstCluster['config']['rsync.user'], log=True) remoteFiles = ([os.path.join(dstTagPath, f) for f in baseDirFiles] + [os.path.join(dstTagPath, _makePathRelative(f)) for f in nonBaseDirFiles]) defer.returnValue(persist.Tag(tagName=remoteTag['tag_name'], files=remoteFiles, metadata=func.updateDict(remoteTag['metadata'], {'tag_base_dir': dstTagPath}), phantom=remoteTag['phantom'], taskName=None))
def _parseInstances(importedClusters): """Parses out all instances associated with this cluster.""" instances = [] for cluster in importedClusters: master = cluster.get('master') if master and master.get('state') == 'running': config = json.loads(cluster.get('config')) srcCluster = config.get('general.src_cluster') clusterKey = config.get('cluster.cluster_public_key') authToken = auth_token.generateToken(clusterKey) remoteClusters = yield clusters_www.listClusters(master.get('public_dns'), {'cluster_name': srcCluster}, cluster.get('user_name'), authToken) remoteCluster = _formatLocalHostname(remoteClusters[0]) instances.extend([instanceFromDict(x) for x in [remoteCluster.get('master'), remoteCluster.get('exec_nodes')] if x and x.get('state') == 'running']) defer.returnValue(instances)
def _sumChildrenPipelines(self, pl): numSteps = 0 completedSteps = 0 messages = [] clusters = yield clusters_client.listClusters('localhost', {}, pl.userName) clusterNames = set([c['cluster_name'] for c in clusters]) for cl, remotePipelineName in pl.children: try: if cl in clusterNames: remotePipelines = yield pipelines_www.pipelineList('localhost', cl, pl.userName, remotePipelineName, True) remotePipeline = remotePipelines[0] remoteTask = yield tasks_www.loadTask('localhost', cl, pl.userName, remotePipeline['task_name']) numSteps += remoteTask['numTasks'] completedSteps += remoteTask['completedTasks'] messages += remoteTask['messages'] except Exception, err: log.err(err)
def _delayAutoshutdown(state, batchState): _log(batchState, 'AUTOSHUTDOWN: Trying to touch autoshutdown file') try: clusters = yield clusters_client.listClusters('localhost', {'cluster_name': batchState['pipeline_config']['cluster.CLUSTER_NAME']}, 'guest') cluster = clusters[0] if batchState.get('state', None) == COMPLETED_STATE: _log(batchState, 'AUTOSHUTDOWN: Pipeline complete, done') if batchState.get('state', None) != RUNNING_STATE: _log(batchState, 'AUTOSHUTDOWN: Pipeline not running, calling later') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState) elif cluster['state'] == 'running': # We need the SSH options from the machine.conf, ugh I hate these OOB dependencies conf = config.configFromStream(open('/tmp/machine.conf')) _log(batchState, 'AUTOSHUTDOWN: Touching delayautoshutdown') yield ssh.runProcessSSH(cluster['master']['public_dns'], 'touch /var/vappio/runtime/delayautoshutdown', stdoutf=None, stderrf=None, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), log=True) _log(batchState, 'AUTOSHUTDOWN: Setting up next call') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState) else: _log(batchState, 'AUTOSHUTDOWN: Cluster not running, calling later') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState) except: _log(batchState, 'AUTOSHUTDOWN: Cluster does not exist, calling later') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState)
def _sumChildrenPipelines(self, pl): numSteps = 0 completedSteps = 0 messages = [] clusters = yield clusters_client.listClusters('localhost', {}, pl.userName) clusterNames = set([c['cluster_name'] for c in clusters]) for cl, remotePipelineName in pl.children: try: if cl in clusterNames: remotePipelines = yield pipelines_www.pipelineList( 'localhost', cl, pl.userName, remotePipelineName, True) remotePipeline = remotePipelines[0] remoteTask = yield tasks_www.loadTask( 'localhost', cl, pl.userName, remotePipeline['task_name']) numSteps += remoteTask['numTasks'] completedSteps += remoteTask['completedTasks'] messages += remoteTask['messages'] except Exception, err: log.err(err)
def _waitForClusterInfo(instance): try: yield clusters_client_www.listClusters(instance['public_dns'], {}, userName) except: defer.returnValue(False) defer.returnValue(True)
def _waitForPopulatedMasterNode(instance): retVal = False try: if instance.get('master'): retVal = True else: cluster = yield clusters_client_www.listClusters( instance.get('public_dns'), {'cluster_name': srcCluster}, instance.get('user_name'), authToken) retVal = cluster.get('master') != None except: defer.returnValue(retVal) defer.returnValue(retVal)
def _waitForPopulatedMasterNode(instance): retVal = False try: if instance.get('master'): retVal = True else: cluster = yield clusters_client_www.listClusters(instance.get('public_dns'), {'cluster_name': srcCluster}, instance.get('user_name'), authToken) retVal = cluster.get('master') != None except: defer.returnValue(retVal) defer.returnValue(retVal)
def handleWWWDeleteCredential(request): clusters = yield clusters_client.listClusters('localhost', {}, 'guest') for cluster in clusters: if cluster['cred_name'] == request.body['credential_name']: raise CredentialInUseError() if not request.body.get('dry_run', False): yield request.state.credentialPersist.deleteCredential(request.body['credential_name']) queue.returnQueueSuccess(request.mq, request.body['return_queue'], True) defer.returnValue(request)
def loadRemoteCluster(state, cl): """ Tries to load a cluster. Returns the cluster on success otherwise throws an error. If the cluster is not actually owned by us throws auth_token.AuthTokenError If it's unresponsive throws errors.RemoteError We also check for SSH being up and throw a RemoteError if it is not responsive """ if cl.master: authToken = auth_token.generateToken( cl.config('cluster.cluster_public_key')) try: clusters = yield clusters_www_client.listClusters( cl.master['public_dns'], {'cluster_name': 'local'}, None, authToken, timeout=10, tries=3) cluster = clusters[0] yield ssh.runProcessSSH(cl.master['public_dns'], 'echo hello', stdoutf=None, stderrf=None, sshUser=state.machineConf('ssh.user'), sshFlags=state.machineConf('ssh.options')) defer.returnValue(cluster) except errors.RemoteError, err: if err.name == 'igs.utils.auth_token.AuthTokenError': raise auth_token.AuthTokenError() else: raise except commands.ProgramRunError: raise errors.RemoteError('SSH failed')
def loadRemoteCluster(state, cl): """ Tries to load a cluster. Returns the cluster on success otherwise throws an error. If the cluster is not actually owned by us throws auth_token.AuthTokenError If it's unresponsive throws errors.RemoteError We also check for SSH being up and throw a RemoteError if it is not responsive """ if cl.master: authToken = auth_token.generateToken(cl.config('cluster.cluster_public_key')) try: clusters = yield clusters_www_client.listClusters(cl.master['public_dns'], {'cluster_name': 'local'}, None, authToken, timeout=10, tries=3) cluster = clusters[0] yield ssh.runProcessSSH(cl.master['public_dns'], 'echo hello', stdoutf=None, stderrf=None, sshUser=state.machineConf('ssh.user'), sshFlags=state.machineConf('ssh.options')) defer.returnValue(cluster) except errors.RemoteError, err: if err.name == 'igs.utils.auth_token.AuthTokenError': raise auth_token.AuthTokenError() else: raise except commands.ProgramRunError: raise errors.RemoteError('SSH failed')
def _delayAutoshutdown(state, batchState): _log(batchState, 'AUTOSHUTDOWN: Trying to touch autoshutdown file') try: clusters = yield clusters_client.listClusters('localhost', { 'cluster_name': batchState['pipeline_config']['cluster.CLUSTER_NAME'] }, 'guest') cluster = clusters[0] if batchState.get('state', None) == COMPLETED_STATE: _log(batchState, 'AUTOSHUTDOWN: Pipeline complete, done') if batchState.get('state', None) != RUNNING_STATE: _log(batchState, 'AUTOSHUTDOWN: Pipeline not running, calling later') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState) elif cluster['state'] == 'running': # We need the SSH options from the machine.conf, ugh I hate these OOB dependencies conf = config.configFromStream(open('/tmp/machine.conf')) _log(batchState, 'AUTOSHUTDOWN: Touching delayautoshutdown') yield ssh.runProcessSSH( cluster['master']['public_dns'], 'touch /var/vappio/runtime/delayautoshutdown', stdoutf=None, stderrf=None, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), log=True) _log(batchState, 'AUTOSHUTDOWN: Setting up next call') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState) else: _log(batchState, 'AUTOSHUTDOWN: Cluster not running, calling later') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState) except: _log(batchState, 'AUTOSHUTDOWN: Cluster does not exist, calling later') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState)
def resume(state, pipeline): # Restart childen first for clusterName, child in pipeline.children: if clusterName == 'local': childPipeline = yield state.pipelinePersist.loadPipelineBy({'pipeline_name': child}, pipeline.userName) yield resume(state, childPipeline) else: try: clusters = yield clusters_www.listClusters('localhost', {'cluster_name': clusterName}, 'guest') cluster = clusters[0] yield pipeline_www.resumePipeline(cluster['master']['public_dns'], 'local', pipeline.userName, child) except Exception, err: log.err('Error trying to resume child') log.err(err)
def handleRealizePhantom(request): yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.setState(tasks_tx.task.TASK_RUNNING).update(numTasks=1)) localClusters = yield www_clusters.listClusters('localhost', {'cluster_name': 'local'}, request.body['user_name']) localCluster = localClusters[0] ctype = localCluster['config']['general.ctype'] dstTagPath = os.path.join(localCluster['config']['dirs.upload_dir'], request.body['tag_name']) metadata = func.updateDict(request.body['metadata'], {'tag_base_dir': dstTagPath}) yield commands.runProcess(['mkdir', '-p', dstTagPath]) try: yield _realizePhantom(ctype, dstTagPath, request.body['phantom']) except RunCommandError, err: yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.addMessage(tasks_tx.task.MSG_ERROR, str(err))) raise err
def _uploadTag(request): localTag = yield request.state.tagPersist.loadTag(request.body['tag_name']) srcClusters = yield www_clusters.listClusters('localhost', {'cluster_name': request.body['src_cluster']}, request.body['user_name']) srcCluster = srcClusters[0] dstClusters = yield www_clusters.listClusters('localhost', {'cluster_name': request.body['dst_cluster']}, request.body['user_name']) dstCluster = dstClusters[0] # We want the trailing '/' so everyone knows it's a directory dstTagPath = os.path.join(dstCluster['config']['dirs.upload_dir'], localTag.tagName) + '/' # Change dstTagPath to use /mnt/nostaging instead of mnt/staging as the destination if the # tag has a nostaging metadata value #if localTag.metadata['nostaging']: # dstTagPath = os.path.join(dstCluster['config']['dirs.nostaging_upload_dir'], localTag.tagName) + '/' if request.body['dst_type'] == 'local' : upload_dir = '/mnt/nostaging/' if 'dirs.nostaging_upload_dir' in dstCluster['config']: upload_dir = dstCluster['config']['dirs.nostaging_upload_dir'] yield _makeDirsOnCluster(dstCluster, [upload_dir]) dstTagPath = os.path.join(upload_dir, localTag.tagName) + '/' baseDirFiles, nonBaseDirFiles = _partitionFiles(localTag.files, localTag.metadata['tag_base_dir']) if baseDirFiles: yield rsync.rsyncTo(dstCluster['master']['public_dns'], localTag.metadata['tag_base_dir'], dstTagPath, baseDirFiles, srcCluster['config']['rsync.options'], srcCluster['config']['rsync.user'], log=True) if nonBaseDirFiles: yield rsync.rsyncTo(dstCluster['master']['public_dns'], '/', dstTagPath, nonBaseDirFiles, srcCluster['config']['rsync.options'], srcCluster['config']['rsync.user'], log=True) remoteFiles = ([os.path.join(dstTagPath, f) for f in baseDirFiles] + [os.path.join(dstTagPath, _makePathRelative(f)) for f in nonBaseDirFiles]) metadata = localTag.metadata if metadata.get('urls', []) and not metadata.get('urls_realized', False): tag = yield _realizeUrls(request) remoteFiles.extend(tag['files']) metadata = func.updateDict(metadata, {'urls_realized': True}) # If we are dealing with an adhoc cluster here there is a chance that # we are attempting to transfer to a local VM's shared folder which does # support chown/chmod so we will skip this step if dstCluster['master']['instance_type'] is not None: yield ssh.runProcessSSH(dstCluster['master']['public_dns'], 'chown -R %s %s' % (dstCluster['config']['vappio.user'], dstTagPath), None, log.err, srcCluster['config']['ssh.user'], srcCluster['config']['ssh.options']) defer.returnValue(persist.Tag(tagName=localTag.tagName, files=remoteFiles, metadata=func.updateDict(metadata, {'tag_base_dir': dstTagPath}), phantom=localTag.phantom, taskName=None))
def loadCluster(host, clusterName, userName): clusters = yield clusters_client.listClusters(host, {'cluster_name': clusterName}, userName) cluster = clusters[0] defer.returnValue(cluster)
def listClusters(criteria, userName, timeout=30, tries=4): return clusters.listClusters('localhost', criteria, userName, timeout=timeout, tries=tries)
def _uploadTag(request): localTag = yield request.state.tagPersist.loadTag(request.body['tag_name']) srcClusters = yield www_clusters.listClusters( 'localhost', {'cluster_name': request.body['src_cluster']}, request.body['user_name']) srcCluster = srcClusters[0] dstClusters = yield www_clusters.listClusters( 'localhost', {'cluster_name': request.body['dst_cluster']}, request.body['user_name']) dstCluster = dstClusters[0] # We want the trailing '/' so everyone knows it's a directory dstTagPath = os.path.join(dstCluster['config']['dirs.upload_dir'], localTag.tagName) + '/' # Change dstTagPath to use /mnt/nostaging instead of mnt/staging as the destination if the # tag has a nostaging metadata value #if localTag.metadata['nostaging']: # dstTagPath = os.path.join(dstCluster['config']['dirs.nostaging_upload_dir'], localTag.tagName) + '/' if request.body['dst_type'] == 'local': upload_dir = '/mnt/nostaging/' if 'dirs.nostaging_upload_dir' in dstCluster['config']: upload_dir = dstCluster['config']['dirs.nostaging_upload_dir'] yield _makeDirsOnCluster(dstCluster, [upload_dir]) dstTagPath = os.path.join(upload_dir, localTag.tagName) + '/' baseDirFiles, nonBaseDirFiles = _partitionFiles( localTag.files, localTag.metadata['tag_base_dir']) if baseDirFiles: yield rsync.rsyncTo(dstCluster['master']['public_dns'], localTag.metadata['tag_base_dir'], dstTagPath, baseDirFiles, srcCluster['config']['rsync.options'], srcCluster['config']['rsync.user'], log=True) if nonBaseDirFiles: yield rsync.rsyncTo(dstCluster['master']['public_dns'], '/', dstTagPath, nonBaseDirFiles, srcCluster['config']['rsync.options'], srcCluster['config']['rsync.user'], log=True) remoteFiles = ([os.path.join(dstTagPath, f) for f in baseDirFiles] + [ os.path.join(dstTagPath, _makePathRelative(f)) for f in nonBaseDirFiles ]) metadata = localTag.metadata if metadata.get('urls', []) and not metadata.get('urls_realized', False): tag = yield _realizeUrls(request) remoteFiles.extend(tag['files']) metadata = func.updateDict(metadata, {'urls_realized': True}) # If we are dealing with an adhoc cluster here there is a chance that # we are attempting to transfer to a local VM's shared folder which does # support chown/chmod so we will skip this step if dstCluster['master']['instance_type'] is not None: yield ssh.runProcessSSH( dstCluster['master']['public_dns'], 'chown -R %s %s' % (dstCluster['config']['vappio.user'], dstTagPath), None, log.err, srcCluster['config']['ssh.user'], srcCluster['config']['ssh.options']) defer.returnValue( persist.Tag(tagName=localTag.tagName, files=remoteFiles, metadata=func.updateDict(metadata, {'tag_base_dir': dstTagPath}), phantom=localTag.phantom, taskName=None))
def _handleTransferTag(request): yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.setState(tasks_tx.task.TASK_RUNNING).update(numTasks=2)) srcTag = yield www_tags.loadTag('localhost', request.body['src_cluster'], request.body['user_name'], request.body['tag_name']) if not srcTag['phantom'] and (request.body['src_cluster'] != 'local' or request.body['dst_cluster'] != 'local'): if request.body['src_cluster'] == 'local': tag = yield _uploadTag(request) elif request.body['dst_cluster'] == 'local': tag = yield _downloadTag(request) else: raise NoLocalClusterError('Source cluster or destination cluster must be local') yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.progress()) if request.body.get('compress', False) or request.body.get('compress_dir', False): defaultDir = '/mnt/output' if request.body['dst_cluster'] == 'local' else tag.metadata['tag_base_dir'] compressDir = request.body.get('compress_dir') if request.body.get('compress_dir', False) else defaultDir else: compressDir = None if request.body['dst_cluster'] == 'local': yield tag_mq_data.tagData(request.state, request.body['tag_name'], request.body['task_name'], files=tag.files, action=tag_mq_data.ACTION_OVERWRITE, metadata=tag.metadata, recursive=False, expand=False, compressDir=compressDir) else: newTag = yield www_tags.tagData('localhost', request.body['dst_cluster'], request.body['user_name'], action=tag_mq_data.ACTION_OVERWRITE, tagName=tag.tagName, files=tag.files, metadata=tag.metadata, recursive=False, expand=False, compressDir=compressDir) localTask = yield tasks_tx.loadTask(request.body['task_name']) endState, tsk = yield tasks_tx.blockOnTaskAndForward('localhost', request.body['dst_cluster'], newTag['task_name'], localTask) if endState == tasks_tx.task.TASK_FAILED: yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.setState(tasks_tx.task.TASK_FAILED)) raise TransferTagError(request.body['tag_name']) yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.progress()) elif not srcTag['phantom'] and srcTag['metadata'].get('urls', []) and not srcTag['metadata'].get('urls_realized', False): # It's a local to local but we have urls and haven't realized them yield _realizeUrls(request) yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.progress(2)) elif srcTag['phantom']: # Upload the depends file srcClusters = yield www_clusters.listClusters('localhost', {'cluster_name': request.body['src_cluster']}, request.body['user_name']) srcCluster = srcClusters[0] dstClusters = yield www_clusters.listClusters('localhost', {'cluster_name': request.body['dst_cluster']}, request.body['user_name']) dstCluster = dstClusters[0] dependsOn = srcTag['phantom'].get('depends_on', '').split() yield rsync.rsyncTo(dstCluster['master']['public_dns'], '/', '/', dependsOn, srcCluster['config']['rsync.options'], srcCluster['config']['rsync.user'], log=True) taskName = yield www_tags.realizePhantom('localhost', request.body['dst_cluster'], request.body['user_name'], srcTag['tag_name'], srcTag['phantom'], srcTag['metadata']) localTask = yield tasks_tx.loadTask(request.body['task_name']) endState, tsk = yield tasks_tx.blockOnTaskAndForward('localhost', request.body['dst_cluster'], taskName, localTask) if endState == tasks_tx.task.TASK_FAILED: yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.setState(tasks_tx.task.TASK_FAILED)) raise RealizePhantomError(request.body['tag_name']) yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.update(numTasks=1).progress()) else: yield tag_mq_data.tagData(request.state, request.body['tag_name'], request.body['task_name'], files=[], action=tag_mq_data.ACTION_APPEND, metadata={}, recursive=False, expand=False, compressDir='/mnt/output' if request.body.get('compress', False) else None) yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.progress(2)) defer.returnValue(request)
def loadCluster(host, clusterName, userName): clusters = yield clusters_client.listClusters( host, {'cluster_name': clusterName}, userName) cluster = clusters[0] defer.returnValue(cluster)
def listClusters(criteria, userName, timeout=30, tries=4): return clusters.listClusters("localhost", criteria, userName, timeout=timeout, tries=tries)
def importCluster(state, credClient, taskName, remoteHost, srcCluster, cluster): """Handles retrieving metadata from the remote host and running through a battery of tests to ensure that the VM being imported is in a running state and reachable. """ @defer.inlineCallbacks def _saveCluster(instances): instances = yield credClient.updateInstances(instances) cl = yield state.persistManager.loadCluster(cluster.clusterName, cluster.userName) cl = cl.setMaster(instances[0]) yield state.persistManager.saveCluster(cl) defer.returnValue(func.Record(succeeded=instances, failed=[])) authToken = auth_token.generateToken( cluster.config('cluster.cluster_public_key')) remoteClusters = yield clusters_client_www.listClusters( remoteHost, {'cluster_name': srcCluster}, cluster.userName, authToken) remoteCluster = remoteClusters[0] if remoteCluster.get('state') in ['terminated', 'failed']: raise Error('Imported cluster in TERMINATED or FAILED state') # If we are importing a local cluster the public and private DNS will # not be valid hostnames that we can query. Need to set them to the # remote host provided in the import-clusters call if 'clovr-' in remoteCluster['master']['public_dns']: remoteCluster['master']['public_dns'] = remoteHost remoteCluster['master']['private_dns'] = remoteHost # Sorta hacky but we need to check whether or not a master node is # associated with the cluster being imported before proceeding _instances = yield waitForInstances([remoteCluster], [ updateTask(taskName, 'Waiting for populated master node'), waitForPopulatedMasterNode(srcCluster, authToken, WAIT_FOR_STATE_TRIES) ]) if not _instances.succeeded: raise Error('Could not retrieve master node from imported cluster.') baseConf = config.configFromMap(cluster.config.conf) remoteClusterConf = config.configFromMap( {'general.src_cluster': srcCluster}, base=baseConf) cl = cluster.update(config=remoteClusterConf) cl = cl.setMaster(remoteCluster.get('master')) yield state.persistManager.saveCluster(cl) log.msg('DEBUG importCluster: About to run tests on master node') _instances = yield waitForInstances([remoteCluster.get('master')], [ updateTask(taskName, 'Waiting for master'), waitForState(credClient, 'running', WAIT_FOR_STATE_TRIES), _saveCluster, waitForSSH(cluster.config('ssh.user'), cluster.config('ssh.options'), WAIT_FOR_SSH_TRIES), _saveCluster, updateTask(taskName, 'SSH up'), updateTask(taskName, 'Master in running state') ]) if not _instances.succeeded: raise Error('Failed to import cluster') # TODO: Maybe implement another set of checks here on our exec nodes. if remoteCluster.get('exec_nodes'): cl = cl.addExecNodes(remoteCluster.get('exec_nodes')) yield state.persistManager.saveCluster(cl) cl = yield state.persistManager.loadCluster(cl.clusterName, cl.userName) cl = cl.setState(cl.RUNNING) yield state.persistManager.saveCluster(cl) defer.returnValue(cl)
def importCluster(state, credClient, taskName, remoteHost, srcCluster, cluster): """Handles retrieving metadata from the remote host and running through a battery of tests to ensure that the VM being imported is in a running state and reachable. """ @defer.inlineCallbacks def _saveCluster(instances): instances = yield credClient.updateInstances(instances) cl = yield state.persistManager.loadCluster(cluster.clusterName, cluster.userName) cl = cl.setMaster(instances[0]) yield state.persistManager.saveCluster(cl) defer.returnValue(func.Record(succeeded=instances, failed=[])) authToken = auth_token.generateToken(cluster.config('cluster.cluster_public_key')) remoteClusters = yield clusters_client_www.listClusters(remoteHost, {'cluster_name': srcCluster}, cluster.userName, authToken) remoteCluster = remoteClusters[0] if remoteCluster.get('state') in ['terminated', 'failed']: raise Error('Imported cluster in TERMINATED or FAILED state') # If we are importing a local cluster the public and private DNS will # not be valid hostnames that we can query. Need to set them to the # remote host provided in the import-clusters call if 'clovr-' in remoteCluster['master']['public_dns']: remoteCluster['master']['public_dns'] = remoteHost remoteCluster['master']['private_dns'] = remoteHost # Sorta hacky but we need to check whether or not a master node is # associated with the cluster being imported before proceeding _instances = yield waitForInstances([remoteCluster], [updateTask(taskName, 'Waiting for populated master node'), waitForPopulatedMasterNode(srcCluster, authToken, WAIT_FOR_STATE_TRIES)]) if not _instances.succeeded: raise Error('Could not retrieve master node from imported cluster.') baseConf = config.configFromMap(cluster.config.conf) remoteClusterConf = config.configFromMap({'general.src_cluster': srcCluster}, base=baseConf) cl = cluster.update(config=remoteClusterConf) cl = cl.setMaster(remoteCluster.get('master')) yield state.persistManager.saveCluster(cl) log.msg('DEBUG importCluster: About to run tests on master node') _instances = yield waitForInstances([remoteCluster.get('master')], [updateTask(taskName, 'Waiting for master'), waitForState(credClient, 'running', WAIT_FOR_STATE_TRIES), _saveCluster, waitForSSH(cluster.config('ssh.user'), cluster.config('ssh.options'), WAIT_FOR_SSH_TRIES), _saveCluster, updateTask(taskName, 'SSH up'), updateTask(taskName, 'Master in running state')]) if not _instances.succeeded: raise Error('Failed to import cluster') # TODO: Maybe implement another set of checks here on our exec nodes. if remoteCluster.get('exec_nodes'): cl = cl.addExecNodes(remoteCluster.get('exec_nodes')) yield state.persistManager.saveCluster(cl) cl = yield state.persistManager.loadCluster(cl.clusterName, cl.userName) cl = cl.setState(cl.RUNNING) yield state.persistManager.saveCluster(cl) defer.returnValue(cl)
def _handleTransferTag(request): yield tasks_tx.updateTask( request.body['task_name'], lambda t: t.setState(tasks_tx.task.TASK_RUNNING).update(numTasks=2)) srcTag = yield www_tags.loadTag('localhost', request.body['src_cluster'], request.body['user_name'], request.body['tag_name']) if not srcTag['phantom'] and (request.body['src_cluster'] != 'local' or request.body['dst_cluster'] != 'local'): if request.body['src_cluster'] == 'local': tag = yield _uploadTag(request) elif request.body['dst_cluster'] == 'local': tag = yield _downloadTag(request) else: raise NoLocalClusterError( 'Source cluster or destination cluster must be local') yield tasks_tx.updateTask(request.body['task_name'], lambda t: t.progress()) if request.body.get('compress', False) or request.body.get( 'compress_dir', False): defaultDir = '/mnt/output' if request.body[ 'dst_cluster'] == 'local' else tag.metadata['tag_base_dir'] compressDir = request.body.get('compress_dir') if request.body.get( 'compress_dir', False) else defaultDir else: compressDir = None if request.body['dst_cluster'] == 'local': yield tag_mq_data.tagData(request.state, request.body['tag_name'], request.body['task_name'], files=tag.files, action=tag_mq_data.ACTION_OVERWRITE, metadata=tag.metadata, recursive=False, expand=False, compressDir=compressDir) else: newTag = yield www_tags.tagData( 'localhost', request.body['dst_cluster'], request.body['user_name'], action=tag_mq_data.ACTION_OVERWRITE, tagName=tag.tagName, files=tag.files, metadata=tag.metadata, recursive=False, expand=False, compressDir=compressDir) localTask = yield tasks_tx.loadTask(request.body['task_name']) endState, tsk = yield tasks_tx.blockOnTaskAndForward( 'localhost', request.body['dst_cluster'], newTag['task_name'], localTask) if endState == tasks_tx.task.TASK_FAILED: yield tasks_tx.updateTask( request.body['task_name'], lambda t: t.setState(tasks_tx.task.TASK_FAILED)) raise TransferTagError(request.body['tag_name']) yield tasks_tx.updateTask(request.body['task_name'], lambda t: t.progress()) elif not srcTag['phantom'] and srcTag['metadata'].get( 'urls', []) and not srcTag['metadata'].get('urls_realized', False): # It's a local to local but we have urls and haven't realized them yield _realizeUrls(request) yield tasks_tx.updateTask(request.body['task_name'], lambda t: t.progress(2)) elif srcTag['phantom']: # Upload the depends file srcClusters = yield www_clusters.listClusters( 'localhost', {'cluster_name': request.body['src_cluster']}, request.body['user_name']) srcCluster = srcClusters[0] dstClusters = yield www_clusters.listClusters( 'localhost', {'cluster_name': request.body['dst_cluster']}, request.body['user_name']) dstCluster = dstClusters[0] dependsOn = srcTag['phantom'].get('depends_on', '').split() yield rsync.rsyncTo(dstCluster['master']['public_dns'], '/', '/', dependsOn, srcCluster['config']['rsync.options'], srcCluster['config']['rsync.user'], log=True) taskName = yield www_tags.realizePhantom('localhost', request.body['dst_cluster'], request.body['user_name'], srcTag['tag_name'], srcTag['phantom'], srcTag['metadata']) localTask = yield tasks_tx.loadTask(request.body['task_name']) endState, tsk = yield tasks_tx.blockOnTaskAndForward( 'localhost', request.body['dst_cluster'], taskName, localTask) if endState == tasks_tx.task.TASK_FAILED: yield tasks_tx.updateTask( request.body['task_name'], lambda t: t.setState(tasks_tx.task.TASK_FAILED)) raise RealizePhantomError(request.body['tag_name']) yield tasks_tx.updateTask(request.body['task_name'], lambda t: t.update(numTasks=1).progress()) else: yield tag_mq_data.tagData( request.state, request.body['tag_name'], request.body['task_name'], files=[], action=tag_mq_data.ACTION_APPEND, metadata={}, recursive=False, expand=False, compressDir='/mnt/output' if request.body.get('compress', False) else None) yield tasks_tx.updateTask(request.body['task_name'], lambda t: t.progress(2)) defer.returnValue(request)