def _realizeUrls(request): localTag = yield request.state.tagPersist.loadTag(request.body['tag_name']) # If we have urls we create a fake phantom tag fakePhantom = {'cluster.ALL.command': 'reliableDownloader.py -m 300 -t 20 -b ${base_dir} ' + ' '.join(localTag.metadata['urls'])} taskName = yield www_tags.realizePhantom('localhost', request.body['dst_cluster'], request.body['user_name'], localTag.tagName, fakePhantom, func.updateDict(localTag.metadata, {'urls_realized': True})) localTask = yield tasks_tx.loadTask(request.body['task_name']) endState, tsk = yield tasks_tx.blockOnTaskAndForward('localhost', request.body['dst_cluster'], taskName, localTask) if endState == tasks_tx.task.TASK_FAILED: yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.setState(tasks_tx.task.TASK_FAILED)) raise RealizePhantomError(request.body['tag_name']) if request.body['dst_cluster'] == 'local': yield tag_mq_data.tagData(request.state, request.body['tag_name'], request.body['task_name'], files=localTag.files, action=tag_mq_data.ACTION_APPEND, metadata={}, recursive=False, expand=False, compressDir=None) else: localTask = yield www_tags.tagData('localhost', request.body['dst_cluster'], request.body['user_name'], action=tag_mq_data.ACTION_APPEND, tagName=localTag.tagName, files=localTag.files, metadata={}, recursive=False, expand=False, compressDir=None) localTask = yield tasks_tx.loadTask(request.body['task_name']) endState, tsk = yield tasks_tx.blockOnTaskAndForward('localhost', request.body['dst_cluster'], taskName, localTask) if endState == tasks_tx.task.TASK_FAILED: yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.setState(tasks_tx.task.TASK_FAILED)) raise RealizePhantomError(request.body['tag_name']) # Load the tag up and return it so we can have the files it created tag = yield www_tags.loadTag('localhost', request.body['dst_cluster'], request.body['user_name'], request.body['tag_name']) defer.returnValue(tag)
def _realizeUrls(request): localTag = yield request.state.tagPersist.loadTag(request.body['tag_name']) # If we have urls we create a fake phantom tag fakePhantom = { 'cluster.ALL.command': 'reliableDownloader.py -m 300 -t 20 -b ${base_dir} ' + ' '.join(localTag.metadata['urls']) } taskName = yield www_tags.realizePhantom( 'localhost', request.body['dst_cluster'], request.body['user_name'], localTag.tagName, fakePhantom, func.updateDict(localTag.metadata, {'urls_realized': True})) localTask = yield tasks_tx.loadTask(request.body['task_name']) endState, tsk = yield tasks_tx.blockOnTaskAndForward( 'localhost', request.body['dst_cluster'], taskName, localTask) if endState == tasks_tx.task.TASK_FAILED: yield tasks_tx.updateTask( request.body['task_name'], lambda t: t.setState(tasks_tx.task.TASK_FAILED)) raise RealizePhantomError(request.body['tag_name']) if request.body['dst_cluster'] == 'local': yield tag_mq_data.tagData(request.state, request.body['tag_name'], request.body['task_name'], files=localTag.files, action=tag_mq_data.ACTION_APPEND, metadata={}, recursive=False, expand=False, compressDir=None) else: localTask = yield www_tags.tagData('localhost', request.body['dst_cluster'], request.body['user_name'], action=tag_mq_data.ACTION_APPEND, tagName=localTag.tagName, files=localTag.files, metadata={}, recursive=False, expand=False, compressDir=None) localTask = yield tasks_tx.loadTask(request.body['task_name']) endState, tsk = yield tasks_tx.blockOnTaskAndForward( 'localhost', request.body['dst_cluster'], taskName, localTask) if endState == tasks_tx.task.TASK_FAILED: yield tasks_tx.updateTask( request.body['task_name'], lambda t: t.setState(tasks_tx.task.TASK_FAILED)) raise RealizePhantomError(request.body['tag_name']) # Load the tag up and return it so we can have the files it created tag = yield www_tags.loadTag('localhost', request.body['dst_cluster'], request.body['user_name'], request.body['tag_name']) defer.returnValue(tag)
def _handleTransferTag(request): yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.setState(tasks_tx.task.TASK_RUNNING).update(numTasks=2)) srcTag = yield www_tags.loadTag('localhost', request.body['src_cluster'], request.body['user_name'], request.body['tag_name']) if not srcTag['phantom'] and (request.body['src_cluster'] != 'local' or request.body['dst_cluster'] != 'local'): if request.body['src_cluster'] == 'local': tag = yield _uploadTag(request) elif request.body['dst_cluster'] == 'local': tag = yield _downloadTag(request) else: raise NoLocalClusterError('Source cluster or destination cluster must be local') yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.progress()) if request.body.get('compress', False) or request.body.get('compress_dir', False): defaultDir = '/mnt/output' if request.body['dst_cluster'] == 'local' else tag.metadata['tag_base_dir'] compressDir = request.body.get('compress_dir') if request.body.get('compress_dir', False) else defaultDir else: compressDir = None if request.body['dst_cluster'] == 'local': yield tag_mq_data.tagData(request.state, request.body['tag_name'], request.body['task_name'], files=tag.files, action=tag_mq_data.ACTION_OVERWRITE, metadata=tag.metadata, recursive=False, expand=False, compressDir=compressDir) else: newTag = yield www_tags.tagData('localhost', request.body['dst_cluster'], request.body['user_name'], action=tag_mq_data.ACTION_OVERWRITE, tagName=tag.tagName, files=tag.files, metadata=tag.metadata, recursive=False, expand=False, compressDir=compressDir) localTask = yield tasks_tx.loadTask(request.body['task_name']) endState, tsk = yield tasks_tx.blockOnTaskAndForward('localhost', request.body['dst_cluster'], newTag['task_name'], localTask) if endState == tasks_tx.task.TASK_FAILED: yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.setState(tasks_tx.task.TASK_FAILED)) raise TransferTagError(request.body['tag_name']) yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.progress()) elif not srcTag['phantom'] and srcTag['metadata'].get('urls', []) and not srcTag['metadata'].get('urls_realized', False): # It's a local to local but we have urls and haven't realized them yield _realizeUrls(request) yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.progress(2)) elif srcTag['phantom']: # Upload the depends file srcClusters = yield www_clusters.listClusters('localhost', {'cluster_name': request.body['src_cluster']}, request.body['user_name']) srcCluster = srcClusters[0] dstClusters = yield www_clusters.listClusters('localhost', {'cluster_name': request.body['dst_cluster']}, request.body['user_name']) dstCluster = dstClusters[0] dependsOn = srcTag['phantom'].get('depends_on', '').split() yield rsync.rsyncTo(dstCluster['master']['public_dns'], '/', '/', dependsOn, srcCluster['config']['rsync.options'], srcCluster['config']['rsync.user'], log=True) taskName = yield www_tags.realizePhantom('localhost', request.body['dst_cluster'], request.body['user_name'], srcTag['tag_name'], srcTag['phantom'], srcTag['metadata']) localTask = yield tasks_tx.loadTask(request.body['task_name']) endState, tsk = yield tasks_tx.blockOnTaskAndForward('localhost', request.body['dst_cluster'], taskName, localTask) if endState == tasks_tx.task.TASK_FAILED: yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.setState(tasks_tx.task.TASK_FAILED)) raise RealizePhantomError(request.body['tag_name']) yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.update(numTasks=1).progress()) else: yield tag_mq_data.tagData(request.state, request.body['tag_name'], request.body['task_name'], files=[], action=tag_mq_data.ACTION_APPEND, metadata={}, recursive=False, expand=False, compressDir='/mnt/output' if request.body.get('compress', False) else None) yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.progress(2)) defer.returnValue(request)
def _handleTransferTag(request): yield tasks_tx.updateTask( request.body['task_name'], lambda t: t.setState(tasks_tx.task.TASK_RUNNING).update(numTasks=2)) srcTag = yield www_tags.loadTag('localhost', request.body['src_cluster'], request.body['user_name'], request.body['tag_name']) if not srcTag['phantom'] and (request.body['src_cluster'] != 'local' or request.body['dst_cluster'] != 'local'): if request.body['src_cluster'] == 'local': tag = yield _uploadTag(request) elif request.body['dst_cluster'] == 'local': tag = yield _downloadTag(request) else: raise NoLocalClusterError( 'Source cluster or destination cluster must be local') yield tasks_tx.updateTask(request.body['task_name'], lambda t: t.progress()) if request.body.get('compress', False) or request.body.get( 'compress_dir', False): defaultDir = '/mnt/output' if request.body[ 'dst_cluster'] == 'local' else tag.metadata['tag_base_dir'] compressDir = request.body.get('compress_dir') if request.body.get( 'compress_dir', False) else defaultDir else: compressDir = None if request.body['dst_cluster'] == 'local': yield tag_mq_data.tagData(request.state, request.body['tag_name'], request.body['task_name'], files=tag.files, action=tag_mq_data.ACTION_OVERWRITE, metadata=tag.metadata, recursive=False, expand=False, compressDir=compressDir) else: newTag = yield www_tags.tagData( 'localhost', request.body['dst_cluster'], request.body['user_name'], action=tag_mq_data.ACTION_OVERWRITE, tagName=tag.tagName, files=tag.files, metadata=tag.metadata, recursive=False, expand=False, compressDir=compressDir) localTask = yield tasks_tx.loadTask(request.body['task_name']) endState, tsk = yield tasks_tx.blockOnTaskAndForward( 'localhost', request.body['dst_cluster'], newTag['task_name'], localTask) if endState == tasks_tx.task.TASK_FAILED: yield tasks_tx.updateTask( request.body['task_name'], lambda t: t.setState(tasks_tx.task.TASK_FAILED)) raise TransferTagError(request.body['tag_name']) yield tasks_tx.updateTask(request.body['task_name'], lambda t: t.progress()) elif not srcTag['phantom'] and srcTag['metadata'].get( 'urls', []) and not srcTag['metadata'].get('urls_realized', False): # It's a local to local but we have urls and haven't realized them yield _realizeUrls(request) yield tasks_tx.updateTask(request.body['task_name'], lambda t: t.progress(2)) elif srcTag['phantom']: # Upload the depends file srcClusters = yield www_clusters.listClusters( 'localhost', {'cluster_name': request.body['src_cluster']}, request.body['user_name']) srcCluster = srcClusters[0] dstClusters = yield www_clusters.listClusters( 'localhost', {'cluster_name': request.body['dst_cluster']}, request.body['user_name']) dstCluster = dstClusters[0] dependsOn = srcTag['phantom'].get('depends_on', '').split() yield rsync.rsyncTo(dstCluster['master']['public_dns'], '/', '/', dependsOn, srcCluster['config']['rsync.options'], srcCluster['config']['rsync.user'], log=True) taskName = yield www_tags.realizePhantom('localhost', request.body['dst_cluster'], request.body['user_name'], srcTag['tag_name'], srcTag['phantom'], srcTag['metadata']) localTask = yield tasks_tx.loadTask(request.body['task_name']) endState, tsk = yield tasks_tx.blockOnTaskAndForward( 'localhost', request.body['dst_cluster'], taskName, localTask) if endState == tasks_tx.task.TASK_FAILED: yield tasks_tx.updateTask( request.body['task_name'], lambda t: t.setState(tasks_tx.task.TASK_FAILED)) raise RealizePhantomError(request.body['tag_name']) yield tasks_tx.updateTask(request.body['task_name'], lambda t: t.update(numTasks=1).progress()) else: yield tag_mq_data.tagData( request.state, request.body['tag_name'], request.body['task_name'], files=[], action=tag_mq_data.ACTION_APPEND, metadata={}, recursive=False, expand=False, compressDir='/mnt/output' if request.body.get('compress', False) else None) yield tasks_tx.updateTask(request.body['task_name'], lambda t: t.progress(2)) defer.returnValue(request)