Ejemplo n.º 1
0
def _realizeUrls(request):
    localTag = yield request.state.tagPersist.loadTag(request.body['tag_name'])
    
    # If we have urls we create a fake phantom tag
    fakePhantom = {'cluster.ALL.command':
                   'reliableDownloader.py -m 300 -t 20 -b ${base_dir} ' + ' '.join(localTag.metadata['urls'])}
    taskName = yield www_tags.realizePhantom('localhost',
                                             request.body['dst_cluster'],
                                             request.body['user_name'],
                                             localTag.tagName,
                                             fakePhantom,
                                             func.updateDict(localTag.metadata, {'urls_realized': True}))
    localTask = yield tasks_tx.loadTask(request.body['task_name'])
    endState, tsk = yield tasks_tx.blockOnTaskAndForward('localhost',
                                                         request.body['dst_cluster'],
                                                         taskName,
                                                         localTask)

    if endState == tasks_tx.task.TASK_FAILED:
        yield tasks_tx.updateTask(request.body['task_name'],
                                  lambda t : t.setState(tasks_tx.task.TASK_FAILED))
        raise RealizePhantomError(request.body['tag_name'])


    if request.body['dst_cluster'] == 'local':
        yield tag_mq_data.tagData(request.state,
                                  request.body['tag_name'],
                                  request.body['task_name'],
                                  files=localTag.files,
                                  action=tag_mq_data.ACTION_APPEND,
                                  metadata={},
                                  recursive=False,
                                  expand=False,
                                  compressDir=None)
    else:
        localTask = yield www_tags.tagData('localhost',
                                           request.body['dst_cluster'],
                                           request.body['user_name'],
                                           action=tag_mq_data.ACTION_APPEND,
                                           tagName=localTag.tagName,
                                           files=localTag.files,
                                           metadata={},
                                           recursive=False,
                                           expand=False,
                                           compressDir=None)

    localTask = yield tasks_tx.loadTask(request.body['task_name'])
    endState, tsk = yield tasks_tx.blockOnTaskAndForward('localhost',
                                                         request.body['dst_cluster'],
                                                         taskName,
                                                         localTask)

    if endState == tasks_tx.task.TASK_FAILED:
        yield tasks_tx.updateTask(request.body['task_name'],
                                  lambda t : t.setState(tasks_tx.task.TASK_FAILED))
        raise RealizePhantomError(request.body['tag_name'])
    
    # Load the tag up and return it so we can have the files it created
    tag = yield www_tags.loadTag('localhost',
                                 request.body['dst_cluster'],
                                 request.body['user_name'],
                                 request.body['tag_name'])

    defer.returnValue(tag)
Ejemplo n.º 2
0
def _handleTransferTag(request):
    yield tasks_tx.updateTask(request.body['task_name'],
                              lambda t : t.setState(tasks_tx.task.TASK_RUNNING).update(numTasks=2))

    srcTag = yield www_tags.loadTag('localhost',
                                    request.body['src_cluster'],
                                    request.body['user_name'],
                                    request.body['tag_name'])

    if not srcTag['phantom'] and (request.body['src_cluster'] != 'local' or request.body['dst_cluster'] != 'local'):
        if request.body['src_cluster'] == 'local':
            tag = yield _uploadTag(request)
        elif request.body['dst_cluster'] == 'local':
            tag = yield _downloadTag(request)
        else:
            raise NoLocalClusterError('Source cluster or destination cluster must be local')

        yield tasks_tx.updateTask(request.body['task_name'],
                                  lambda t : t.progress())

        if request.body.get('compress', False) or request.body.get('compress_dir', False):
            defaultDir = '/mnt/output' if request.body['dst_cluster'] == 'local' else tag.metadata['tag_base_dir']
            compressDir = request.body.get('compress_dir') if request.body.get('compress_dir', False) else defaultDir 
        else:
            compressDir = None
        
        if request.body['dst_cluster'] == 'local':
            yield tag_mq_data.tagData(request.state,
                                      request.body['tag_name'],
                                      request.body['task_name'],
                                      files=tag.files,
                                      action=tag_mq_data.ACTION_OVERWRITE,
                                      metadata=tag.metadata,
                                      recursive=False,
                                      expand=False,
                                      compressDir=compressDir)
        else:
            newTag = yield www_tags.tagData('localhost',
                                            request.body['dst_cluster'],
                                            request.body['user_name'],
                                            action=tag_mq_data.ACTION_OVERWRITE,
                                            tagName=tag.tagName,
                                            files=tag.files,
                                            metadata=tag.metadata,
                                            recursive=False,
                                            expand=False,
                                            compressDir=compressDir)

            localTask = yield tasks_tx.loadTask(request.body['task_name'])
            endState, tsk = yield tasks_tx.blockOnTaskAndForward('localhost',
                                                                 request.body['dst_cluster'],
                                                                 newTag['task_name'],
                                                                 localTask)
            if endState == tasks_tx.task.TASK_FAILED:
                yield tasks_tx.updateTask(request.body['task_name'],
                                          lambda t : t.setState(tasks_tx.task.TASK_FAILED))
                raise TransferTagError(request.body['tag_name'])
    
        yield tasks_tx.updateTask(request.body['task_name'],
                                  lambda t : t.progress())
    elif not srcTag['phantom'] and srcTag['metadata'].get('urls', []) and not srcTag['metadata'].get('urls_realized', False):
        # It's a local to local but we have urls and haven't realized them
        yield _realizeUrls(request)
        yield tasks_tx.updateTask(request.body['task_name'],
                                  lambda t : t.progress(2))
    elif srcTag['phantom']:
        # Upload the depends file
        srcClusters = yield www_clusters.listClusters('localhost',
                                                      {'cluster_name': request.body['src_cluster']},
                                                      request.body['user_name'])

        srcCluster = srcClusters[0]
        
        dstClusters = yield www_clusters.listClusters('localhost',
                                                      {'cluster_name': request.body['dst_cluster']},
                                                      request.body['user_name'])

        dstCluster = dstClusters[0]
        
        dependsOn = srcTag['phantom'].get('depends_on', '').split()
        yield rsync.rsyncTo(dstCluster['master']['public_dns'],
                            '/',
                            '/',
                            dependsOn,
                            srcCluster['config']['rsync.options'],
                            srcCluster['config']['rsync.user'],
                            log=True)
        
        
        taskName = yield www_tags.realizePhantom('localhost',
                                                 request.body['dst_cluster'],
                                                 request.body['user_name'],
                                                 srcTag['tag_name'],
                                                 srcTag['phantom'],
                                                 srcTag['metadata'])
        localTask = yield tasks_tx.loadTask(request.body['task_name'])
        endState, tsk = yield tasks_tx.blockOnTaskAndForward('localhost',
                                                             request.body['dst_cluster'],
                                                             taskName,
                                                             localTask)
        if endState == tasks_tx.task.TASK_FAILED:
            yield tasks_tx.updateTask(request.body['task_name'],
                                      lambda t : t.setState(tasks_tx.task.TASK_FAILED))
            raise RealizePhantomError(request.body['tag_name'])
        yield tasks_tx.updateTask(request.body['task_name'],
                                  lambda t : t.update(numTasks=1).progress())
    else:
        yield tag_mq_data.tagData(request.state,
                                  request.body['tag_name'],
                                  request.body['task_name'],
                                  files=[],
                                  action=tag_mq_data.ACTION_APPEND,
                                  metadata={},
                                  recursive=False,
                                  expand=False,
                                  compressDir='/mnt/output' if request.body.get('compress', False) else None)
        
        yield tasks_tx.updateTask(request.body['task_name'],
                                  lambda t : t.progress(2))
        
    defer.returnValue(request)
Ejemplo n.º 3
0
def _realizeUrls(request):
    localTag = yield request.state.tagPersist.loadTag(request.body['tag_name'])

    # If we have urls we create a fake phantom tag
    fakePhantom = {
        'cluster.ALL.command':
        'reliableDownloader.py -m 300 -t 20 -b ${base_dir} ' +
        ' '.join(localTag.metadata['urls'])
    }
    taskName = yield www_tags.realizePhantom(
        'localhost', request.body['dst_cluster'], request.body['user_name'],
        localTag.tagName, fakePhantom,
        func.updateDict(localTag.metadata, {'urls_realized': True}))
    localTask = yield tasks_tx.loadTask(request.body['task_name'])
    endState, tsk = yield tasks_tx.blockOnTaskAndForward(
        'localhost', request.body['dst_cluster'], taskName, localTask)

    if endState == tasks_tx.task.TASK_FAILED:
        yield tasks_tx.updateTask(
            request.body['task_name'],
            lambda t: t.setState(tasks_tx.task.TASK_FAILED))
        raise RealizePhantomError(request.body['tag_name'])

    if request.body['dst_cluster'] == 'local':
        yield tag_mq_data.tagData(request.state,
                                  request.body['tag_name'],
                                  request.body['task_name'],
                                  files=localTag.files,
                                  action=tag_mq_data.ACTION_APPEND,
                                  metadata={},
                                  recursive=False,
                                  expand=False,
                                  compressDir=None)
    else:
        localTask = yield www_tags.tagData('localhost',
                                           request.body['dst_cluster'],
                                           request.body['user_name'],
                                           action=tag_mq_data.ACTION_APPEND,
                                           tagName=localTag.tagName,
                                           files=localTag.files,
                                           metadata={},
                                           recursive=False,
                                           expand=False,
                                           compressDir=None)

    localTask = yield tasks_tx.loadTask(request.body['task_name'])
    endState, tsk = yield tasks_tx.blockOnTaskAndForward(
        'localhost', request.body['dst_cluster'], taskName, localTask)

    if endState == tasks_tx.task.TASK_FAILED:
        yield tasks_tx.updateTask(
            request.body['task_name'],
            lambda t: t.setState(tasks_tx.task.TASK_FAILED))
        raise RealizePhantomError(request.body['tag_name'])

    # Load the tag up and return it so we can have the files it created
    tag = yield www_tags.loadTag('localhost', request.body['dst_cluster'],
                                 request.body['user_name'],
                                 request.body['tag_name'])

    defer.returnValue(tag)
Ejemplo n.º 4
0
    metadata = func.updateDict(request.body['metadata'], {'tag_base_dir': dstTagPath})
    
    yield commands.runProcess(['mkdir', '-p', dstTagPath])

    try:
        yield _realizePhantom(ctype, dstTagPath, request.body['phantom'])
    except RunCommandError, err:
        yield tasks_tx.updateTask(request.body['task_name'],
                                  lambda t : t.addMessage(tasks_tx.task.MSG_ERROR, str(err)))
        raise err

    yield tag_mq_data.tagData(request.state,
                              tagName=request.body['tag_name'],
                              taskName=request.body['task_name'],
                              files=[dstTagPath],
                              metadata=metadata,
                              action=tag_mq_data.ACTION_OVERWRITE,
                              recursive=True,
                              expand=True,
                              compressDir=None,
                              deleteOnExpand=True)

    yield tasks_tx.updateTask(request.body['task_name'],
                              lambda t : t.progress())

    defer.returnValue(request)

def _forwardToCluster(conf, queueUrl):
    return queue.forwardRequestToCluster(conf('www.url_prefix') + '/' + os.path.basename(queueUrl))

def subscribe(mq, state):
    processRealizePhantom = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
Ejemplo n.º 5
0
def _handleTransferTag(request):
    yield tasks_tx.updateTask(
        request.body['task_name'],
        lambda t: t.setState(tasks_tx.task.TASK_RUNNING).update(numTasks=2))

    srcTag = yield www_tags.loadTag('localhost', request.body['src_cluster'],
                                    request.body['user_name'],
                                    request.body['tag_name'])

    if not srcTag['phantom'] and (request.body['src_cluster'] != 'local'
                                  or request.body['dst_cluster'] != 'local'):
        if request.body['src_cluster'] == 'local':
            tag = yield _uploadTag(request)
        elif request.body['dst_cluster'] == 'local':
            tag = yield _downloadTag(request)
        else:
            raise NoLocalClusterError(
                'Source cluster or destination cluster must be local')

        yield tasks_tx.updateTask(request.body['task_name'],
                                  lambda t: t.progress())

        if request.body.get('compress', False) or request.body.get(
                'compress_dir', False):
            defaultDir = '/mnt/output' if request.body[
                'dst_cluster'] == 'local' else tag.metadata['tag_base_dir']
            compressDir = request.body.get('compress_dir') if request.body.get(
                'compress_dir', False) else defaultDir
        else:
            compressDir = None

        if request.body['dst_cluster'] == 'local':
            yield tag_mq_data.tagData(request.state,
                                      request.body['tag_name'],
                                      request.body['task_name'],
                                      files=tag.files,
                                      action=tag_mq_data.ACTION_OVERWRITE,
                                      metadata=tag.metadata,
                                      recursive=False,
                                      expand=False,
                                      compressDir=compressDir)
        else:
            newTag = yield www_tags.tagData(
                'localhost',
                request.body['dst_cluster'],
                request.body['user_name'],
                action=tag_mq_data.ACTION_OVERWRITE,
                tagName=tag.tagName,
                files=tag.files,
                metadata=tag.metadata,
                recursive=False,
                expand=False,
                compressDir=compressDir)

            localTask = yield tasks_tx.loadTask(request.body['task_name'])
            endState, tsk = yield tasks_tx.blockOnTaskAndForward(
                'localhost', request.body['dst_cluster'], newTag['task_name'],
                localTask)
            if endState == tasks_tx.task.TASK_FAILED:
                yield tasks_tx.updateTask(
                    request.body['task_name'],
                    lambda t: t.setState(tasks_tx.task.TASK_FAILED))
                raise TransferTagError(request.body['tag_name'])

        yield tasks_tx.updateTask(request.body['task_name'],
                                  lambda t: t.progress())
    elif not srcTag['phantom'] and srcTag['metadata'].get(
            'urls', []) and not srcTag['metadata'].get('urls_realized', False):
        # It's a local to local but we have urls and haven't realized them
        yield _realizeUrls(request)
        yield tasks_tx.updateTask(request.body['task_name'],
                                  lambda t: t.progress(2))
    elif srcTag['phantom']:
        # Upload the depends file
        srcClusters = yield www_clusters.listClusters(
            'localhost', {'cluster_name': request.body['src_cluster']},
            request.body['user_name'])

        srcCluster = srcClusters[0]

        dstClusters = yield www_clusters.listClusters(
            'localhost', {'cluster_name': request.body['dst_cluster']},
            request.body['user_name'])

        dstCluster = dstClusters[0]

        dependsOn = srcTag['phantom'].get('depends_on', '').split()
        yield rsync.rsyncTo(dstCluster['master']['public_dns'],
                            '/',
                            '/',
                            dependsOn,
                            srcCluster['config']['rsync.options'],
                            srcCluster['config']['rsync.user'],
                            log=True)

        taskName = yield www_tags.realizePhantom('localhost',
                                                 request.body['dst_cluster'],
                                                 request.body['user_name'],
                                                 srcTag['tag_name'],
                                                 srcTag['phantom'],
                                                 srcTag['metadata'])
        localTask = yield tasks_tx.loadTask(request.body['task_name'])
        endState, tsk = yield tasks_tx.blockOnTaskAndForward(
            'localhost', request.body['dst_cluster'], taskName, localTask)
        if endState == tasks_tx.task.TASK_FAILED:
            yield tasks_tx.updateTask(
                request.body['task_name'],
                lambda t: t.setState(tasks_tx.task.TASK_FAILED))
            raise RealizePhantomError(request.body['tag_name'])
        yield tasks_tx.updateTask(request.body['task_name'],
                                  lambda t: t.update(numTasks=1).progress())
    else:
        yield tag_mq_data.tagData(
            request.state,
            request.body['tag_name'],
            request.body['task_name'],
            files=[],
            action=tag_mq_data.ACTION_APPEND,
            metadata={},
            recursive=False,
            expand=False,
            compressDir='/mnt/output'
            if request.body.get('compress', False) else None)

        yield tasks_tx.updateTask(request.body['task_name'],
                                  lambda t: t.progress(2))

    defer.returnValue(request)