示例#1
0
文件: tasks.py 项目: FBK-WED/wed-pipe
def evaluate_dispatcher_and_run_workflow(scheduler, file_meta):
    """ Evaluates the dispatcher of the given source for the given dataset
    and runs the corresponding workflow.
    """
    loggy = local.logger
    dataset = scheduler.content_object

    file_target = file_meta.get('file_path', file_meta['file_name'])
    archive_item, dummy = ArchiveItem.objects.get_or_create(
        dataset=dataset,
        file_target=file_target
    )

    archive_item.file_hash = file_meta['md5sum']
    archive_item.save(force_update=True)

    wf_input_params = []
    wf_exec_results = []

    # Run dispatcher to configure dataset.
    result = _evaluate_dispatcher(archive_item, file_meta)

    if result == RECURSE_MARKER:
        archive_item.delete()  # archive_item cleanup
        return _handle_recursion(scheduler, file_meta)
    else:
        # if any of the ArchiveItems have no rules attached, mark the
        # scheduler as incomplete
        if not archive_item.get_refine_rule():
            scheduler.status = Scheduler.INCOMPLETE

        if not type(result) == list:
            result = [result]
        for result_elem in result:
            if not len(result_elem) == 2:
                raise Exception(
                    'Invalid workflow configuration from dispatcher. '
                    'Expected couple( "<conf-name>", { <conf-input params> } '
                    ') . Found: ' + result_elem
                )

            configuration, parameters = result_elem

            wf_input_params.append({
                'workflow': configuration,
                'parameters': parameters
            })

            loggy.info('Running workflow: %s', result_elem)
            wf_exec_results.append(Workflow(
                config_name_to_file(configuration),
                __expand_parameters(dataset, parameters),
                store_output=True
            ).run_module())
            loggy.info('Workflow completed')

    return wf_input_params, wf_exec_results
示例#2
0
文件: tasks.py 项目: FBK-WED/wed-pipe
def evaluate_dispatcher_and_run_workflow(scheduler, file_meta):
    """ Evaluates the dispatcher of the given source for the given dataset
    and runs the corresponding workflow.
    """
    loggy = local.logger
    dataset = scheduler.content_object

    file_target = file_meta.get('file_path', file_meta['file_name'])
    archive_item, dummy = ArchiveItem.objects.get_or_create(
        dataset=dataset, file_target=file_target)

    archive_item.file_hash = file_meta['md5sum']
    archive_item.save(force_update=True)

    wf_input_params = []
    wf_exec_results = []

    # Run dispatcher to configure dataset.
    result = _evaluate_dispatcher(archive_item, file_meta)

    if result == RECURSE_MARKER:
        archive_item.delete()  # archive_item cleanup
        return _handle_recursion(scheduler, file_meta)
    else:
        # if any of the ArchiveItems have no rules attached, mark the
        # scheduler as incomplete
        if not archive_item.get_refine_rule():
            scheduler.status = Scheduler.INCOMPLETE

        if not type(result) == list:
            result = [result]
        for result_elem in result:
            if not len(result_elem) == 2:
                raise Exception(
                    'Invalid workflow configuration from dispatcher. '
                    'Expected couple( "<conf-name>", { <conf-input params> } '
                    ') . Found: ' + result_elem)

            configuration, parameters = result_elem

            wf_input_params.append({
                'workflow': configuration,
                'parameters': parameters
            })

            loggy.info('Running workflow: %s', result_elem)
            wf_exec_results.append(
                Workflow(config_name_to_file(configuration),
                         __expand_parameters(dataset, parameters),
                         store_output=True).run_module())
            loggy.info('Workflow completed')

    return wf_input_params, wf_exec_results
示例#3
0
文件: tasks.py 项目: FBK-WED/wed-pipe
def _run_workflow_from_handler(couple):
    """Runs a workflow from Celery."""
    try:
        configuration, parameters = couple
    except ValueError:
        raise Exception(
            'Invalid workflow configuration from handler. Expected '
            'couple( "<conf-name>", { <conf-input params> } ) . '
            'Found: ' + couple)
    return Workflow(
        config_name_to_file(configuration),
        parameters,
        store_output=True,
    ).run_module()
示例#4
0
文件: tasks.py 项目: FBK-WED/wed-pipe
def _run_workflow_from_handler(couple):
    """Runs a workflow from Celery."""
    try:
        configuration, parameters = couple
    except ValueError:
        raise Exception(
            'Invalid workflow configuration from handler. Expected '
            'couple( "<conf-name>", { <conf-input params> } ) . '
            'Found: ' + couple
        )
    return Workflow(
        config_name_to_file(configuration),
        parameters,
        store_output=True,
    ).run_module()