Пример #1
0
    def flush_bundles():
        """
        Having collected bundles in |bundle_infos|, flush them into |blocks|,
        potentially as a single table depending on the mode.
        """
        if len(bundle_infos) == 0:
            return

        def raise_genpath_usage_error():
            raise UsageError('Expected \'% display ' + mode +
                             ' (genpath)\', but got \'% display ' +
                             ' '.join([mode] + args) + '\'')

        # Print out the curent bundles somehow
        mode = current_display[0]
        args = current_display[1:]
        if mode == 'hidden':
            pass
        elif mode == 'contents' or mode == 'image':
            for item_index, bundle_info in bundle_infos:
                if is_missing(bundle_info):
                    blocks.append(MarkupBlockSchema().load({
                        'text': 'ERROR: cannot access bundle',
                        'error': True
                    }).data)
                    continue

                # Parse arguments
                if len(args) == 0:
                    raise_genpath_usage_error()
                # these two are required for the target
                (bundle_uuid,
                 target_genpath) = genpath_to_target(bundle_info, args[0])
                properties = parse_properties(args[1:])

                block_object = {
                    'target_genpath':
                    target_genpath,
                    'bundles_spec':
                    BundleUUIDSpecSchema().load(
                        BundleUUIDSpecSchema.create_json([bundle_info])).data,
                    'status':
                    FetchStatusSchema.get_unknown_status(),
                }

                if mode == 'contents':
                    try:
                        block_object['max_lines'] = int(
                            properties.get('maxlines',
                                           DEFAULT_CONTENTS_MAX_LINES))
                    except ValueError:
                        raise UsageError("maxlines must be integer")
                    blocks.append(
                        BundleContentsBlockSchema().load(block_object).data)
                elif mode == 'image':
                    block_object['width'] = properties.get('width', None)
                    block_object['height'] = properties.get('height', None)
                    blocks.append(
                        BundleImageBlockSchema().load(block_object).data)
        elif mode == 'record':
            # display record schema =>
            # key1: value1
            # key2: value2
            # ...
            schema = get_schema(args)
            for item_index, bundle_info in bundle_infos:
                header = ('key', 'value')
                rows = []
                for (name, genpath, post) in schema:
                    rows.append(RecordsRowSchema().load({
                        'key':
                        name + ':',
                        'value':
                        apply_func(post,
                                   interpret_genpath(bundle_info, genpath)),
                    }).data)
                blocks.append(RecordsBlockSchema().load({
                    'bundles_spec':
                    BundleUUIDSpecSchema().load(
                        BundleUUIDSpecSchema.create_json([bundle_info])).data,
                    'status':
                    FetchStatusSchema.get_unknown_status(),
                    'header':
                    header,
                    'rows':
                    rows,
                }).data)
        elif mode == 'table':
            # display table schema =>
            # key1       key2
            # b1_value1  b1_value2
            # b2_value1  b2_value2
            schema = get_schema(args)
            header = tuple(name for (name, genpath, post) in schema)
            rows = []
            processed_bundle_infos = []
            # Cache the mapping between owner_id to owner on current worksheet
            owner_cache = {}
            for item_index, bundle_info in bundle_infos:
                if 'metadata' in bundle_info:
                    rows.append({
                        name: apply_func(
                            post,
                            interpret_genpath(bundle_info,
                                              genpath,
                                              db_model=db_model,
                                              owner_cache=owner_cache),
                        )
                        for (name, genpath, post) in schema
                    })
                    processed_bundle_infos.append(copy.deepcopy(bundle_info))
                else:
                    # The front-end relies on the name metadata field existing
                    processed_bundle_info = copy.deepcopy(bundle_info)
                    processed_bundle_info['metadata'] = {'name': '<invalid>'}
                    rows.append({
                        name: apply_func(
                            post,
                            interpret_genpath(processed_bundle_info, genpath))
                        for (name, genpath, post) in schema
                    })
                    processed_bundle_infos.append(processed_bundle_info)

            blocks.append(TableBlockSchema().load({
                'bundles_spec':
                BundleUUIDSpecSchema().load(
                    BundleUUIDSpecSchema.create_json(
                        processed_bundle_infos)).data,
                'status':
                FetchStatusSchema.get_unknown_status(),
                'header':
                header,
                'rows':
                rows,
            }).data)

        elif mode == 'graph':
            # display graph <genpath> <properties>
            if len(args) == 0:
                raise_genpath_usage_error()
            # trajectories is list of {
            #   'uuid': ...,
            #   'display_name': ..., # What to show as the description of a bundle
            #   'target': (bundle_uuid, subpath)
            # }
            properties = parse_properties(args[1:])

            trajectories = [{
                'bundle_uuid':
                bundle_info['uuid'],
                'display_name':
                interpret_genpath(bundle_info,
                                  properties.get('display_name', 'name')),
                'target_genpath':
                genpath_to_target(bundle_info, args[0])[1],
            } for item_index, bundle_info in bundle_infos]

            try:
                max_lines = int(
                    properties.get('maxlines', DEFAULT_CONTENTS_MAX_LINES))
            except ValueError:
                raise UsageError("maxlines must be integer")

            blocks.append(GraphBlockSchema().load({
                'trajectories':
                trajectories,
                'bundles_spec':
                BundleUUIDSpecSchema().load(
                    BundleUUIDSpecSchema.create_json([
                        bundle_infos[0][1]
                    ])).data,  # Only show the first one for now
                # 'bundles_spec': BundleUUIDSpecSchema().load(BundleUUIDSpecSchema.create_json(
                #     [copy.deepcopy(bundle_info) for item_index, bundle_info in bundle_infos]).data,
                'max_lines':
                max_lines,
                'xlabel':
                properties.get('xlabel', None),
                'ylabel':
                properties.get('ylabel', None),
            }).data)
        else:
            raise UsageError('Unknown display mode: %s' % mode)
        bundle_infos[:] = []  # Clear
Пример #2
0
def interpret_items(schemas, raw_items, db_model=None):
    """
    Interpret different items based on their types.
    :param schemas: initial mapping from name to list of schema items (columns of a table)
    :param raw_items: list of (raw) worksheet items (triples) to interpret
    :param db_model: database model which is used to query database
    :return: {'items': interpreted_items, ...}, where interpreted_items is a list of:
    {
        'mode': display mode ('markup' | 'contents' | 'image' | 'html', etc.)
        'interpreted': one of
            - rendered string
            - target = (bundle_uuid, genpath)
            - (header = (col1, ..., coln), rows = [{col1:value1, ..., coln:valuen}, ...]) [for tables]
            - {keywords: [...]} for mode = 'search' or 'wsearch'
        'properties': dict of properties (e.g., width, maxlines, etc.),
        'bundle_info': bundle_info or list of bundle_infos,
        'subworksheet_info': subworksheet,
    }
    In addition, return an alignment between the raw items and the interpreted items.
    Each interpreted item has a focusIndex, and possibly consists of a list of
    table rows (indexed by subFocusIndex).  Here is an example:
      --- Raw ---                   --- Interpreted ---
      rawIndex                                         (focusIndex, subFocusIndex)
      0        % display table
      1        [bundle]             [table - row 0     (0, 0)
      2        [bundle]                    - row 1]    (0, 1)
      3
      4        hello                [markup            (1, 0)
      5        world                       ]
      6        [worksheet]          [worksheet]        (2, 0)
      7
    The mapping should be computed as follows:
    - Some raw items contribute directly to a particular interpreted item.
    - Others (blank lines, directives, schema definitions) don't.
    - Those that don't should get mapped to the next interpreted item.
    """
    raw_to_block = []  # rawIndex => (focusIndex, subFocusIndex)

    # Set default schema
    current_schema = None
    default_display = ('table', 'default')
    current_display = default_display
    blocks = []
    bundle_infos = []
    worksheet_infos = []

    def get_schema(args):  # args is a list of schema names
        args = args if len(args) > 0 else ['default']
        schema = []
        for arg in args:
            # If schema doesn't exist, then treat as item (e.g., uuid).
            schema += schemas.get(arg, canonicalize_schema_items([arg.split(':', 2)]))
        return schema

    def is_missing(info):
        return 'metadata' not in info

    def parse_properties(args):
        properties = {}
        for item in args:
            if '=' not in item:
                raise UsageError('Expected <key>=<value>, but got %s' % item)
            key, value = item.split('=', 1)
            properties[key] = value
        return properties

    def genpath_to_target(bundle_info, genpath):
        # bundle_info, '/stdout' => target = (uuid, 'stdout')
        if not is_file_genpath(genpath):
            raise UsageError('Not file genpath: %s' % genpath)
        # strip off the leading / from genpath to create a subpath in the target.
        return (bundle_info['uuid'], genpath[1:])

    def flush_bundles(bundle_block_start_index):
        """
        Having collected bundles in |bundle_infos|, flush them into |blocks|,
        potentially as a single table depending on the mode.
        bundle_block_start_index: The raw index for % display <mode> schema
        """
        if len(bundle_infos) == 0:
            return

        def raise_genpath_usage_error():
            raise UsageError(
                'Expected \'% display '
                + mode
                + ' (genpath)\', but got \'% display '
                + ' '.join([mode] + args)
                + '\''
            )

        # Print out the curent bundles somehow
        mode = current_display[0]
        args = current_display[1:]
        if mode == 'hidden':
            pass
        elif mode == 'contents' or mode == 'image':
            for item_index, bundle_info in bundle_infos:
                if is_missing(bundle_info):
                    blocks.append(
                        MarkupBlockSchema()
                        .load({'text': 'ERROR: cannot access bundle', 'error': True})
                        .data
                    )
                    continue

                # Parse arguments
                if len(args) == 0:
                    raise_genpath_usage_error()
                # these two are required for the target
                (bundle_uuid, target_genpath) = genpath_to_target(bundle_info, args[0])
                properties = parse_properties(args[1:])

                block_object = {
                    'target_genpath': target_genpath,
                    'bundles_spec': BundleUUIDSpecSchema()
                    .load(BundleUUIDSpecSchema.create_json([bundle_info]))
                    .data,
                    'status': FetchStatusSchema.get_unknown_status(),
                }

                if mode == 'contents':
                    try:
                        block_object['max_lines'] = int(
                            properties.get('maxlines', DEFAULT_CONTENTS_MAX_LINES)
                        )
                    except ValueError:
                        raise UsageError("maxlines must be integer")
                    blocks.append(BundleContentsBlockSchema().load(block_object).data)
                elif mode == 'image':
                    block_object['width'] = properties.get('width', None)
                    block_object['height'] = properties.get('height', None)
                    blocks.append(BundleImageBlockSchema().load(block_object).data)
        elif mode == 'record':
            # display record schema =>
            # key1: value1
            # key2: value2
            # ...
            schema = get_schema(args)
            for item_index, bundle_info in bundle_infos:
                header = ('key', 'value')
                rows = []
                for (name, genpath, post, _) in schema:
                    rows.append(
                        RecordsRowSchema()
                        .load(
                            {
                                'key': name + ':',
                                'value': apply_func(post, interpret_genpath(bundle_info, genpath)),
                            }
                        )
                        .data
                    )
                blocks.append(
                    RecordsBlockSchema()
                    .load(
                        {
                            'bundles_spec': BundleUUIDSpecSchema()
                            .load(BundleUUIDSpecSchema.create_json([bundle_info]))
                            .data,
                            'status': FetchStatusSchema.get_unknown_status(),
                            'header': header,
                            'rows': rows,
                            'sort_keys': [bundle_info["sort_key"]],
                            'first_bundle_source_index': bundle_block_start_index,
                            'using_schemas': args if len(args) > 0 else ['default'],
                        }
                    )
                    .data
                )
        elif mode == 'table':
            # display table schema =>
            # key1       key2
            # b1_value1  b1_value2
            # b2_value1  b2_value2
            schema = get_schema(args)
            header = tuple(name for (name, genpath, post, _) in schema)
            rows = []
            processed_bundle_infos = []
            # Cache the mapping between owner_id to owner on current worksheet
            owner_cache = {}
            for item_index, bundle_info in bundle_infos:
                if 'metadata' in bundle_info:
                    rows.append(
                        {
                            name: apply_func(
                                post,
                                interpret_genpath(
                                    bundle_info, genpath, db_model=db_model, owner_cache=owner_cache
                                ),
                            )
                            for (name, genpath, post, _) in schema
                        }
                    )
                    processed_bundle_infos.append(copy.deepcopy(bundle_info))
                else:
                    # The front-end relies on the name metadata field existing
                    processed_bundle_info = copy.deepcopy(bundle_info)
                    processed_bundle_info['metadata'] = {'name': '<invalid>'}
                    rows.append(
                        {
                            name: apply_func(
                                post, interpret_genpath(processed_bundle_info, genpath)
                            )
                            for (name, genpath, post, _) in schema
                        }
                    )
                    processed_bundle_infos.append(processed_bundle_info)

            blocks.append(
                TableBlockSchema()
                .load(
                    {
                        'bundles_spec': BundleUUIDSpecSchema()
                        .load(BundleUUIDSpecSchema.create_json(processed_bundle_infos))
                        .data,
                        'status': FetchStatusSchema.get_unknown_status(),
                        'header': header,
                        'rows': rows,
                        'sort_keys': [
                            processed_bundle_info["sort_key"]
                            for processed_bundle_info in processed_bundle_infos
                        ],
                        'first_bundle_source_index': bundle_block_start_index,
                        'using_schemas': args if len(args) > 0 else ['default'],
                    }
                )
                .data
            )

        elif mode == 'graph':
            # display graph <genpath> <properties>
            if len(args) == 0:
                raise_genpath_usage_error()
            # trajectories is list of {
            #   'uuid': ...,
            #   'display_name': ..., # What to show as the description of a bundle
            #   'target': (bundle_uuid, subpath)
            # }
            properties = parse_properties(args[1:])

            trajectories = [
                {
                    'bundle_uuid': bundle_info['uuid'],
                    'display_name': interpret_genpath(
                        bundle_info, properties.get('display_name', 'name')
                    ),
                    'target_genpath': genpath_to_target(bundle_info, args[0])[1],
                }
                for item_index, bundle_info in bundle_infos
            ]

            try:
                max_lines = int(properties.get('maxlines', DEFAULT_CONTENTS_MAX_LINES))
            except ValueError:
                raise UsageError("maxlines must be integer")

            blocks.append(
                GraphBlockSchema()
                .load(
                    {
                        'trajectories': trajectories,
                        'bundles_spec': BundleUUIDSpecSchema()
                        .load(BundleUUIDSpecSchema.create_json([bundle_infos[0][1]]))
                        .data,  # Only show the first one for now
                        # 'bundles_spec': BundleUUIDSpecSchema().load(BundleUUIDSpecSchema.create_json(
                        #     [copy.deepcopy(bundle_info) for item_index, bundle_info in bundle_infos]).data,
                        'max_lines': max_lines,
                        'xlabel': properties.get('xlabel', None),
                        'ylabel': properties.get('ylabel', None),
                    }
                )
                .data
            )
        else:
            raise UsageError('Unknown display mode: %s' % mode)
        bundle_infos[:] = []  # Clear

    def flush_worksheets():
        if len(worksheet_infos) == 0:
            return

        blocks.append(
            SubworksheetsBlock()
            .load(
                {
                    'subworksheet_infos': copy.deepcopy(worksheet_infos),
                    'sort_keys': [worksheet_info["sort_key"] for worksheet_info in worksheet_infos],
                }
            )
            .data
        )

        worksheet_infos[:] = []

    # Go through all the raw items...
    last_was_empty_line = False
    bundle_block_start_index = -1  # records line for
    current_schema_name = None
    current_schema_ids = []
    for raw_index, item in enumerate(raw_items):
        new_last_was_empty_line = True
        try:
            (bundle_info, subworksheet_info, value_obj, item_type, item_id, sort_key) = item

            is_bundle = item_type == TYPE_BUNDLE
            is_search = item_type == TYPE_DIRECTIVE and get_command(value_obj) == 'search'
            is_directive = item_type == TYPE_DIRECTIVE
            is_worksheet = item_type == TYPE_WORKSHEET
            if not is_bundle:
                flush_bundles(bundle_block_start_index)
                bundle_block_start_index = -1

            if not is_worksheet:
                flush_worksheets()

            # Reset display to minimize long distance dependencies of directives
            if not (is_bundle or is_search):
                current_display = default_display

            # Reset schema to minimize long distance dependencies of directives
            command = get_command(value_obj)
            if not is_directive or (command != "add" and command != "addschema"):
                if current_schema is not None:
                    blocks.append(
                        SchemaBlockSchema()
                        .load(
                            {
                                'status': FetchStatusSchema.get_unknown_status(),
                                'header': ["field", "generalized-path", "post-processor"],
                                'schema_name': current_schema_name,
                                'field_rows': [
                                    {
                                        "field": name,
                                        "generalized-path": path,
                                        "post-processor": post,
                                        "from_schema_name": from_schema_name,
                                    }
                                    for name, path, post, from_schema_name in current_schema
                                ],
                                'sort_keys': [sort_key],
                                'ids': current_schema_ids,
                            }
                        )
                        .data
                    )
                current_schema = None
                current_schema_ids = []

            if item_type == TYPE_BUNDLE:
                if bundle_block_start_index == -1:
                    bundle_block_start_index = raw_index
                bundle_info = dict(bundle_info, sort_key=sort_key)
                raw_to_block.append((len(blocks), len(bundle_infos)))
                bundle_infos.append((raw_index, bundle_info))
            elif item_type == TYPE_WORKSHEET:
                subworksheet_info = dict(subworksheet_info, sort_key=sort_key)
                raw_to_block.append((len(blocks), len(worksheet_infos)))
                worksheet_infos.append(subworksheet_info)
            elif item_type == TYPE_MARKUP:
                new_last_was_empty_line = value_obj == ''
                if (
                    len(blocks) > 0
                    and blocks[-1]['mode'] == BlockModes.markup_block
                    and not last_was_empty_line
                    and not new_last_was_empty_line
                ):
                    # Join with previous markup item
                    blocks[-1]['text'] += '\n' + value_obj
                    # Ids
                    blocks[-1]['ids'] = blocks[-1].get('ids', [])
                    blocks[-1]['ids'].append(item_id)
                    blocks[-1]['sort_keys'] = blocks[-1].get('sort_keys', [])
                    blocks[-1]['sort_keys'].append(sort_key)
                elif not new_last_was_empty_line:
                    block = (
                        MarkupBlockSchema()
                        .load(
                            {
                                'id': len(blocks),
                                'text': value_obj,
                                'ids': [item_id],
                                'sort_keys': [sort_key],
                            }
                        )
                        .data
                    )
                    blocks.append(block)
                # Important: set raw_to_block after so we can focus on current item.
                if new_last_was_empty_line:
                    raw_to_block.append(None)
                else:
                    raw_to_block.append((len(blocks) - 1, 0))
            elif item_type == TYPE_DIRECTIVE:
                command = get_command(value_obj)
                appended_directive_blocks_index = False
                if command == '%' or command == '' or command is None:
                    # Comment
                    pass
                elif command == 'schema':
                    # Start defining new schema
                    if len(value_obj) < 2:
                        raise UsageError("`schema` missing name")
                    name = value_obj[1]
                    current_schema_ids.append(item_id)
                    current_schema_name = name
                    schemas[name] = current_schema = []
                    # Schema block should also be considered when calculating the focus index
                    raw_to_block.append((len(blocks) - 1 + len(current_schema_ids), 0))
                    appended_directive_blocks_index = True
                elif command == 'addschema':
                    # Add to schema
                    if current_schema is None:
                        raise UsageError("`addschema` must be preceded by `schema` directive")
                    if len(value_obj) < 2:
                        raise UsageError("`addschema` missing name")
                    name = value_obj[1]
                    current_schema_ids.append(item_id)
                    current_schema += schemas[name]
                elif command == 'add':
                    # Add to schema
                    if current_schema is None:
                        raise UsageError("`add` must be preceded by `schema` directive")
                    current_schema_ids.append(item_id)
                    schema_item = canonicalize_schema_item(value_obj[1:], current_schema_name)
                    current_schema.append(schema_item)
                elif command == 'display':
                    # Set display
                    current_display = value_obj[1:]
                elif command in ('search', 'wsearch'):
                    # Show item placeholders in brief mode
                    blocks.append(
                        PlaceholderBlockSchema()
                        .load(
                            {
                                'directive': formatting.tokens_to_string(value_obj),
                                'sort_keys': [sort_key],
                            }
                        )
                        .data
                    )
                    appended_directive_blocks_index = True
                    raw_to_block.append((len(blocks) - 1, 0))
                else:
                    raise UsageError("unknown directive `%s`" % command)
                # Add an empty item placeholder for other directives, since they do not represent substantial items
                if not appended_directive_blocks_index:
                    raw_to_block.append(None)
            else:
                raise RuntimeError('Unknown worksheet item type: %s' % item_type)

            # Flush bundles, subworksheets and schema items once more at the end
            if raw_index == len(raw_items) - 1:
                flush_bundles(bundle_block_start_index)
                bundle_block_start_index = -1
                flush_worksheets()
                if current_schema is not None:
                    blocks.append(
                        SchemaBlockSchema()
                        .load(
                            {
                                'status': FetchStatusSchema.get_unknown_status(),
                                'header': ["field", "generalized-path", "post-processor"],
                                'schema_name': current_schema_name,
                                'field_rows': [
                                    {
                                        "field": name,
                                        "generalized-path": path,
                                        "post-processor": post,
                                        "from_schema_name": from_schema_name,
                                    }
                                    for name, path, post, from_schema_name in current_schema
                                ],
                                'sort_keys': [sort_key],
                                'ids': current_schema_ids,
                            }
                        )
                        .data
                    )

        except UsageError as e:
            current_schema = None
            bundle_infos[:] = []
            worksheet_infos[:] = []
            blocks.append(
                MarkupBlockSchema()
                .load(
                    {'text': 'Error in source line %d: %s' % (raw_index + 1, str(e)), 'error': True}
                )
                .data
            )

            raw_to_block.append((len(blocks) - 1, 0))

        except Exception:
            current_schema = None
            bundle_infos[:] = []
            worksheet_infos[:] = []
            import traceback

            traceback.print_exc()
            blocks.append(
                MarkupBlockSchema()
                .load(
                    {
                        'text': 'Unexpected error while parsing line %d' % (raw_index + 1),
                        'error': True,
                    }
                )
                .data
            )

            raw_to_block.append((len(blocks) - 1, 0))

        finally:
            last_was_empty_line = new_last_was_empty_line

    # TODO: fix inconsistencies resulting from UsageErrors thrown in flush_bundles()
    if len(raw_to_block) != len(raw_items):
        print("WARNING: Length of raw_to_block does not match length of raw_items", file=sys.stderr)

    # Package the result
    block_to_raw = {}
    next_interpreted_index = None
    # Go in reverse order so we can assign raw items that map to None to the next interpreted item
    for raw_index, interpreted_index in reversed(list(enumerate(raw_to_block))):
        if interpreted_index is None:  # e.g., blank line, directive
            interpreted_index = next_interpreted_index
            raw_to_block[raw_index] = interpreted_index
        else:
            interpreted_index_str = str(interpreted_index[0]) + ',' + str(interpreted_index[1])
            if interpreted_index_str not in block_to_raw:  # Bias towards the last item
                block_to_raw[interpreted_index_str] = raw_index
        next_interpreted_index = interpreted_index

    # Return the result
    result = {}
    result['blocks'] = blocks
    result['raw_to_block'] = raw_to_block
    result['block_to_raw'] = block_to_raw
    return result
Пример #3
0
def resolve_interpreted_blocks(interpreted_blocks, brief):
    """
    Called by the web interface.  Takes a list of interpreted worksheet
    items (returned by worksheet_util.interpret_items) and fetches the
    appropriate information, replacing the 'interpreted' field in each item.
    The result can be serialized via JSON.
    """
    def set_error_data(block_index, message):
        interpreted_blocks[block_index] = (MarkupBlockSchema().load({
            'id':
            block_index,
            'text':
            'ERROR: ' + message
        }).data)

    for block_index, block in enumerate(interpreted_blocks):
        if block is None:
            continue
        mode = block['mode']

        try:
            # Replace data with a resolved version.
            if mode in (BlockModes.markup_block, BlockModes.placeholder_block):
                # no need to do anything
                pass
            elif mode == BlockModes.record_block or mode == BlockModes.table_block:
                # header_name_posts is a list of (name, post-processing) pairs.
                # Request information
                if brief:
                    # In brief mode, only calculate whether we should interpret genpaths, and if so, set status to briefly_loaded.
                    should_interpret_genpaths = (len(
                        get_genpaths_table_contents_requests(block['rows'])) >
                                                 0)
                    block['status'] = (
                        FetchStatusSchema.get_briefly_loaded_status()
                        if should_interpret_genpaths else
                        FetchStatusSchema.get_ready_status())
                else:
                    block['rows'] = interpret_genpath_table_contents(
                        block['rows'])
                    block['status'] = FetchStatusSchema.get_ready_status()
            elif mode == BlockModes.contents_block or mode == BlockModes.image_block:
                bundle_uuid = block['bundles_spec']['bundle_infos'][0]['uuid']
                target_path = block['target_genpath']
                target = BundleTarget(bundle_uuid, target_path)
                try:
                    target_info = rest_util.get_target_info(target, 0)
                    if target_info[
                            'type'] == 'directory' and mode == BlockModes.contents_block:
                        block['status']['code'] = FetchStatusCodes.ready
                        block['lines'] = ['<directory>']
                    elif target_info['type'] == 'file':
                        block['status']['code'] = FetchStatusCodes.ready
                        if mode == BlockModes.contents_block:
                            block['lines'] = head_target(
                                target_info['resolved_target'],
                                block['max_lines'])
                        elif mode == BlockModes.image_block:
                            block['status']['code'] = FetchStatusCodes.ready
                            block['image_data'] = base64.b64encode(
                                bytes(
                                    cat_target(target_info['resolved_target']))
                            ).decode('utf-8')
                    else:
                        block['status']['code'] = FetchStatusCodes.not_found
                        if mode == BlockModes.contents_block:
                            block['lines'] = None
                        elif mode == BlockModes.image_block:
                            block['image_data'] = None
                except NotFoundError:
                    block['status']['code'] = FetchStatusCodes.not_found
                    if mode == BlockModes.contents_block:
                        block['lines'] = None
                    elif mode == BlockModes.image_block:
                        block['image_data'] = None

            elif mode == BlockModes.graph_block:
                # data = list of {'target': ...}
                # Add a 'points' field that contains the contents of the target.
                for info in block['trajectories']:
                    target = BundleTarget(info['bundle_uuid'],
                                          info['target_genpath'])
                    try:
                        target_info = rest_util.get_target_info(target, 0)
                    except NotFoundError:
                        continue
                    if target_info['type'] == 'file':
                        contents = head_target(target_info['resolved_target'],
                                               block['max_lines'])
                        # Assume TSV file without header for now, just return each line as a row
                        info['points'] = points = []
                        for line in contents:
                            row = line.split('\t')
                            points.append(row)
            elif mode == BlockModes.subworksheets_block:
                # do nothing
                pass
            elif mode == BlockModes.schema_block:
                pass
            else:
                raise UsageError('Invalid display mode: %s' % mode)

        except UsageError as e:
            set_error_data(block_index, str(e))

        except Exception:
            import traceback

            traceback.print_exc()
            set_error_data(block_index, "Unexpected error interpreting item")

        block['is_refined'] = True

    return interpreted_blocks
Пример #4
0
    def flush_bundles():
        """
        Having collected bundles in |bundle_infos|, flush them into |blocks|,
        potentially as a single table depending on the mode.
        """
        if len(bundle_infos) == 0:
            return

        def raise_genpath_usage_error():
            raise UsageError(
                'Expected \'% display '
                + mode
                + ' (genpath)\', but got \'% display '
                + ' '.join([mode] + args)
                + '\''
            )

        # Print out the curent bundles somehow
        mode = current_display[0]
        args = current_display[1:]
        if mode == 'hidden':
            pass
        elif mode == 'contents' or mode == 'image':
            for item_index, bundle_info in bundle_infos:
                if is_missing(bundle_info):
                    blocks.append(
                        MarkupBlockSchema().load({'text': 'ERROR: cannot access bundle'}).data
                    )
                    continue

                # Parse arguments
                if len(args) == 0:
                    raise_genpath_usage_error()
                # these two are required for the target
                (bundle_uuid, target_genpath) = genpath_to_target(bundle_info, args[0])
                properties = parse_properties(args[1:])

                block_object = {
                    'target_genpath': target_genpath,
                    'bundles_spec': BundleUUIDSpecSchema()
                    .load(BundleUUIDSpecSchema.create_json([bundle_info]))
                    .data,
                    'status': FetchStatusSchema.get_unknown_status(),
                }

                if mode == 'contents':
                    try:
                        block_object['max_lines'] = int(
                            properties.get('maxlines', DEFAULT_CONTENTS_MAX_LINES)
                        )
                    except ValueError:
                        raise UsageError("maxlines must be integer")
                    blocks.append(BundleContentsBlockSchema().load(block_object).data)
                elif mode == 'image':
                    block_object['width'] = properties.get('width', None)
                    block_object['height'] = properties.get('height', None)
                    blocks.append(BundleImageBlockSchema().load(block_object).data)
        elif mode == 'record':
            # display record schema =>
            # key1: value1
            # key2: value2
            # ...
            schema = get_schema(args)
            for item_index, bundle_info in bundle_infos:
                header = ('key', 'value')
                rows = []
                for (name, genpath, post) in schema:
                    rows.append(
                        RecordsRowSchema()
                        .load(
                            {
                                'key': name + ':',
                                'value': apply_func(post, interpret_genpath(bundle_info, genpath)),
                            }
                        )
                        .data
                    )
                blocks.append(
                    RecordsBlockSchema()
                    .load(
                        {
                            'bundles_spec': BundleUUIDSpecSchema()
                            .load(BundleUUIDSpecSchema.create_json([bundle_info]))
                            .data,
                            'status': FetchStatusSchema.get_unknown_status(),
                            'header': header,
                            'rows': rows,
                        }
                    )
                    .data
                )
        elif mode == 'table':
            # display table schema =>
            # key1       key2
            # b1_value1  b1_value2
            # b2_value1  b2_value2
            schema = get_schema(args)
            header = tuple(name for (name, genpath, post) in schema)
            rows = []
            processed_bundle_infos = []
            for item_index, bundle_info in bundle_infos:
                if 'metadata' in bundle_info:
                    rows.append(
                        {
                            name: apply_func(post, interpret_genpath(bundle_info, genpath))
                            for (name, genpath, post) in schema
                        }
                    )
                    processed_bundle_infos.append(copy.deepcopy(bundle_info))
                else:
                    # The front-end relies on the name metadata field existing
                    processed_bundle_info = copy.deepcopy(bundle_info)
                    processed_bundle_info['metadata'] = {'name': '<invalid>'}
                    rows.append(
                        {
                            name: apply_func(
                                post, interpret_genpath(processed_bundle_info, genpath)
                            )
                            for (name, genpath, post) in schema
                        }
                    )
                    processed_bundle_infos.append(processed_bundle_info)
            blocks.append(
                TableBlockSchema()
                .load(
                    {
                        'bundles_spec': BundleUUIDSpecSchema()
                        .load(BundleUUIDSpecSchema.create_json(processed_bundle_infos))
                        .data,
                        'status': FetchStatusSchema.get_unknown_status(),
                        'header': header,
                        'rows': rows,
                    }
                )
                .data
            )

        elif mode == 'graph':
            # display graph <genpath> <properties>
            if len(args) == 0:
                raise_genpath_usage_error()
            # trajectories is list of {
            #   'uuid': ...,
            #   'display_name': ..., # What to show as the description of a bundle
            #   'target': (bundle_uuid, subpath)
            # }
            properties = parse_properties(args[1:])

            trajectories = [
                {
                    'bundle_uuid': bundle_info['uuid'],
                    'display_name': interpret_genpath(
                        bundle_info, properties.get('display_name', 'name')
                    ),
                    'target_genpath': genpath_to_target(bundle_info, args[0])[1],
                }
                for item_index, bundle_info in bundle_infos
            ]

            try:
                max_lines = int(properties.get('maxlines', DEFAULT_CONTENTS_MAX_LINES))
            except ValueError:
                raise UsageError("maxlines must be integer")

            blocks.append(
                GraphBlockSchema()
                .load(
                    {
                        'trajectories': trajectories,
                        'bundles_spec': BundleUUIDSpecSchema()
                        .load(BundleUUIDSpecSchema.create_json([bundle_infos[0][1]]))
                        .data,  # Only show the first one for now
                        # 'bundles_spec': BundleUUIDSpecSchema().load(BundleUUIDSpecSchema.create_json(
                        #     [copy.deepcopy(bundle_info) for item_index, bundle_info in bundle_infos]).data,
                        'max_lines': max_lines,
                        'xlabel': properties.get('xlabel', None),
                        'ylabel': properties.get('ylabel', None),
                    }
                )
                .data
            )
        else:
            raise UsageError('Unknown display mode: %s' % mode)
        bundle_infos[:] = []  # Clear