Exemple #1
0
        def worksheet(self, uuid, interpreted=False, fetch_items=True, get_raw=True):
            try:
                worksheet_info  = self.client.get_worksheet_info(
                                            uuid,
                                            fetch_items,  # fetch_items
                                            True,  # get_permissions
                                )
            except PermissionError:
                raise UsageError # forces a not found
            if get_raw:
                worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info)
            # set permissions
            worksheet_info['edit_permission'] = False
            if worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL:
                worksheet_info['edit_permission'] = True

            worksheet_info['permission_str'] = permission_str(worksheet_info['permission'])
            # format each groups as well
            for group_permission in worksheet_info['group_permissions']:
                group_permission['permission_str'] = permission_str(group_permission['permission'])


            if interpreted:
                interpreted_items = worksheet_util.interpret_items(
                                    worksheet_util.get_default_schemas(),
                                    worksheet_info['items']
                                )
                worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items'])
                # Currently, only certain fields are base64 encoded.
                for item in worksheet_info['items']:
                    if item['mode'] in ['html', 'contents']:
                        # item['name'] in ['stdout', 'stderr']
                        if item['interpreted'] is None:
                            item['interpreted'] = ['MISSING']
                        else:
                            item['interpreted'] = map(base64.b64decode, item['interpreted'])
                    elif item['mode'] == 'table':
                        for row_map in item['interpreted'][1]:
                            for k, v in row_map.iteritems():
                                if v is None:
                                     row_map[k] = 'MISSING'
                    if 'bundle_info' in item:
                        if isinstance(item['bundle_info'], list):
                            for bundle_info in item['bundle_info']:
                                try:
                                    ## sometimes bundle_info is a string. when item['mode'] is image
                                    if isinstance(bundle_info, dict):
                                        worksheet_util.format_metadata(bundle_info.get('metadata'))
                                        if bundle_info.get('bundle_type', None) == 'run':
                                            if 'stdout' in bundle_info.keys():
                                                bundle_info['stdout'] = base64.b64decode(bundle_info['stdout'])
                                            if 'stderr' in bundle_info.keys():
                                                bundle_info['stderr'] = base64.b64decode(bundle_info['stderr'])
                                except Exception, e:
                                    print e
                                    import ipdb; ipdb.set_trace()
                        elif isinstance(item['bundle_info'], dict):
                            worksheet_util.format_metadata(item['bundle_info'].get('metadata'))

                return worksheet_info
Exemple #2
0
        def get_bundle_info(self, uuid):
            ## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False):
            bundle_info = _call_with_retries(
                lambda: self.client.get_bundle_info(uuid, True, True, True))
            # format permission data
            bundle_info['permission_str'] = permission_str(
                bundle_info['permission'])
            # format each groups as well
            for group_permission in bundle_info['group_permissions']:
                group_permission['permission_str'] = permission_str(
                    group_permission['permission'])

            metadata = bundle_info['metadata']

            cls = get_bundle_subclass(bundle_info['bundle_type'])
            # format based on specs from the cli
            for spec in cls.METADATA_SPECS:
                key = spec.key
                if key not in metadata:
                    continue
                if metadata[key] == '' or metadata[key] == []:
                    continue
                value = worksheet_util.apply_func(spec.formatting,
                                                  metadata.get(key))
                # if isinstance(value, list):
                #     value = ', '.join(value)
                metadata[key] = value

            bundle_info['metadata'] = metadata

            return bundle_info
Exemple #3
0
        def get_bundle_info(self, uuid):
            ## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False):
            bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))
            # format permission data
            bundle_info['permission_str'] = permission_str(bundle_info['permission'])
            # format each groups as well
            for group_permission in bundle_info['group_permissions']:
                group_permission['permission_str'] = permission_str(group_permission['permission'])

            metadata = bundle_info['metadata']

            cls = get_bundle_subclass(bundle_info['bundle_type'])
            # format based on specs from the cli
            for spec in cls.METADATA_SPECS:
                key = spec.key
                if key not in metadata:
                    continue
                if metadata[key] == '' or metadata[key] == []:
                    continue
                value = worksheet_util.apply_func(spec.formatting, metadata.get(key))
                # if isinstance(value, list):
                #     value = ', '.join(value)
                metadata[key] = value

            bundle_info['metadata'] = metadata

            return bundle_info
Exemple #4
0
    def full_worksheet(self, uuid):
        """
        Return information about a worksheet. Calls
        - get_worksheet_info: get basic info
        - resolve_interpreted_items: get more information about a worksheet.
        In the future, for large worksheets, might want to break this up so
        that we can render something basic.
        """
        worksheet_info = self.client.get_worksheet_info(uuid, True, True)

        # Fetch items.
        worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info)

        # Set permissions
        worksheet_info['edit_permission'] = (worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
        # Check enable chat box
        worksheet_info['enable_chat'] = local.config.get('enable_chat', False)
        # Format permissions into strings
        worksheet_info['permission_str'] = permission_str(worksheet_info['permission'])
        for group_permission in worksheet_info['group_permissions']:
            group_permission['permission_str'] = permission_str(group_permission['permission'])

        # Go and fetch more information about the worksheet contents by
        # resolving the interpreted items.
        try:
            interpreted_items = worksheet_util.interpret_items(
                                worksheet_util.get_default_schemas(),
                                worksheet_info['items'])
        except UsageError, e:
            interpreted_items = {'items': []}
            worksheet_info['error'] = str(e)
Exemple #5
0
        def get_bundle_info(self, uuid):
            bundle_info = _call_with_retries(
                lambda: self.client.get_bundle_info(uuid, True, True, True))

            if bundle_info is None:
                return None
            # Set permissions
            bundle_info['edit_permission'] = (
                bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
            # Format permissions into strings
            bundle_info['permission_str'] = permission_str(
                bundle_info['permission'])
            for group_permission in bundle_info['group_permissions']:
                group_permission['permission_str'] = permission_str(
                    group_permission['permission'])

            metadata = bundle_info['metadata']

            cls = get_bundle_subclass(bundle_info['bundle_type'])
            for key, value in worksheet_util.get_formatted_metadata(
                    cls, metadata):
                metadata[key] = value

            bundle_info['metadata'] = metadata
            bundle_info[
                'editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields(
                    cls, metadata)

            return bundle_info
Exemple #6
0
        def worksheet(self, uuid, interpreted=False):
            try:
                worksheet_info  = self.client.get_worksheet_info(
                                            uuid,
                                            True,  #fetch_items
                                            True,  # get_permissions

                                )
            except PermissionError:
                raise UsageError # forces a not found
            worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info)
            # set permissions
            worksheet_info['edit_permission'] = False
            if worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL:
                worksheet_info['edit_permission'] = True

            worksheet_info['permission_str'] = permission_str(worksheet_info['permission'])
            # format each groups as well
            for group_permission in worksheet_info['group_permissions']:
                group_permission['permission_str'] = permission_str(group_permission['permission'])


            if interpreted:
                interpreted_items = worksheet_util.interpret_items(
                                    worksheet_util.get_default_schemas(),
                                    worksheet_info['items']
                                )
                worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items'])
                return worksheet_info
            else:
                return worksheet_info
Exemple #7
0
    def full_worksheet(self, uuid, bundle_uuids=None):
        """
        Return information about a worksheet. Calls
        - get_worksheet_info: get basic info
        - resolve_interpreted_items: get more information about a worksheet.
        In the future, for large worksheets, might want to break this up so
        that we can render something basic.
        """
        worksheet_info = get_worksheet_info(uuid, fetch_items=True, fetch_permission=True, legacy=True)

        # Fetch items.
        worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info)

        # Set permissions
        worksheet_info['edit_permission'] = (worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
        # Check enable chat box
        worksheet_info['enable_chat'] = local.config.get('enable_chat', False)
        # Format permissions into strings
        worksheet_info['permission_str'] = permission_str(worksheet_info['permission'])
        for group_permission in worksheet_info['group_permissions']:
            group_permission['permission_str'] = permission_str(group_permission['permission'])

        # Go and fetch more information about the worksheet contents by
        # resolving the interpreted items.
        try:
            interpreted_items = worksheet_util.interpret_items(
                                worksheet_util.get_default_schemas(),
                                worksheet_info['items'])
        except UsageError, e:
            interpreted_items = {'items': []}
            worksheet_info['error'] = str(e)
Exemple #8
0
        def worksheet(self, uuid, fetch_items, get_permissions, interpreted):
            """
            Return information about a worksheet. Calls
            - get_worksheet_info: get basic info
            - resolve_interpreted_items: get more information about a worksheet.
            In the future, for large worksheets, might want to break this up so
            that we can render something basic.
            """
            worksheet_info = self.client.get_worksheet_info(uuid, fetch_items, get_permissions)

            if fetch_items:
                worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info)

            # Set permissions
            worksheet_info['edit_permission'] = (worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
            # Format permissions into strings
            worksheet_info['permission_str'] = permission_str(worksheet_info['permission'])
            for group_permission in worksheet_info['group_permissions']:
                group_permission['permission_str'] = permission_str(group_permission['permission'])

            # Go and fetch more information about the worksheet contents by
            # resolving the interpreted items.
            if interpreted:
                try:
                    interpreted_items = worksheet_util.interpret_items(
                                        worksheet_util.get_default_schemas(),
                                        worksheet_info['items'])
                except UsageError, e:
                    interpreted_items = {'items': []}
                    worksheet_info['error'] = str(e)

                worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items'])
                # Currently, only certain fields are base64 encoded.
                for item in worksheet_info['items']:
                    if item['mode'] in ['html', 'contents']:
                        if item['interpreted'] is None:
                            item['interpreted'] = [formatting.contents_str(item['interpreted'])]
                        else:
                            item['interpreted'] = map(base64.b64decode, item['interpreted'])
                    elif item['mode'] == 'table':
                        for row_map in item['interpreted'][1]:
                            for k, v in row_map.iteritems():
                                if v is None:
                                     row_map[k] = formatting.contents_str(v)
                    elif 'bundle_info' in item:
                        infos = []
                        if isinstance(item['bundle_info'], list):
                            infos = item['bundle_info']
                        elif isinstance(item['bundle_info'], dict):
                            infos = [item['bundle_info']]
                        for bundle_info in infos:
                            try:
                                if isinstance(bundle_info, dict):
                                    worksheet_util.format_metadata(bundle_info.get('metadata'))
                            except Exception, e:
                                print e
                                import ipdb; ipdb.set_trace()
Exemple #9
0
def fetch_interpreted_worksheet(uuid):
    """
    Return information about a worksheet. Calls
    - get_worksheet_info: get basic info
    - resolve_interpreted_items: get more information about a worksheet.
    In the future, for large worksheets, might want to break this up so
    that we can render something basic.
    """
    bundle_uuids = request.query.getall('bundle_uuid')
    worksheet_info = get_worksheet_info(uuid,
                                        fetch_items=True,
                                        fetch_permissions=True)

    # Shim in additional data for the frontend
    worksheet_info['items'] = resolve_items_into_infos(worksheet_info['items'])
    if worksheet_info['owner_id'] is None:
        worksheet_info['owner_name'] = None
    else:
        owner = local.model.get_user(user_id=worksheet_info['owner_id'])
        worksheet_info['owner_name'] = owner.user_name

    # Fetch items.
    worksheet_info['raw'] = get_worksheet_lines(worksheet_info)

    # Replace searches with raw items.
    # This needs to be done before get_worksheet_lines because this replaces
    # user-written raw items.
    worksheet_info['items'] = expand_raw_items(worksheet_info['items'])

    # Set permissions
    worksheet_info['edit_permission'] = (
        worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
    # Check enable chat box
    worksheet_info['enable_chat'] = local.config.get('enable_chat', False)
    # Format permissions into strings
    worksheet_info['permission_spec'] = permission_str(
        worksheet_info['permission'])
    for group_permission in worksheet_info['group_permissions']:
        group_permission['permission_spec'] = permission_str(
            group_permission['permission'])

    # Go and fetch more information about the worksheet contents by
    # resolving the interpreted items.
    try:
        interpreted_items = interpret_items(get_default_schemas(),
                                            worksheet_info['items'])
    except UsageError, e:
        interpreted_items = {'items': []}
        worksheet_info['error'] = str(e)
Exemple #10
0
        def worksheet(self, uuid, interpreted=False):
            try:
                worksheet_info  = self.client.get_worksheet_info(
                                            uuid,
                                            True,  #fetch_items
                                            True,  # get_permissions
                                )
            except PermissionError:
                raise UsageError # forces a not found
            worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info)
            # set permissions
            worksheet_info['edit_permission'] = False
            if worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL:
                worksheet_info['edit_permission'] = True

            worksheet_info['permission_str'] = permission_str(worksheet_info['permission'])
            # format each groups as well
            for group_permission in worksheet_info['group_permissions']:
                group_permission['permission_str'] = permission_str(group_permission['permission'])


            if interpreted:
                interpreted_items = worksheet_util.interpret_items(
                                    worksheet_util.get_default_schemas(),
                                    worksheet_info['items']
                                )
                worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items'])
                # Currently, only certain fields are base64 encoded.
                for item in worksheet_info['items']:
                    if item['mode'] in ['html', 'contents']:
                        # item['name'] in ['stdout', 'stderr']
                        item['interpreted'] = map(base64.b64decode, item['interpreted'])
                    elif 'bundle_info' in item:
                        for bundle_info in item['bundle_info']:
                            try:
                                ## sometimes bundle_info is a string. when item['mode'] is image
                                if isinstance(bundle_info, dict) and bundle_info.get('bundle_type', None) == 'run':
                                    if 'stdout' in bundle_info.keys():
                                        bundle_info['stdout'] = base64.b64decode(bundle_info['stdout'])
                                    if 'stderr' in bundle_info.keys():
                                        bundle_info['stderr'] = base64.b64decode(bundle_info['stderr'])
                            except Exception, e:
                                print e
                                import ipdb; ipdb.set_trace()


                return worksheet_info
Exemple #11
0
        def get_bundle_info(self, uuid):
            bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))

            # Set permissions
            bundle_info['edit_permission'] = (bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
            # Format permissions into strings
            bundle_info['permission_str'] = permission_str(bundle_info['permission'])
            for group_permission in bundle_info['group_permissions']:
                group_permission['permission_str'] = permission_str(group_permission['permission'])

            metadata = bundle_info['metadata']

            cls = get_bundle_subclass(bundle_info['bundle_type'])
            for key, value in worksheet_util.get_formatted_metadata(cls, metadata):
                metadata[key] = value

            bundle_info['metadata'] = metadata
            bundle_info['editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields(cls, metadata)

            return bundle_info
Exemple #12
0
        def worksheet(self, uuid, interpreted=False):
            try:
                worksheet_info  = self.client.get_worksheet_info(
                                            uuid,
                                            True,  #fetch_items
                                            True,  # get_permissions
                                )
            except PermissionError:
                raise UsageError # forces a not found
            worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info)
            # set permissions
            worksheet_info['edit_permission'] = False
            if worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL:
                worksheet_info['edit_permission'] = True

            worksheet_info['permission_str'] = permission_str(worksheet_info['permission'])
            # format each groups as well
            for group_permission in worksheet_info['group_permissions']:
                group_permission['permission_str'] = permission_str(group_permission['permission'])


            if interpreted:
                interpreted_items = worksheet_util.interpret_items(
                                    worksheet_util.get_default_schemas(),
                                    worksheet_info['items']
                                )
                worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items'])
                # Currently, only certain fields are base64 encoded.
                import base64
                for item in worksheet_info['items']:
                    if item['mode'] in ['html', 'contents']:
                        item['interpreted'] = map(base64.b64decode, item['interpreted'])

                return worksheet_info
            else:
                return worksheet_info
Exemple #13
0
        def worksheet(self, uuid, fetch_items, get_permissions, interpreted):
            """
            Return information about a worksheet. Calls
            - get_worksheet_info: get basic info
            - resolve_interpreted_items: get more information about a worksheet.
            In the future, for large worksheets, might want to break this up so
            that we can render something basic.
            """
            worksheet_info = self.client.get_worksheet_info(
                uuid, fetch_items, get_permissions)

            if fetch_items:
                worksheet_info['raw'] = worksheet_util.get_worksheet_lines(
                    worksheet_info)

            # Set permissions
            worksheet_info['edit_permission'] = (
                worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
            # Format permissions into strings
            worksheet_info['permission_str'] = permission_str(
                worksheet_info['permission'])
            for group_permission in worksheet_info['group_permissions']:
                group_permission['permission_str'] = permission_str(
                    group_permission['permission'])

            # Go and fetch more information about the worksheet contents by
            # resolving the interpreted items.
            if interpreted:
                try:
                    interpreted_items = worksheet_util.interpret_items(
                        worksheet_util.get_default_schemas(),
                        worksheet_info['items'])
                except UsageError, e:
                    interpreted_items = {'items': []}
                    worksheet_info['error'] = str(e)

                worksheet_info[
                    'items'] = self.client.resolve_interpreted_items(
                        interpreted_items['items'])
                # Currently, only certain fields are base64 encoded.
                for item in worksheet_info['items']:
                    if item['mode'] in ['html', 'contents']:
                        if item['interpreted'] is None:
                            item['interpreted'] = [
                                formatting.contents_str(item['interpreted'])
                            ]
                        else:
                            item['interpreted'] = map(base64.b64decode,
                                                      item['interpreted'])
                    elif item['mode'] == 'table':
                        for row_map in item['interpreted'][1]:
                            for k, v in row_map.iteritems():
                                if v is None:
                                    row_map[k] = formatting.contents_str(v)
                    elif 'bundle_info' in item:
                        infos = []
                        if isinstance(item['bundle_info'], list):
                            infos = item['bundle_info']
                        elif isinstance(item['bundle_info'], dict):
                            infos = [item['bundle_info']]
                        for bundle_info in infos:
                            try:
                                if isinstance(bundle_info, dict):
                                    worksheet_util.format_metadata(
                                        bundle_info.get('metadata'))
                            except Exception, e:
                                print e
                                import ipdb
                                ipdb.set_trace()
def interpret_genpath(bundle_info, genpath):
    """
    This function is called in the first server call to a BundleClient to
    quickly interpret the genpaths (generalized path) that only require looking
    bundle_info (e.g., 'time', 'command').  The interpretation of generalized
    paths that require reading files is done by interpret_file_genpath.
    """
    # If genpath is referring to a file, then just returns instructions for
    # fetching that file rather than actually doing it.
    if is_file_genpath(genpath):
        return (bundle_info['uuid'], genpath)

    # Render dependencies
    deps = bundle_info.get('dependencies', [])
    anonymous = len(deps) == 1 and deps[0]['child_path'] == ''

    def render_dep(dep, show_key=True, show_uuid=False):
        if show_key and not anonymous:
            if show_uuid or dep['child_path'] != dep['parent_name']:
                a = dep['child_path'] + ':'
            else:
                a = ':'
        else:
            a = ''
        b = dep['parent_uuid'] if show_uuid else (dep['parent_name'] or '')
        c = '/' + dep['parent_path'] if dep['parent_path'] else ''
        return a + b + c

    # Special genpaths (dependencies, args)
    if genpath == 'dependencies':
        return ','.join([render_dep(dep) for dep in deps])
    elif genpath.startswith('dependencies/'):
        # Look up the particular dependency
        _, name = genpath.split('/', 1)
        for dep in deps:
            if dep['child_path'] == name:
                return render_dep(dep, show_key=False)
        return formatting.verbose_contents_str(None)
    elif genpath == 'args':
        # Arguments that we would pass to 'cl'
        args = []
        bundle_type = bundle_info.get('bundle_type')
        if bundle_type not in ('make', 'run'): return None
        args += [bundle_type]
        # Dependencies
        for dep in deps:
            args.append(render_dep(dep, show_uuid=True))
        # Command
        if bundle_info['command']:
            args.append(formatting.quote(bundle_info['command']))
        # Add request arguments from metadata
        metadata = bundle_info['metadata']
        for key, value in metadata.items():
            if key.startswith('request_') and value:
                key = key.replace('_', '-')
                if isinstance(value, bool):
                    args.append('--' + key)
                else:
                    args.extend(['--' + key, formatting.quote(str(value))])
        return ' '.join(args)
    elif genpath == 'summary':
        def friendly_render_dep(dep):
            key = dep['child_path'] or dep['parent_name']
            value = key + '{' + (dep['parent_name'] + ':' if key != dep['parent_name'] else '') + \
                dep['parent_uuid'][0:4] + '}'
            return key, value
        # Nice easy-to-ready description of how this bundle got created.
        bundle_type = bundle_info.get('bundle_type')
        if bundle_type in ('dataset', 'program'):
            return '[uploaded]'
        if bundle_type == 'make':
            args = []
            for dep in deps:
                args.append(friendly_render_dep(dep)[1])
            return '= ' + ' '.join(args)
        elif bundle_type == 'run':
            command = bundle_info['command']
            for dep in deps:
                key, value = friendly_render_dep(dep)
                command = command.replace(key, value)
            return '! ' + command
    elif genpath == 'host_worksheets':
        if 'host_worksheets' in bundle_info:
            return ' '.join('%s(%s)' % (info['name'], info['uuid']) for info in bundle_info['host_worksheets'])
    elif genpath == 'permission':
        if 'permission' in bundle_info:
            return permission_str(bundle_info['permission'])
    elif genpath == 'group_permissions':
        if 'group_permissions' in bundle_info:
            return group_permissions_str(bundle_info['group_permissions'])

    # Bundle field?
    value = bundle_info.get(genpath)
    if value is not None: return value

    # Metadata field?
    value = bundle_info.get('metadata', {}).get(genpath)
    if value is not None: return value

    return None
Exemple #15
0
def fetch_interpreted_worksheet(uuid):
    """
    Return information about a worksheet. Calls
    - get_worksheet_info: get basic info
    - resolve_interpreted_items: get more information about a worksheet.
    In the future, for large worksheets, might want to break this up so
    that we can render something basic.
    Return:
        worksheet_info dict{}:
            key:[value_type] <description>
            blocks:[list]
                    Resolved worksheet blocks from raw_items.
                        Bundles will be grouped into table block items,
                        text items might be grouped into one markdown block etc.
            source:[list] source lines
            raw_to_block:[list]
                            Raw_items to its block index pair.
                                For example, assume the first resolved block item is a bundle table that has 2 rows,
                                then the 2nd element in the list would be [0, 1]
                                [0, 1]: 0 means the item belongs to the first block,
                                        1 means the item is the second item of the block (2nd bundle in our example)
                                NOTE: Used for setting focus on frontend
            block_to_raw:[dict]
                            Maps the blocks (table, markdown, records) to their corresponding source line indices,
                            it's mostly a reverse mapping of raw_to_block, by mostly: raw_to_block has some bug,
                            please refer to worksheet_utils flush_bundles function.
                            This can be used to index the source on the frontend
                            Example:
                            [0, 0]: 0
                            [0, 1]: 1
                            [1, 0]: 9
                            This means the first blocks' first item corresponds to the first line in source,
                            the second item corresponds to the second line in source
                            The second block corresponds the 10th line in source.
                            2-8 can be skipped for multiple reasons: blank lines, comments, schema lines etc.
                                NOTE: Used for setting focus on frontend
    This endpoint can be called with &brief=1 in order to give an abbreviated version,
    which does not resolve searches or wsearches.
    To return an interpreted worksheet that only resolves a particular search/wsearch,
    pass in the search query to the "directive" argument. The value for this argument
    must be a search/wsearch query -- for example, &directive=search 0x .limit=100
    """
    bundle_uuids = request.query.getall('bundle_uuid')
    brief = request.query.get("brief", "0") == "1"

    directive = request.query.get("directive", None)
    search_results = []

    worksheet_info = get_worksheet_info(uuid,
                                        fetch_items=True,
                                        fetch_permissions=True)

    # Shim in additional data for the frontend
    worksheet_info['items'] = resolve_items_into_infos(worksheet_info['items'])

    if worksheet_info['owner_id'] is None:
        worksheet_info['owner_name'] = None
    else:
        owner = local.model.get_user(user_id=worksheet_info['owner_id'])
        worksheet_info['owner_name'] = owner.user_name

    # Fetch items.
    worksheet_info['source'] = get_worksheet_lines(worksheet_info)

    if not directive and not brief:
        expanded_items = []
        for index, raw_item in enumerate(worksheet_info['items']):
            expanded = expand_search_item(raw_item)
            expanded_items.append(expanded)
            # Multiple items can correspond to the same source line (i.e: search directives)
            # raw_items_to_source_index.extend([index] * len(expanded))
        worksheet_info['items'] = list(chain.from_iterable(expanded_items))
    elif directive:
        # Only expand the search item corresponding to the given directive.
        # Used in async loading to only load a single table.
        items_to_show = []
        for i, item in enumerate(worksheet_info['items']):
            (bundle_info, subworksheet_info, value_obj, item_type, id,
             sort_key) = item
            if directive == formatting.tokens_to_string(value_obj):
                search_results = perform_search_query(value_obj)
                items_to_show.append(item)
                break
            elif item_type == TYPE_DIRECTIVE:
                # We need to include previous directives
                # so that the final search result can be properly
                # rendered (it may depend on a schema defined earlier
                # in the worksheet).
                items_to_show.append(item)
        # Make sure the search item is at the end of worksheet_info['items'],
        # so we can isolate it later after interpret_items is called.
        worksheet_info['items'] = items_to_show
        worksheet_info['items'].extend(search_results)

    # Set permissions
    worksheet_info['edit_permission'] = worksheet_info[
        'permission'] == GROUP_OBJECT_PERMISSION_ALL
    # Check enable chat box
    worksheet_info['enable_chat'] = local.config.get('enable_chat', False)
    # Format permissions into strings
    worksheet_info['permission_spec'] = permission_str(
        worksheet_info['permission'])
    for group_permission in worksheet_info['group_permissions']:
        group_permission['permission_spec'] = permission_str(
            group_permission['permission'])

    # Go and fetch more information about the worksheet contents by
    # resolving the interpreted items.
    try:
        interpreted_blocks = interpret_items(get_default_schemas(),
                                             worksheet_info['items'],
                                             db_model=local.model)
    except UsageError as e:
        interpreted_blocks = {'blocks': []}
        worksheet_info['error'] = str(e)

    # bundle_uuids is an optional argument that, if exists, contain the uuids of all the unfinished run bundles that need updating
    # In this case, full_worksheet will return a list of item parallel to ws.info.items that contain only items that need updating.
    # More specifically, all blocks that don't contain run bundles that need updating are None.
    # Also, a non-None block could contain a list of bundle_infos, which represent a list of bundles. Usually not all of them need updating.
    # The bundle_infos for bundles that don't need updating are also None.
    if bundle_uuids:
        for i, block in enumerate(interpreted_blocks['blocks']):
            if 'bundle_info' not in block:
                interpreted_blocks['blocks'][i] = None
            else:
                if isinstance(block['bundle_info'], dict):
                    block['bundle_info'] = [block['bundle_info']]
                is_relevant_block = False
                for j, bundle in enumerate(block['bundle_info']):
                    if bundle['uuid'] in bundle_uuids:
                        is_relevant_block = True
                    else:
                        block['bundle_info'][j] = None
                if not is_relevant_block:
                    interpreted_blocks['blocks'][i] = None
    # Grouped individual items into blocks
    worksheet_info['blocks'] = resolve_interpreted_blocks(
        interpreted_blocks['blocks'], brief=brief)
    worksheet_info['raw_to_block'] = interpreted_blocks['raw_to_block']
    worksheet_info['block_to_raw'] = interpreted_blocks['block_to_raw']

    if directive:
        # If we're only async loading a single table_block / subworksheets_block,
        # return only that block (which is at the end of worksheet_info['items'])
        worksheet_info['blocks'] = [worksheet_info['blocks'][-1]
                                    ] if len(search_results) else []

    for block in worksheet_info['blocks']:
        if block is None:
            continue
        if block['mode'] == 'table':
            for row_map in block['rows']:
                for k, v in row_map.items():
                    if v is None:
                        row_map[k] = formatting.contents_str(v)
        if 'bundle_info' in block:
            infos = []
            if isinstance(block['bundle_info'], list):
                infos = block['bundle_info']
            elif isinstance(block['bundle_info'], dict):
                infos = [block['bundle_info']]
            for bundle_info in infos:
                if bundle_info is None:
                    continue
                if 'bundle_type' not in bundle_info:
                    continue  # empty info: invalid bundle reference
                if isinstance(bundle_info, dict):
                    format_metadata(bundle_info.get('metadata'))
    # Frontend doesn't use individual 'items' for now
    del worksheet_info['items']
    if bundle_uuids:
        return {'blocks': worksheet_info['blocks']}
    return worksheet_info
Exemple #16
0
 def _serialize(self, value, attr, obj):
     try:
         return permission_str(value)
     except UsageError as e:
         raise ValidationError(e.message)
def interpret_genpath(bundle_info, genpath):
    '''
    This function is called in the first server call to a BundleClient to
    quickly interpret the genpaths (generalized path) that only require looking
    bundle_info (e.g., 'time', 'command').  The interpretation of generalized
    paths that require reading files is done by interpret_file_genpath.
    '''
    # If genpath is referring to a file, then just returns instructions for
    # fetching that file rather than actually doing it.
    if is_file_genpath(genpath):
        return (bundle_info['uuid'], genpath)

    # Render dependencies
    deps = bundle_info.get('dependencies', [])
    anonymous = len(deps) == 1 and deps[0]['child_path'] == ''
    def render_dep(dep, show_key=True, show_uuid=False):
        if show_key and not anonymous:
            if show_uuid or dep['child_path'] != dep['parent_name']:
                a = dep['child_path'] + ':'
            else:
                a = ':'
        else:
            a = ''
        b = dep['parent_uuid'] if show_uuid else (dep['parent_name'] or '')
        c = '/' + dep['parent_path'] if dep['parent_path'] else ''
        return a + b + c

    # Special genpaths (dependencies, args)
    if genpath == 'dependencies':
        return ','.join([render_dep(dep) for dep in deps])
    elif genpath.startswith('dependencies/'):
        # Look up the particular dependency
        _, name = genpath.split('/', 1)
        for dep in deps:
            if dep['child_path'] == name:
                return render_dep(dep, show_key=False)
        return 'n/a'
    elif genpath == 'args':
        # Arguments that we would pass to 'cl'
        args = []
        bundle_type = bundle_info.get('bundle_type')
        if bundle_type not in ('make', 'run'): return None
        args += [bundle_type]
        for dep in deps:
            args.append(render_dep(dep, show_uuid=True))
        if bundle_info['command']:
            args.append(quote(bundle_info['command']))
        return ' '.join(args)
    elif genpath == 'host_worksheets':
        if 'host_worksheets' in bundle_info:
            return ' '.join('%s(%s)' % (info['name'], info['uuid']) for info in bundle_info['host_worksheets'])
    elif genpath == 'permission':
        if 'permission' in bundle_info:
            return permission_str(bundle_info['permission'])
    elif genpath == 'group_permissions':
        if 'group_permissions' in bundle_info:
            return group_permissions_str(bundle_info['group_permissions'])

    # Bundle field?
    value = bundle_info.get(genpath)
    if value != None: return value

    # Metadata field?
    value = bundle_info.get('metadata', {}).get(genpath)
    if value != None: return value

    return None
Exemple #18
0
def interpret_genpath(bundle_info, genpath):
    """
    Quickly interpret the genpaths (generalized path) that only require looking
    bundle_info (e.g., 'time', 'command').  The interpretation of generalized
    paths that require reading files is done by interpret_file_genpath.
    """
    # If genpath is referring to a file, then just returns instructions for
    # fetching that file rather than actually doing it.
    if is_file_genpath(genpath):
        return (bundle_info['uuid'], genpath)

    # Render dependencies
    deps = bundle_info.get('dependencies', [])
    anonymous = len(deps) == 1 and deps[0]['child_path'] == ''

    def render_dep(dep, show_key=True, show_uuid=False):
        if show_key and not anonymous:
            if show_uuid or dep['child_path'] != dep['parent_name']:
                a = dep['child_path'] + ':'
            else:
                a = ':'
        else:
            a = ''
        b = dep['parent_uuid'] if show_uuid else (dep['parent_name'] or '')
        c = '/' + dep['parent_path'] if dep['parent_path'] else ''
        return a + b + c

    # Special genpaths (dependencies, args)
    if genpath == 'dependencies':
        return ','.join([render_dep(dep) for dep in deps])
    elif genpath.startswith('dependencies/'):
        # Look up the particular dependency
        _, name = genpath.split('/', 1)
        for dep in deps:
            if dep['child_path'] == name:
                return render_dep(dep, show_key=False)
        return formatting.verbose_contents_str(None)
    elif genpath == 'args':
        # Arguments that we would pass to 'cl'
        args = []
        bundle_type = bundle_info.get('bundle_type')
        if bundle_type not in ('make', 'run'): return None
        args += [bundle_type]
        # Dependencies
        for dep in deps:
            args.append(render_dep(dep, show_uuid=True))
        # Command
        if bundle_info['command']:
            args.append(formatting.quote(bundle_info['command']))
        # Add request arguments from metadata
        metadata = bundle_info['metadata']
        for key, value in metadata.items():
            if key.startswith('request_') and value:
                key = key.replace('_', '-')
                if isinstance(value, bool):
                    args.append('--' + key)
                else:
                    args.extend(['--' + key, formatting.quote(str(value))])
        return ' '.join(args)
    elif genpath == 'summary':
        def friendly_render_dep(dep):
            key = dep['child_path'] or dep['parent_name']
            friendly_parent_name = formatting.verbose_contents_str(dep['parent_name'])
            value = key + '{' + (friendly_parent_name + ':' if key != dep['parent_name'] else '') + \
                dep['parent_uuid'][0:4] + '}'
            return key, value
        # Nice easy-to-ready description of how this bundle got created.
        bundle_type = bundle_info.get('bundle_type')
        if bundle_type in ('dataset', 'program'):
            return '[uploaded]'
        if bundle_type == 'make':
            args = []
            for dep in deps:
                args.append(friendly_render_dep(dep)[1])
            return '= ' + ' '.join(args)
        elif bundle_type == 'run':
            command = bundle_info['command']
            for dep in deps:
                key, value = friendly_render_dep(dep)
                # Replace full-word occurrences of key in the command with an indicator of the dependency.
                # Of course, a string match in the command isn't necessary a semantic reference to the dependency,
                # and there are some dependencies which are not explicit in the command.
                # But this can be seen as a best-effort attempt.
                command = re.sub(r'\b%s\b' % key, value, command)
            return '! ' + command
    elif genpath == 'host_worksheets':
        if 'host_worksheets' in bundle_info:
            return ' '.join('%s(%s)' % (info['name'], info['uuid']) for info in bundle_info['host_worksheets'])
    elif genpath == 'permission':
        if 'permission' in bundle_info:
            return permission_str(bundle_info['permission'])
    elif genpath == 'group_permissions':
        if 'group_permissions' in bundle_info:
            # FIXME(sckoo): we will be passing the old permissions format into this
            # which has been updated to accommodate the new formatting
            return group_permissions_str(bundle_info['group_permissions'])

    # Bundle field?
    value = bundle_info.get(genpath)
    if value is not None: return value

    # Metadata field?
    value = bundle_info.get('metadata', {}).get(genpath)
    if value is not None: return value

    return None
 def _bundle_inherit_workheet_permissions(self, bundle_uuid, worksheet_uuid):
     group_permissions = self.model.get_group_worksheet_permissions(self._current_user_id(), worksheet_uuid)
     for permissions in group_permissions:
         self.set_bundles_perm([bundle_uuid], permissions['group_uuid'], permission_str(permissions['permission']))
def interpret_genpath(bundle_info, genpath):
    '''
    This function is called in the first server call to a BundleClient to
    quickly interpret the genpaths (generalized path) that only require looking
    bundle_info (e.g., 'time', 'command').  The interpretation of generalized
    paths that require reading files is done by interpret_file_genpath.
    '''
    # If genpath is referring to a file, then just returns instructions for
    # fetching that file rather than actually doing it.
    if is_file_genpath(genpath):
        return (bundle_info['uuid'], genpath)

    # Render dependencies
    deps = bundle_info.get('dependencies', [])
    anonymous = len(deps) == 1 and deps[0]['child_path'] == ''

    def render_dep(dep, show_key=True, show_uuid=False):
        if show_key and not anonymous:
            if show_uuid or dep['child_path'] != dep['parent_name']:
                a = dep['child_path'] + ':'
            else:
                a = ':'
        else:
            a = ''
        b = dep['parent_uuid'] if show_uuid else (dep['parent_name'] or '')
        c = '/' + dep['parent_path'] if dep['parent_path'] else ''
        return a + b + c

    # Special genpaths (dependencies, args)
    if genpath == 'dependencies':
        return ','.join([render_dep(dep) for dep in deps])
    elif genpath.startswith('dependencies/'):
        # Look up the particular dependency
        _, name = genpath.split('/', 1)
        for dep in deps:
            if dep['child_path'] == name:
                return render_dep(dep, show_key=False)
        return 'n/a'
    elif genpath == 'args':
        # Arguments that we would pass to 'cl'
        args = []
        bundle_type = bundle_info.get('bundle_type')
        if bundle_type not in ('make', 'run'): return None
        args += [bundle_type]
        for dep in deps:
            args.append(render_dep(dep, show_uuid=True))
        if bundle_info['command']:
            args.append(quote(bundle_info['command']))
        return ' '.join(args)
    elif genpath == 'host_worksheets':
        if 'host_worksheets' in bundle_info:
            return ' '.join('%s(%s)' % (info['name'], info['uuid'])
                            for info in bundle_info['host_worksheets'])
    elif genpath == 'permission':
        if 'permission' in bundle_info:
            return permission_str(bundle_info['permission'])
    elif genpath == 'group_permissions':
        if 'group_permissions' in bundle_info:
            return group_permissions_str(bundle_info['group_permissions'])

    # Bundle field?
    value = bundle_info.get(genpath)
    if value != None: return value

    # Metadata field?
    value = bundle_info.get('metadata', {}).get(genpath)
    if value != None: return value

    return None
Exemple #21
0
def fetch_interpreted_worksheet(uuid):
    """
    Return information about a worksheet. Calls
    - get_worksheet_info: get basic info
    - resolve_interpreted_items: get more information about a worksheet.
    In the future, for large worksheets, might want to break this up so
    that we can render something basic.
    """
    bundle_uuids = request.query.getall('bundle_uuid')
    worksheet_info = get_worksheet_info(uuid, fetch_items=True, fetch_permissions=True)

    # Shim in additional data for the frontend
    worksheet_info['items'] = resolve_items_into_infos(worksheet_info['items'])
    if worksheet_info['owner_id'] is None:
        worksheet_info['owner_name'] = None
    else:
        owner = local.model.get_user(user_id=worksheet_info['owner_id'])
        worksheet_info['owner_name'] = owner.user_name

    # Fetch items.
    worksheet_info['raw'] = get_worksheet_lines(worksheet_info)

    # Replace searches with raw items.
    # This needs to be done before get_worksheet_lines because this replaces
    # user-written raw items.
    worksheet_info['items'] = expand_raw_items(worksheet_info['items'])

    # Set permissions
    worksheet_info['edit_permission'] = worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL
    # Check enable chat box
    worksheet_info['enable_chat'] = local.config.get('enable_chat', False)
    # Format permissions into strings
    worksheet_info['permission_spec'] = permission_str(worksheet_info['permission'])
    for group_permission in worksheet_info['group_permissions']:
        group_permission['permission_spec'] = permission_str(group_permission['permission'])

    # Go and fetch more information about the worksheet contents by
    # resolving the interpreted items.
    try:
        interpreted_blocks = interpret_items(
            get_default_schemas(), worksheet_info['items'], db_model=local.model
        )
    except UsageError as e:
        interpreted_blocks = {'blocks': []}
        worksheet_info['error'] = str(e)

    # bundle_uuids is an optional argument that, if exists, contain the uuids of all the unfinished run bundles that need updating
    # In this case, full_worksheet will return a list of item parallel to ws.info.items that contain only items that need updating.
    # More specifically, all blocks that don't contain run bundles that need updating are None.
    # Also, a non-None block could contain a list of bundle_infos, which represent a list of bundles. Usually not all of them need updating.
    # The bundle_infos for bundles that don't need updating are also None.
    if bundle_uuids:
        for i, block in enumerate(interpreted_blocks['blocks']):
            if 'bundle_info' not in block:
                interpreted_blocks['blocks'][i] = None
            else:
                if isinstance(block['bundle_info'], dict):
                    block['bundle_info'] = [block['bundle_info']]
                is_relevant_block = False
                for j, bundle in enumerate(block['bundle_info']):
                    if bundle['uuid'] in bundle_uuids:
                        is_relevant_block = True
                    else:
                        block['bundle_info'][j] = None
                if not is_relevant_block:
                    interpreted_blocks['blocks'][i] = None

    worksheet_info['items'] = resolve_interpreted_blocks(interpreted_blocks['blocks'])
    worksheet_info['raw_to_block'] = interpreted_blocks['raw_to_block']
    worksheet_info['block_to_raw'] = interpreted_blocks['block_to_raw']

    for item in worksheet_info['items']:
        if item is None:
            continue
        if item['mode'] == 'table':
            for row_map in item['rows']:
                for k, v in row_map.items():
                    if v is None:
                        row_map[k] = formatting.contents_str(v)
        if 'bundle_info' in item:
            infos = []
            if isinstance(item['bundle_info'], list):
                infos = item['bundle_info']
            elif isinstance(item['bundle_info'], dict):
                infos = [item['bundle_info']]
            for bundle_info in infos:
                if bundle_info is None:
                    continue
                if 'bundle_type' not in bundle_info:
                    continue  # empty info: invalid bundle reference
                if isinstance(bundle_info, dict):
                    format_metadata(bundle_info.get('metadata'))
    if bundle_uuids:
        return {'items': worksheet_info['items']}
    return worksheet_info
Exemple #22
0
def fetch_interpreted_worksheet(uuid):
    """
    Return information about a worksheet. Calls
    - get_worksheet_info: get basic info
    - resolve_interpreted_items: get more information about a worksheet.
    In the future, for large worksheets, might want to break this up so
    that we can render something basic.
    """
    bundle_uuids = request.query.getall('bundle_uuid')
    worksheet_info = get_worksheet_info(uuid, fetch_items=True, fetch_permissions=True)

    # Shim in additional data for the frontend
    worksheet_info['items'] = resolve_items_into_infos(worksheet_info['items'])
    if worksheet_info['owner_id'] is None:
        worksheet_info['owner_name'] = None
    else:
        owner = local.model.get_user(user_id=worksheet_info['owner_id'])
        worksheet_info['owner_name'] = owner.user_name

    # Fetch items.
    worksheet_info['raw'] = get_worksheet_lines(worksheet_info)

    # Replace searches with raw items.
    # This needs to be done before get_worksheet_lines because this replaces
    # user-written raw items.
    worksheet_info['items'] = expand_raw_items(worksheet_info['items'])

    # Set permissions
    worksheet_info['edit_permission'] = worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL
    # Check enable chat box
    worksheet_info['enable_chat'] = local.config.get('enable_chat', False)
    # Format permissions into strings
    worksheet_info['permission_spec'] = permission_str(worksheet_info['permission'])
    for group_permission in worksheet_info['group_permissions']:
        group_permission['permission_spec'] = permission_str(group_permission['permission'])

    # Go and fetch more information about the worksheet contents by
    # resolving the interpreted items.
    try:
        interpreted_blocks = interpret_items(get_default_schemas(), worksheet_info['items'])
    except UsageError as e:
        interpreted_blocks = {'blocks': []}
        worksheet_info['error'] = str(e)

    # bundle_uuids is an optional argument that, if exists, contain the uuids of all the unfinished run bundles that need updating
    # In this case, full_worksheet will return a list of item parallel to ws.info.items that contain only items that need updating.
    # More specifically, all blocks that don't contain run bundles that need updating are None.
    # Also, a non-None block could contain a list of bundle_infos, which represent a list of bundles. Usually not all of them need updating.
    # The bundle_infos for bundles that don't need updating are also None.
    if bundle_uuids:
        for i, block in enumerate(interpreted_blocks['blocks']):
            if 'bundle_info' not in block:
                interpreted_blocks['blocks'][i] = None
            else:
                if isinstance(block['bundle_info'], dict):
                    block['bundle_info'] = [block['bundle_info']]
                is_relevant_block = False
                for j, bundle in enumerate(block['bundle_info']):
                    if bundle['uuid'] in bundle_uuids:
                        is_relevant_block = True
                    else:
                        block['bundle_info'][j] = None
                if not is_relevant_block:
                    interpreted_blocks['blocks'][i] = None

    worksheet_info['items'] = resolve_interpreted_blocks(interpreted_blocks['blocks'])
    worksheet_info['raw_to_block'] = interpreted_blocks['raw_to_block']
    worksheet_info['block_to_raw'] = interpreted_blocks['block_to_raw']

    for item in worksheet_info['items']:
        if item is None:
            continue
        if item['mode'] == 'table':
            for row_map in item['rows']:
                for k, v in row_map.iteritems():
                    if v is None:
                        row_map[k] = formatting.contents_str(v)
        if 'bundle_info' in item:
            infos = []
            if isinstance(item['bundle_info'], list):
                infos = item['bundle_info']
            elif isinstance(item['bundle_info'], dict):
                infos = [item['bundle_info']]
            for bundle_info in infos:
                if bundle_info is None:
                    continue
                if 'bundle_type' not in bundle_info:
                    continue  # empty info: invalid bundle reference
                if isinstance(bundle_info, dict):
                    format_metadata(bundle_info.get('metadata'))
    if bundle_uuids:
        return {'items': worksheet_info['items']}
    return worksheet_info
Exemple #23
0
 def _serialize(self, value, attr, obj):
     try:
         return permission_str(value)
     except UsageError as e:
         raise ValidationError(e.message)
def interpret_genpath(bundle_info, genpath, db_model=None, owner_cache=None):
    """
    Quickly interpret the genpaths (generalized path) that only require looking
    bundle_info (e.g., 'time', 'command').  The interpretation of generalized
    paths that require reading files is done by interpret_file_genpath.
    If genpath is referring to a file, then just returns instructions for fetching that file rather than actually doing it.
    :param bundle_info: dictionary which contains metadata of current bundle's information, e.g. uuid, bundle_type, owner_id, etc.
    :param genpath: a generalized path, e.g. column names(summary, owner, etc.), args.
    :param db_model (optional): database model which is used to query database
    :param owner_cache (optional): a dictionary stores mappings from owner_id to owner
    :return: the interpretation of genpath
    """

    if is_file_genpath(genpath):
        return (bundle_info['uuid'], genpath)

    # Render dependencies
    deps = bundle_info.get('dependencies', [])
    anonymous = len(deps) == 1 and deps[0]['child_path'] == ''

    def render_dep(dep, show_key=True, show_uuid=False):
        if show_key and not anonymous:
            if show_uuid or dep['child_path'] != dep['parent_name']:
                a = dep['child_path'] + ':'
            else:
                a = ':'
        else:
            a = ''
        b = dep['parent_uuid'] if show_uuid else (dep['parent_name'] or '')
        c = '/' + dep['parent_path'] if dep['parent_path'] else ''
        return a + b + c

    # Special genpaths (dependencies, args)
    if genpath == 'dependencies':
        return ','.join([render_dep(dep) for dep in deps])
    elif genpath.startswith('dependencies/'):
        # Look up the particular dependency
        _, name = genpath.split('/', 1)
        for dep in deps:
            if dep['child_path'] == name:
                return render_dep(dep, show_key=False)
        return formatting.verbose_contents_str(None)
    elif genpath == 'args':
        # Arguments that we would pass to 'cl'
        args = []
        bundle_type = bundle_info.get('bundle_type')
        if bundle_type not in ('make', 'run'):
            return None
        args += [bundle_type]
        # Dependencies
        for dep in deps:
            args.append(render_dep(dep, show_uuid=True))
        # Command
        if bundle_info['command']:
            args.append(formatting.quote(bundle_info['command']))
        # Add request arguments from metadata
        metadata = bundle_info['metadata']
        for key, value in metadata.items():
            if key.startswith('request_') and value:
                key = key.replace('_', '-')
                if isinstance(value, bool):
                    args.append('--' + key)
                else:
                    args.extend(['--' + key, formatting.quote(str(value))])
        return ' '.join(args)
    elif genpath == 'summary':

        def friendly_render_dep(dep):
            key = dep['child_path'] or dep['parent_name']
            friendly_parent_name = formatting.verbose_contents_str(
                dep['parent_name'])
            value = (key + '{' + (friendly_parent_name +
                                  ':' if key != dep['parent_name'] else '') +
                     dep['parent_uuid'][0:4] + '}')
            return key, value

        # Nice easy-to-ready description of how this bundle got created.
        bundle_type = bundle_info.get('bundle_type')
        if bundle_type in ('dataset', 'program'):
            return '[uploaded]'
        if bundle_type == 'make':
            args = []
            for dep in deps:
                args.append(friendly_render_dep(dep)[1])
            return '= ' + ' '.join(args)
        elif bundle_type == 'run':
            return '! ' + bundle_info['command']
    elif genpath == 'host_worksheets':
        if 'host_worksheets' in bundle_info:
            return ' '.join('%s(%s)' % (info['name'], info['uuid'])
                            for info in bundle_info['host_worksheets'])
    elif genpath == 'permission':
        if 'permission' in bundle_info:
            return permission_str(bundle_info['permission'])
    elif genpath == 'group_permissions':
        if 'group_permissions' in bundle_info:
            # FIXME(sckoo): we will be passing the old permissions format into this
            # which has been updated to accommodate the new formatting
            return group_permissions_str(bundle_info['group_permissions'])
    elif genpath == 'owner':
        if 'owner_id' in bundle_info:
            if owner_cache is not None and bundle_info[
                    'owner_id'] in owner_cache:
                return owner_cache[bundle_info['owner_id']]
            else:
                # We might batch this database operation in the future
                owner = db_model.get_user(user_id=bundle_info['owner_id'])
                owner_cache[bundle_info['owner_id']] = owner.user_name
                return owner.user_name

    # Bundle field?
    value = bundle_info.get(genpath)
    if value is not None:
        return value

    # Metadata field?
    value = bundle_info.get('metadata', {}).get(genpath)
    if value is not None:
        return value

    return None
Exemple #25
0
def interpret_genpath(bundle_info, genpath):
    """
    Quickly interpret the genpaths (generalized path) that only require looking
    bundle_info (e.g., 'time', 'command').  The interpretation of generalized
    paths that require reading files is done by interpret_file_genpath.
    """
    # If genpath is referring to a file, then just returns instructions for
    # fetching that file rather than actually doing it.
    if is_file_genpath(genpath):
        return (bundle_info['uuid'], genpath)

    # Render dependencies
    deps = bundle_info.get('dependencies', [])
    anonymous = len(deps) == 1 and deps[0]['child_path'] == ''

    def render_dep(dep, show_key=True, show_uuid=False):
        if show_key and not anonymous:
            if show_uuid or dep['child_path'] != dep['parent_name']:
                a = dep['child_path'] + ':'
            else:
                a = ':'
        else:
            a = ''
        b = dep['parent_uuid'] if show_uuid else (dep['parent_name'] or '')
        c = '/' + dep['parent_path'] if dep['parent_path'] else ''
        return a + b + c

    # Special genpaths (dependencies, args)
    if genpath == 'dependencies':
        return ','.join([render_dep(dep) for dep in deps])
    elif genpath.startswith('dependencies/'):
        # Look up the particular dependency
        _, name = genpath.split('/', 1)
        for dep in deps:
            if dep['child_path'] == name:
                return render_dep(dep, show_key=False)
        return formatting.verbose_contents_str(None)
    elif genpath == 'args':
        # Arguments that we would pass to 'cl'
        args = []
        bundle_type = bundle_info.get('bundle_type')
        if bundle_type not in ('make', 'run'):
            return None
        args += [bundle_type]
        # Dependencies
        for dep in deps:
            args.append(render_dep(dep, show_uuid=True))
        # Command
        if bundle_info['command']:
            args.append(formatting.quote(bundle_info['command']))
        # Add request arguments from metadata
        metadata = bundle_info['metadata']
        for key, value in metadata.items():
            if key.startswith('request_') and value:
                key = key.replace('_', '-')
                if isinstance(value, bool):
                    args.append('--' + key)
                else:
                    args.extend(['--' + key, formatting.quote(str(value))])
        return ' '.join(args)
    elif genpath == 'summary':

        def friendly_render_dep(dep):
            key = dep['child_path'] or dep['parent_name']
            friendly_parent_name = formatting.verbose_contents_str(dep['parent_name'])
            value = (
                key
                + '{'
                + (friendly_parent_name + ':' if key != dep['parent_name'] else '')
                + dep['parent_uuid'][0:4]
                + '}'
            )
            return key, value

        # Nice easy-to-ready description of how this bundle got created.
        bundle_type = bundle_info.get('bundle_type')
        if bundle_type in ('dataset', 'program'):
            return '[uploaded]'
        if bundle_type == 'make':
            args = []
            for dep in deps:
                args.append(friendly_render_dep(dep)[1])
            return '= ' + ' '.join(args)
        elif bundle_type == 'run':
            command = bundle_info['command']
            for dep in deps:
                key, value = friendly_render_dep(dep)
                # Replace full-word occurrences of key in the command with an indicator of the dependency.
                # Of course, a string match in the command isn't necessary a semantic reference to the dependency,
                # and there are some dependencies which are not explicit in the command.
                # But this can be seen as a best-effort attempt.
                command = re.sub(r'\b%s\b' % key, value, command)
            return '! ' + command
    elif genpath == 'host_worksheets':
        if 'host_worksheets' in bundle_info:
            return ' '.join(
                '%s(%s)' % (info['name'], info['uuid']) for info in bundle_info['host_worksheets']
            )
    elif genpath == 'permission':
        if 'permission' in bundle_info:
            return permission_str(bundle_info['permission'])
    elif genpath == 'group_permissions':
        if 'group_permissions' in bundle_info:
            # FIXME(sckoo): we will be passing the old permissions format into this
            # which has been updated to accommodate the new formatting
            return group_permissions_str(bundle_info['group_permissions'])

    # Bundle field?
    value = bundle_info.get(genpath)
    if value is not None:
        return value

    # Metadata field?
    value = bundle_info.get('metadata', {}).get(genpath)
    if value is not None:
        return value

    return None