Example #1
0
            def get_lines(name):
                lines = self.head_target((uuid, name), self.HEAD_MAX_LINES)
                if lines is not None:
                    import base64
                    lines = ''.join(map(base64.b64decode, lines))

                return formatting.verbose_contents_str(lines)
Example #2
0
 def friendly_render_dep(dep):
     key = dep['child_path'] or dep['parent_name']
     friendly_parent_name = formatting.verbose_contents_str(
         dep['parent_name'])
     value = key + '{' + (friendly_parent_name + ':' if key != dep['parent_name'] else '') + \
         dep['parent_uuid'][0:4] + '}'
     return key, value
Example #3
0
        def get_lines(name):
            lines = self.head_target((uuid, name), self.HEAD_MAX_LINES)
            if lines is not None:
                import base64
                lines = ''.join(map(base64.b64decode, lines))

            return formatting.verbose_contents_str(lines)
 def friendly_render_dep(dep):
     key = dep['child_path'] or dep['parent_name']
     friendly_parent_name = formatting.verbose_contents_str(dep['parent_name'])
     value = "%s{%s%s}" % (
         key,
         friendly_parent_name + ':' if key != dep['parent_name'] else '',
         dep['parent_uuid'][0:4],
     )
     return key, value
Example #5
0
 def friendly_render_dep(dep):
     key = dep['child_path'] or dep['parent_name']
     friendly_parent_name = formatting.verbose_contents_str(dep['parent_name'])
     value = (
         key
         + '{'
         + (friendly_parent_name + ':' if key != dep['parent_name'] else '')
         + dep['parent_uuid'][0:4]
         + '}'
     )
     return key, value
Example #6
0
 def get_summary(info, name):
     if info['type'] == 'file':
         TRUNCATION_TEXT = (
             '\n'
             '... Truncated. Click link above to see full file. ...\n'
             '\n')
         contents = local.download_manager.summarize_file(
             uuid, name,
             num_head_lines=50, num_tail_lines=50, max_line_length=128,
             truncation_text=TRUNCATION_TEXT, gzipped=False)
         return formatting.verbose_contents_str(contents)
     elif info['type'] == 'link':
         return  ' -> ' + info['link']
Example #7
0
 def get_summary(download_manager, info, name):
     if info['type'] == 'file':
         TRUNCATION_TEXT = (
             '\n'
             '... Truncated. Click link above to see full file. ...\n'
             '\n')
         contents = download_manager.summarize_file(
             uuid, name,
             num_head_lines=50, num_tail_lines=50, max_line_length=128,
             truncation_text=TRUNCATION_TEXT, gzipped=False)
         return formatting.verbose_contents_str(contents)
     elif info['type'] == 'link':
         return  ' -> ' + info['link']
def interpret_genpath(bundle_info, genpath, db_model=None, owner_cache=None):
    """
    Quickly interpret the genpaths (generalized path) that only require looking
    bundle_info (e.g., 'time', 'command').  The interpretation of generalized
    paths that require reading files is done by interpret_file_genpath.
    If genpath is referring to a file, then just returns instructions for fetching that file rather than actually doing it.
    :param bundle_info: dictionary which contains metadata of current bundle's information, e.g. uuid, bundle_type, owner_id, etc.
    :param genpath: a generalized path, e.g. column names(summary, owner, etc.), args.
    :param db_model (optional): database model which is used to query database
    :param owner_cache (optional): a dictionary stores mappings from owner_id to owner
    :return: the interpretation of genpath
    """

    if is_file_genpath(genpath):
        return (bundle_info['uuid'], genpath)

    # Render dependencies
    deps = bundle_info.get('dependencies', [])
    anonymous = len(deps) == 1 and deps[0]['child_path'] == ''

    def render_dep(dep, show_key=True, show_uuid=False):
        if show_key and not anonymous:
            if show_uuid or dep['child_path'] != dep['parent_name']:
                a = dep['child_path'] + ':'
            else:
                a = ':'
        else:
            a = ''
        b = dep['parent_uuid'] if show_uuid else (dep['parent_name'] or '')
        c = '/' + dep['parent_path'] if dep['parent_path'] else ''
        return a + b + c

    # Special genpaths (dependencies, args)
    if genpath == 'dependencies':
        return ','.join([render_dep(dep) for dep in deps])
    elif genpath.startswith('dependencies/'):
        # Look up the particular dependency
        _, name = genpath.split('/', 1)
        for dep in deps:
            if dep['child_path'] == name:
                return render_dep(dep, show_key=False)
        return formatting.verbose_contents_str(None)
    elif genpath == 'args':
        # Arguments that we would pass to 'cl'
        args = []
        bundle_type = bundle_info.get('bundle_type')
        if bundle_type not in ('make', 'run'):
            return None
        args += [bundle_type]
        # Dependencies
        for dep in deps:
            args.append(render_dep(dep, show_uuid=True))
        # Command
        if bundle_info['command']:
            args.append(formatting.quote(bundle_info['command']))
        # Add request arguments from metadata
        metadata = bundle_info['metadata']
        for key, value in metadata.items():
            if key.startswith('request_') and value:
                key = key.replace('_', '-')
                if isinstance(value, bool):
                    args.append('--' + key)
                else:
                    args.extend(['--' + key, formatting.quote(str(value))])
        return ' '.join(args)
    elif genpath == 'summary':

        def friendly_render_dep(dep):
            key = dep['child_path'] or dep['parent_name']
            friendly_parent_name = formatting.verbose_contents_str(
                dep['parent_name'])
            value = (key + '{' + (friendly_parent_name +
                                  ':' if key != dep['parent_name'] else '') +
                     dep['parent_uuid'][0:4] + '}')
            return key, value

        # Nice easy-to-ready description of how this bundle got created.
        bundle_type = bundle_info.get('bundle_type')
        if bundle_type in ('dataset', 'program'):
            return '[uploaded]'
        if bundle_type == 'make':
            args = []
            for dep in deps:
                args.append(friendly_render_dep(dep)[1])
            return '= ' + ' '.join(args)
        elif bundle_type == 'run':
            return '! ' + bundle_info['command']
    elif genpath == 'host_worksheets':
        if 'host_worksheets' in bundle_info:
            return ' '.join('%s(%s)' % (info['name'], info['uuid'])
                            for info in bundle_info['host_worksheets'])
    elif genpath == 'permission':
        if 'permission' in bundle_info:
            return permission_str(bundle_info['permission'])
    elif genpath == 'group_permissions':
        if 'group_permissions' in bundle_info:
            # FIXME(sckoo): we will be passing the old permissions format into this
            # which has been updated to accommodate the new formatting
            return group_permissions_str(bundle_info['group_permissions'])
    elif genpath == 'owner':
        if 'owner_id' in bundle_info:
            if owner_cache is not None and bundle_info[
                    'owner_id'] in owner_cache:
                return owner_cache[bundle_info['owner_id']]
            else:
                # We might batch this database operation in the future
                owner = db_model.get_user(user_id=bundle_info['owner_id'])
                owner_cache[bundle_info['owner_id']] = owner.user_name
                return owner.user_name

    # Bundle field?
    value = bundle_info.get(genpath)
    if value is not None:
        return value

    # Metadata field?
    value = bundle_info.get('metadata', {}).get(genpath)
    if value is not None:
        return value

    return None
Example #9
0
def interpret_genpath(bundle_info, genpath):
    """
    This function is called in the first server call to a BundleClient to
    quickly interpret the genpaths (generalized path) that only require looking
    bundle_info (e.g., 'time', 'command').  The interpretation of generalized
    paths that require reading files is done by interpret_file_genpath.
    """
    # If genpath is referring to a file, then just returns instructions for
    # fetching that file rather than actually doing it.
    if is_file_genpath(genpath):
        return (bundle_info['uuid'], genpath)

    # Render dependencies
    deps = bundle_info.get('dependencies', [])
    anonymous = len(deps) == 1 and deps[0]['child_path'] == ''

    def render_dep(dep, show_key=True, show_uuid=False):
        if show_key and not anonymous:
            if show_uuid or dep['child_path'] != dep['parent_name']:
                a = dep['child_path'] + ':'
            else:
                a = ':'
        else:
            a = ''
        b = dep['parent_uuid'] if show_uuid else (dep['parent_name'] or '')
        c = '/' + dep['parent_path'] if dep['parent_path'] else ''
        return a + b + c

    # Special genpaths (dependencies, args)
    if genpath == 'dependencies':
        return ','.join([render_dep(dep) for dep in deps])
    elif genpath.startswith('dependencies/'):
        # Look up the particular dependency
        _, name = genpath.split('/', 1)
        for dep in deps:
            if dep['child_path'] == name:
                return render_dep(dep, show_key=False)
        return formatting.verbose_contents_str(None)
    elif genpath == 'args':
        # Arguments that we would pass to 'cl'
        args = []
        bundle_type = bundle_info.get('bundle_type')
        if bundle_type not in ('make', 'run'): return None
        args += [bundle_type]
        # Dependencies
        for dep in deps:
            args.append(render_dep(dep, show_uuid=True))
        # Command
        if bundle_info['command']:
            args.append(formatting.quote(bundle_info['command']))
        # Add request arguments from metadata
        metadata = bundle_info['metadata']
        for key, value in metadata.items():
            if key.startswith('request_') and value:
                key = key.replace('_', '-')
                if isinstance(value, bool):
                    args.append('--' + key)
                else:
                    args.extend(['--' + key, formatting.quote(str(value))])
        return ' '.join(args)
    elif genpath == 'summary':
        def friendly_render_dep(dep):
            key = dep['child_path'] or dep['parent_name']
            value = key + '{' + (dep['parent_name'] + ':' if key != dep['parent_name'] else '') + \
                dep['parent_uuid'][0:4] + '}'
            return key, value
        # Nice easy-to-ready description of how this bundle got created.
        bundle_type = bundle_info.get('bundle_type')
        if bundle_type in ('dataset', 'program'):
            return '[uploaded]'
        if bundle_type == 'make':
            args = []
            for dep in deps:
                args.append(friendly_render_dep(dep)[1])
            return '= ' + ' '.join(args)
        elif bundle_type == 'run':
            command = bundle_info['command']
            for dep in deps:
                key, value = friendly_render_dep(dep)
                command = command.replace(key, value)
            return '! ' + command
    elif genpath == 'host_worksheets':
        if 'host_worksheets' in bundle_info:
            return ' '.join('%s(%s)' % (info['name'], info['uuid']) for info in bundle_info['host_worksheets'])
    elif genpath == 'permission':
        if 'permission' in bundle_info:
            return permission_str(bundle_info['permission'])
    elif genpath == 'group_permissions':
        if 'group_permissions' in bundle_info:
            return group_permissions_str(bundle_info['group_permissions'])

    # Bundle field?
    value = bundle_info.get(genpath)
    if value is not None: return value

    # Metadata field?
    value = bundle_info.get('metadata', {}).get(genpath)
    if value is not None: return value

    return None
Example #10
0
def interpret_genpath(bundle_info, genpath):
    """
    Quickly interpret the genpaths (generalized path) that only require looking
    bundle_info (e.g., 'time', 'command').  The interpretation of generalized
    paths that require reading files is done by interpret_file_genpath.
    """
    # If genpath is referring to a file, then just returns instructions for
    # fetching that file rather than actually doing it.
    if is_file_genpath(genpath):
        return (bundle_info['uuid'], genpath)

    # Render dependencies
    deps = bundle_info.get('dependencies', [])
    anonymous = len(deps) == 1 and deps[0]['child_path'] == ''

    def render_dep(dep, show_key=True, show_uuid=False):
        if show_key and not anonymous:
            if show_uuid or dep['child_path'] != dep['parent_name']:
                a = dep['child_path'] + ':'
            else:
                a = ':'
        else:
            a = ''
        b = dep['parent_uuid'] if show_uuid else (dep['parent_name'] or '')
        c = '/' + dep['parent_path'] if dep['parent_path'] else ''
        return a + b + c

    # Special genpaths (dependencies, args)
    if genpath == 'dependencies':
        return ','.join([render_dep(dep) for dep in deps])
    elif genpath.startswith('dependencies/'):
        # Look up the particular dependency
        _, name = genpath.split('/', 1)
        for dep in deps:
            if dep['child_path'] == name:
                return render_dep(dep, show_key=False)
        return formatting.verbose_contents_str(None)
    elif genpath == 'args':
        # Arguments that we would pass to 'cl'
        args = []
        bundle_type = bundle_info.get('bundle_type')
        if bundle_type not in ('make', 'run'): return None
        args += [bundle_type]
        # Dependencies
        for dep in deps:
            args.append(render_dep(dep, show_uuid=True))
        # Command
        if bundle_info['command']:
            args.append(formatting.quote(bundle_info['command']))
        # Add request arguments from metadata
        metadata = bundle_info['metadata']
        for key, value in metadata.items():
            if key.startswith('request_') and value:
                key = key.replace('_', '-')
                if isinstance(value, bool):
                    args.append('--' + key)
                else:
                    args.extend(['--' + key, formatting.quote(str(value))])
        return ' '.join(args)
    elif genpath == 'summary':
        def friendly_render_dep(dep):
            key = dep['child_path'] or dep['parent_name']
            friendly_parent_name = formatting.verbose_contents_str(dep['parent_name'])
            value = key + '{' + (friendly_parent_name + ':' if key != dep['parent_name'] else '') + \
                dep['parent_uuid'][0:4] + '}'
            return key, value
        # Nice easy-to-ready description of how this bundle got created.
        bundle_type = bundle_info.get('bundle_type')
        if bundle_type in ('dataset', 'program'):
            return '[uploaded]'
        if bundle_type == 'make':
            args = []
            for dep in deps:
                args.append(friendly_render_dep(dep)[1])
            return '= ' + ' '.join(args)
        elif bundle_type == 'run':
            command = bundle_info['command']
            for dep in deps:
                key, value = friendly_render_dep(dep)
                # Replace full-word occurrences of key in the command with an indicator of the dependency.
                # Of course, a string match in the command isn't necessary a semantic reference to the dependency,
                # and there are some dependencies which are not explicit in the command.
                # But this can be seen as a best-effort attempt.
                command = re.sub(r'\b%s\b' % key, value, command)
            return '! ' + command
    elif genpath == 'host_worksheets':
        if 'host_worksheets' in bundle_info:
            return ' '.join('%s(%s)' % (info['name'], info['uuid']) for info in bundle_info['host_worksheets'])
    elif genpath == 'permission':
        if 'permission' in bundle_info:
            return permission_str(bundle_info['permission'])
    elif genpath == 'group_permissions':
        if 'group_permissions' in bundle_info:
            # FIXME(sckoo): we will be passing the old permissions format into this
            # which has been updated to accommodate the new formatting
            return group_permissions_str(bundle_info['group_permissions'])

    # Bundle field?
    value = bundle_info.get(genpath)
    if value is not None: return value

    # Metadata field?
    value = bundle_info.get('metadata', {}).get(genpath)
    if value is not None: return value

    return None
Example #11
0
def interpret_genpath(bundle_info, genpath):
    """
    Quickly interpret the genpaths (generalized path) that only require looking
    bundle_info (e.g., 'time', 'command').  The interpretation of generalized
    paths that require reading files is done by interpret_file_genpath.
    """
    # If genpath is referring to a file, then just returns instructions for
    # fetching that file rather than actually doing it.
    if is_file_genpath(genpath):
        return (bundle_info['uuid'], genpath)

    # Render dependencies
    deps = bundle_info.get('dependencies', [])
    anonymous = len(deps) == 1 and deps[0]['child_path'] == ''

    def render_dep(dep, show_key=True, show_uuid=False):
        if show_key and not anonymous:
            if show_uuid or dep['child_path'] != dep['parent_name']:
                a = dep['child_path'] + ':'
            else:
                a = ':'
        else:
            a = ''
        b = dep['parent_uuid'] if show_uuid else (dep['parent_name'] or '')
        c = '/' + dep['parent_path'] if dep['parent_path'] else ''
        return a + b + c

    # Special genpaths (dependencies, args)
    if genpath == 'dependencies':
        return ','.join([render_dep(dep) for dep in deps])
    elif genpath.startswith('dependencies/'):
        # Look up the particular dependency
        _, name = genpath.split('/', 1)
        for dep in deps:
            if dep['child_path'] == name:
                return render_dep(dep, show_key=False)
        return formatting.verbose_contents_str(None)
    elif genpath == 'args':
        # Arguments that we would pass to 'cl'
        args = []
        bundle_type = bundle_info.get('bundle_type')
        if bundle_type not in ('make', 'run'):
            return None
        args += [bundle_type]
        # Dependencies
        for dep in deps:
            args.append(render_dep(dep, show_uuid=True))
        # Command
        if bundle_info['command']:
            args.append(formatting.quote(bundle_info['command']))
        # Add request arguments from metadata
        metadata = bundle_info['metadata']
        for key, value in metadata.items():
            if key.startswith('request_') and value:
                key = key.replace('_', '-')
                if isinstance(value, bool):
                    args.append('--' + key)
                else:
                    args.extend(['--' + key, formatting.quote(str(value))])
        return ' '.join(args)
    elif genpath == 'summary':

        def friendly_render_dep(dep):
            key = dep['child_path'] or dep['parent_name']
            friendly_parent_name = formatting.verbose_contents_str(dep['parent_name'])
            value = (
                key
                + '{'
                + (friendly_parent_name + ':' if key != dep['parent_name'] else '')
                + dep['parent_uuid'][0:4]
                + '}'
            )
            return key, value

        # Nice easy-to-ready description of how this bundle got created.
        bundle_type = bundle_info.get('bundle_type')
        if bundle_type in ('dataset', 'program'):
            return '[uploaded]'
        if bundle_type == 'make':
            args = []
            for dep in deps:
                args.append(friendly_render_dep(dep)[1])
            return '= ' + ' '.join(args)
        elif bundle_type == 'run':
            command = bundle_info['command']
            for dep in deps:
                key, value = friendly_render_dep(dep)
                # Replace full-word occurrences of key in the command with an indicator of the dependency.
                # Of course, a string match in the command isn't necessary a semantic reference to the dependency,
                # and there are some dependencies which are not explicit in the command.
                # But this can be seen as a best-effort attempt.
                command = re.sub(r'\b%s\b' % key, value, command)
            return '! ' + command
    elif genpath == 'host_worksheets':
        if 'host_worksheets' in bundle_info:
            return ' '.join(
                '%s(%s)' % (info['name'], info['uuid']) for info in bundle_info['host_worksheets']
            )
    elif genpath == 'permission':
        if 'permission' in bundle_info:
            return permission_str(bundle_info['permission'])
    elif genpath == 'group_permissions':
        if 'group_permissions' in bundle_info:
            # FIXME(sckoo): we will be passing the old permissions format into this
            # which has been updated to accommodate the new formatting
            return group_permissions_str(bundle_info['group_permissions'])

    # Bundle field?
    value = bundle_info.get(genpath)
    if value is not None:
        return value

    # Metadata field?
    value = bundle_info.get('metadata', {}).get(genpath)
    if value is not None:
        return value

    return None