예제 #1
0
파일: bundles.py 프로젝트: prags/codalab
    class BundleService():

        def __init__(self, user=None):
            self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
                                             lambda command: get_user_token(user), verbose=1)

        def items(self):
            return _call_with_retries(lambda: self.client.search())

        def item(self, uuid):
            return _call_with_retries(lambda: self.client.get_bundle_info(uuid))

        def worksheets(self):
            return _call_with_retries(lambda: self.client.list_worksheets())

        def create_worksheet(self, name):
            return _call_with_retries(lambda: self.client.new_worksheet(name))

        def worksheet(self, uuid, interpreted=False):
            worksheet_info  = _call_with_retries(
                    lambda: self.client.get_worksheet_info(
                            uuid,
                            True
                    )
                )
            if interpreted:
                interpreted_items = worksheet_util.interpret_items(
                                    worksheet_util.get_default_schemas(),
                                    worksheet_info['items']
                                )
                worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items'])
                return worksheet_info
            else:
                return worksheet_info

        def ls(self, uuid, path):
            return _call_with_retries(lambda: self.client.ls((uuid, path)))

        MAX_BYTES = 1024*1024
        def read_file(self, uuid, path):
            fid = self.client.open_target((uuid, path))
            try:
                while True:
                    bytes = self.client.read_file(fid, BundleService.MAX_BYTES)
                    yield bytes.data
                    if len(bytes.data) < BundleService.MAX_BYTES:
                        break
            finally:
                self.client.close_file(fid)

        def download_target(self, uuid, return_zip=False):
            target = (uuid, '')
            result_path, container_path = self.client.download_target(target=target, follow_symlinks=True, return_zip=return_zip)
            return (result_path, container_path)

        def http_status_from_exception(self, ex):
            # This is brittle. See https://github.com/codalab/codalab/issues/345.
            if type(ex) == UsageError:
                return 404
            return 500
예제 #2
0
파일: bundles.py 프로젝트: v-bech/codalab
    class BundleService():

        def __init__(self, user=None):
            self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
                                             lambda command: get_user_token(user), verbose=1)

        def items(self):
            return _call_with_retries(lambda: self.client.search())

        def item(self, uuid):
            return _call_with_retries(lambda: self.client.get_bundle_info(uuid))

        def worksheets(self):
            return _call_with_retries(lambda: self.client.list_worksheets())

        def create_worksheet(self, name):
            return _call_with_retries(lambda: self.client.new_worksheet(name))

        def worksheet(self, uuid):
            return _call_with_retries(lambda: self.client.get_worksheet_info(uuid))

        def ls(self, uuid, path):
            return _call_with_retries(lambda: self.client.ls((uuid, path)))

        MAX_BYTES = 1024*1024
        def read_file(self, uuid, path):
            fid = self.client.open_target((uuid, path))
            try:
                while True:
                    bytes = self.client.read_file(fid, BundleService.MAX_BYTES)
                    yield bytes.data
                    if len(bytes.data) < BundleService.MAX_BYTES:
                        break
            finally:
                self.client.close_file(fid)

        def http_status_from_exception(self, ex):
            # This is brittle. See https://github.com/codalab/codalab/issues/345.
            if type(ex) == UsageError:
                return 404
            return 500
예제 #3
0
    class BundleService(object):
        # Maximum number of lines of files to show
        HEAD_MAX_LINES = 100

        def __init__(self, user=None):
            self.client = RemoteBundleClient(
                settings.BUNDLE_SERVICE_URL,
                lambda command: get_user_token(user),
                verbose=1)

        def items(self):
            return _call_with_retries(lambda: self.client.search())

        def get_bundle_info(self, uuid):
            bundle_info = _call_with_retries(
                lambda: self.client.get_bundle_info(uuid, True, True, True))

            if bundle_info is None:
                return None
            # Set permissions
            bundle_info['edit_permission'] = (
                bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
            # Format permissions into strings
            bundle_info['permission_str'] = permission_str(
                bundle_info['permission'])
            for group_permission in bundle_info['group_permissions']:
                group_permission['permission_str'] = permission_str(
                    group_permission['permission'])

            metadata = bundle_info['metadata']

            cls = get_bundle_subclass(bundle_info['bundle_type'])
            for key, value in worksheet_util.get_formatted_metadata(
                    cls, metadata):
                metadata[key] = value

            bundle_info['metadata'] = metadata
            bundle_info[
                'editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields(
                    cls, metadata)

            return bundle_info

        def head_target(self, target, max_num_lines=HEAD_MAX_LINES):
            return self.client.head_target(target, max_num_lines)

        def search_bundles(self, keywords, worksheet_uuid=None):
            bundle_uuids = self.client.search_bundle_uuids(
                worksheet_uuid, keywords)
            bundle_infos = self.client.get_bundle_infos(bundle_uuids)
            return bundle_infos

        def get_worksheet_bundles(self, worksheet_uuid):
            worksheet_info = self.client.get_worksheet_info(
                worksheet_uuid, True, True)
            bundle_info_list = []
            for (bundle_info, subworksheet_info, value_obj,
                 item_type) in worksheet_info['items']:
                if item_type == worksheet_util.TYPE_BUNDLE:
                    bundle_info_list.append(bundle_info)
            return bundle_info_list

        def worksheets(self):
            return _call_with_retries(lambda: self.client.list_worksheets())

        def search_worksheets(self, keywords, worksheet_uuid=None):
            return _call_with_retries(
                lambda: self.client.search_worksheets(keywords))

        def create_worksheet(self, name):
            return _call_with_retries(
                lambda: self.client.new_worksheet(name, None))

        def get_worksheet_uuid(self, spec):
            # generic clean up just in case
            spec = smart_str(spec)

            # generic function sometimes get uuid already just return it.
            if spec_util.UUID_REGEX.match(spec):
                return spec
            else:
                return worksheet_util.get_worksheet_uuid(
                    self.client, None, spec)

        def basic_worksheet(self, uuid):
            return self.worksheet(uuid,
                                  fetch_items=False,
                                  get_permissions=True,
                                  interpreted=False)

        def full_worksheet(self, uuid):
            return self.worksheet(uuid,
                                  fetch_items=True,
                                  get_permissions=True,
                                  interpreted=True)

        def worksheet(self, uuid, fetch_items, get_permissions, interpreted):
            """
            Return information about a worksheet. Calls
            - get_worksheet_info: get basic info
            - resolve_interpreted_items: get more information about a worksheet.
            In the future, for large worksheets, might want to break this up so
            that we can render something basic.
            """
            worksheet_info = self.client.get_worksheet_info(
                uuid, fetch_items, get_permissions)

            if fetch_items:
                worksheet_info['raw'] = worksheet_util.get_worksheet_lines(
                    worksheet_info)

            # Set permissions
            worksheet_info['edit_permission'] = (
                worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
            # Format permissions into strings
            worksheet_info['permission_str'] = permission_str(
                worksheet_info['permission'])
            for group_permission in worksheet_info['group_permissions']:
                group_permission['permission_str'] = permission_str(
                    group_permission['permission'])

            # Go and fetch more information about the worksheet contents by
            # resolving the interpreted items.
            if interpreted:
                try:
                    interpreted_items = worksheet_util.interpret_items(
                        worksheet_util.get_default_schemas(),
                        worksheet_info['items'])
                except UsageError, e:
                    interpreted_items = {'items': []}
                    worksheet_info['error'] = str(e)

                worksheet_info[
                    'items'] = self.client.resolve_interpreted_items(
                        interpreted_items['items'])
                # Currently, only certain fields are base64 encoded.
                for item in worksheet_info['items']:
                    if item['mode'] in ['html', 'contents']:
                        if item['interpreted'] is None:
                            item['interpreted'] = [
                                formatting.contents_str(item['interpreted'])
                            ]
                        else:
                            item['interpreted'] = map(base64.b64decode,
                                                      item['interpreted'])
                    elif item['mode'] == 'table':
                        for row_map in item['interpreted'][1]:
                            for k, v in row_map.iteritems():
                                if v is None:
                                    row_map[k] = formatting.contents_str(v)
                    elif 'bundle_info' in item:
                        infos = []
                        if isinstance(item['bundle_info'], list):
                            infos = item['bundle_info']
                        elif isinstance(item['bundle_info'], dict):
                            infos = [item['bundle_info']]
                        for bundle_info in infos:
                            try:
                                if isinstance(bundle_info, dict):
                                    worksheet_util.format_metadata(
                                        bundle_info.get('metadata'))
                            except Exception, e:
                                print e
                                import ipdb
                                ipdb.set_trace()
예제 #4
0
파일: bundles.py 프로젝트: abmnv/codalab
    class BundleService():
        def __init__(self, user=None):
            self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
                                             lambda command: get_user_token(user), verbose=1)

        def items(self):
            return _call_with_retries(lambda: self.client.search())

        def get_bundle_info(self, uuid):
            ## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False):
            bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))
            # format permission data
            bundle_info['permission_str'] = permission_str(bundle_info['permission'])
            # format each groups as well
            for group_permission in bundle_info['group_permissions']:
                group_permission['permission_str'] = permission_str(group_permission['permission'])

            metadata = bundle_info['metadata']

            cls = get_bundle_subclass(bundle_info['bundle_type'])
            # format based on specs from the cli
            for spec in cls.METADATA_SPECS:
                key = spec.key
                if key not in metadata:
                    continue
                if metadata[key] == '' or metadata[key] == []:
                    continue
                value = worksheet_util.apply_func(spec.formatting, metadata.get(key))
                # if isinstance(value, list):
                #     value = ', '.join(value)
                metadata[key] = value

            bundle_info['metadata'] = metadata

            return bundle_info

        def head_target(self, target, maxlines=100):
            return self.client.head_target(target, maxlines)

        def search_bundles(self, keywords, worksheet_uuid=None):
            bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords)
            bundle_infos = self.client.get_bundle_infos(bundle_uuids)
            return bundle_infos

        def worksheets(self):
            return _call_with_retries(lambda: self.client.list_worksheets())

        def search_worksheets(self, keywords, worksheet_uuid=None):
            return _call_with_retries(lambda: self.client.search_worksheets(keywords))

        def create_worksheet(self, name):
            return _call_with_retries(lambda: self.client.new_worksheet(name, None))

        def worksheet(self, uuid, interpreted=False):
            try:
                worksheet_info  = self.client.get_worksheet_info(
                                            uuid,
                                            True,  #fetch_items
                                            True,  # get_permissions
                                )
            except PermissionError:
                raise UsageError # forces a not found
            worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info)
            # set permissions
            worksheet_info['edit_permission'] = False
            if worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL:
                worksheet_info['edit_permission'] = True

            worksheet_info['permission_str'] = permission_str(worksheet_info['permission'])
            # format each groups as well
            for group_permission in worksheet_info['group_permissions']:
                group_permission['permission_str'] = permission_str(group_permission['permission'])


            if interpreted:
                interpreted_items = worksheet_util.interpret_items(
                                    worksheet_util.get_default_schemas(),
                                    worksheet_info['items']
                                )
                worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items'])
                # Currently, only certain fields are base64 encoded.
                for item in worksheet_info['items']:
                    if item['mode'] in ['html', 'contents']:
                        # item['name'] in ['stdout', 'stderr']
                        item['interpreted'] = map(base64.b64decode, item['interpreted'])
                    elif 'bundle_info' in item:
                        for bundle_info in item['bundle_info']:
                            try:
                                ## sometimes bundle_info is a string. when item['mode'] is image
                                if isinstance(bundle_info, dict) and bundle_info.get('bundle_type', None) == 'run':
                                    if 'stdout' in bundle_info.keys():
                                        bundle_info['stdout'] = base64.b64decode(bundle_info['stdout'])
                                    if 'stderr' in bundle_info.keys():
                                        bundle_info['stderr'] = base64.b64decode(bundle_info['stderr'])
                            except Exception, e:
                                print e
                                import ipdb; ipdb.set_trace()


                return worksheet_info
예제 #5
0
    class BundleService():
        def __init__(self, user=None):
            self.client = RemoteBundleClient(
                settings.BUNDLE_SERVICE_URL,
                lambda command: get_user_token(user),
                verbose=1)

        def items(self):
            return _call_with_retries(lambda: self.client.search())

        def item(self, uuid):
            return _call_with_retries(
                lambda: self.client.get_bundle_info(uuid))

        def worksheets(self):
            return _call_with_retries(lambda: self.client.list_worksheets())

        def create_worksheet(self, name):
            return _call_with_retries(lambda: self.client.new_worksheet(name))

        def worksheet(self, uuid, interpreted=False):
            worksheet_info = _call_with_retries(
                lambda: self.client.get_worksheet_info(uuid, True))
            if interpreted:
                interpreted_items = worksheet_util.interpret_items(
                    worksheet_util.get_default_schemas(),
                    worksheet_info['items'])
                worksheet_info[
                    'items'] = self.client.resolve_interpreted_items(
                        interpreted_items['items'])
                return worksheet_info
            else:
                return worksheet_info

        def ls(self, uuid, path):
            return _call_with_retries(lambda: self.client.ls((uuid, path)))

        MAX_BYTES = 1024 * 1024

        def read_file(self, uuid, path):
            fid = self.client.open_target((uuid, path))
            try:
                while True:
                    bytes = self.client.read_file(fid, BundleService.MAX_BYTES)
                    yield bytes.data
                    if len(bytes.data) < BundleService.MAX_BYTES:
                        break
            finally:
                self.client.close_file(fid)

        def download_target(self, uuid, return_zip=False):
            target = (uuid, '')
            result_path, container_path = self.client.download_target(
                target=target, follow_symlinks=True, return_zip=return_zip)
            return (result_path, container_path)

        def http_status_from_exception(self, ex):
            # This is brittle. See https://github.com/codalab/codalab/issues/345.
            if type(ex) == UsageError:
                return 404
            return 500
예제 #6
0
class BundleService(object):
    # Maximum number of lines of files to show
    HEAD_MAX_LINES = 100

    def __init__(self, user=None):
        self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
                                         lambda command: get_user_token(user), verbose=1)

    def items(self):
        return _call_with_retries(lambda: self.client.search())

    def get_bundle_info(self, uuid):
        bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))

        if bundle_info is None:
            return None
        # Set permissions
        bundle_info['edit_permission'] = (bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
        # Format permissions into strings
        bundle_info['permission_str'] = permission_str(bundle_info['permission'])
        for group_permission in bundle_info['group_permissions']:
            group_permission['permission_str'] = permission_str(group_permission['permission'])

        metadata = bundle_info['metadata']

        cls = get_bundle_subclass(bundle_info['bundle_type'])
        for key, value in worksheet_util.get_formatted_metadata(cls, metadata):
            metadata[key] = value

        bundle_info['metadata'] = metadata
        bundle_info['editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields(cls, metadata)

        return bundle_info

    def head_target(self, target, max_num_lines=HEAD_MAX_LINES):
        return self.client.head_target(target, max_num_lines)

    def search_bundles(self, keywords, worksheet_uuid=None):
        bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords)
        bundle_infos = self.client.get_bundle_infos(bundle_uuids)
        return bundle_infos

    def get_worksheet_bundles(self, worksheet_uuid):
        worksheet_info = self.client.get_worksheet_info(worksheet_uuid, True, True)
        bundle_info_list = []
        for (bundle_info, subworksheet_info, value_obj, item_type) in worksheet_info['items']:
            if item_type == worksheet_util.TYPE_BUNDLE:
                bundle_info_list.append(bundle_info)
        return bundle_info_list

    def worksheets(self):
        return _call_with_retries(lambda: self.client.list_worksheets())

    def search_worksheets(self, keywords, worksheet_uuid=None):
        return _call_with_retries(lambda: self.client.search_worksheets(keywords))

    def create_worksheet(self, name):
        return _call_with_retries(lambda: self.client.new_worksheet(name, None))

    def get_worksheet_uuid(self, spec):
        # generic clean up just in case
        spec = smart_str(spec)

        # generic function sometimes get uuid already just return it.
        if spec_util.UUID_REGEX.match(spec):
            return spec
        else:
            return worksheet_util.get_worksheet_uuid(self.client, None, spec)

    def basic_worksheet(self, uuid):
        return self.worksheet(uuid, fetch_items=False, get_permissions=True, interpreted=False)

    def full_worksheet(self, uuid):
        return self.worksheet(uuid, fetch_items=True, get_permissions=True, interpreted=True)

    def worksheet(self, uuid, fetch_items, get_permissions, interpreted):
        """
        Return information about a worksheet. Calls
        - get_worksheet_info: get basic info
        - resolve_interpreted_items: get more information about a worksheet.
        In the future, for large worksheets, might want to break this up so
        that we can render something basic.
        """
        worksheet_info = self.client.get_worksheet_info(uuid, fetch_items, get_permissions)

        if fetch_items:
            worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info)

        # Set permissions
        worksheet_info['edit_permission'] = (worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
        # Format permissions into strings
        worksheet_info['permission_str'] = permission_str(worksheet_info['permission'])
        for group_permission in worksheet_info['group_permissions']:
            group_permission['permission_str'] = permission_str(group_permission['permission'])

        # Go and fetch more information about the worksheet contents by
        # resolving the interpreted items.
        if interpreted:
            try:
                interpreted_items = worksheet_util.interpret_items(
                                    worksheet_util.get_default_schemas(),
                                    worksheet_info['items'])
            except UsageError, e:
                interpreted_items = {'items': []}
                worksheet_info['error'] = str(e)

            worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items'])
            worksheet_info['raw_to_interpreted'] = interpreted_items['raw_to_interpreted']
            worksheet_info['interpreted_to_raw'] = interpreted_items['interpreted_to_raw']

            def decode_lines(interpreted):
                # interpreted is None or list of base64 encoded lines
                if interpreted is None:
                    return formatting.contents_str(None)
                else:
                    return map(base64.b64decode, interpreted)

            # Currently, only certain fields are base64 encoded.
            for item in worksheet_info['items']:
                if item['mode'] in ['html', 'contents']:
                    item['interpreted'] = decode_lines(item['interpreted'])
                elif item['mode'] == 'table':
                    for row_map in item['interpreted'][1]:
                        for k, v in row_map.iteritems():
                            if v is None:
                                 row_map[k] = formatting.contents_str(v)
                elif 'bundle_info' in item:
                    infos = []
                    if isinstance(item['bundle_info'], list):
                        infos = item['bundle_info']
                    elif isinstance(item['bundle_info'], dict):
                        infos = [item['bundle_info']]
                    for bundle_info in infos:
                        try:
                            if isinstance(bundle_info, dict):
                                worksheet_util.format_metadata(bundle_info.get('metadata'))
                        except Exception, e:
                            print e
                            import ipdb; ipdb.set_trace()
예제 #7
0
    class BundleService():
        # Maximum number of lines of files to show
        HEAD_MAX_LINES = 100

        def __init__(self, user=None):
            self.client = RemoteBundleClient(
                settings.BUNDLE_SERVICE_URL,
                lambda command: get_user_token(user),
                verbose=1)

        def items(self):
            return _call_with_retries(lambda: self.client.search())

        def get_bundle_info(self, uuid):
            bundle_info = _call_with_retries(
                lambda: self.client.get_bundle_info(uuid, True, True, True))

            # Set permissions
            bundle_info['edit_permission'] = (
                bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
            # Format permissions into strings
            bundle_info['permission_str'] = permission_str(
                bundle_info['permission'])
            for group_permission in bundle_info['group_permissions']:
                group_permission['permission_str'] = permission_str(
                    group_permission['permission'])

            metadata = bundle_info['metadata']

            cls = get_bundle_subclass(bundle_info['bundle_type'])
            for key, value in worksheet_util.get_formatted_metadata(
                    cls, metadata):
                metadata[key] = value

            bundle_info['metadata'] = metadata
            bundle_info[
                'editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields(
                    cls, metadata)

            return bundle_info

        def head_target(self, target, maxlines=100):
            return self.client.head_target(target, maxlines)

        def search_bundles(self, keywords, worksheet_uuid=None):
            bundle_uuids = self.client.search_bundle_uuids(
                worksheet_uuid, keywords)
            bundle_infos = self.client.get_bundle_infos(bundle_uuids)
            return bundle_infos

        def get_worksheet_bundles(self, worksheet_uuid):
            worksheet_info = self.client.get_worksheet_info(
                worksheet_uuid, True, True)
            bundle_info_list = []
            for (bundle_info, subworksheet_info, value_obj,
                 item_type) in worksheet_info['items']:
                if item_type == worksheet_util.TYPE_BUNDLE:
                    bundle_info_list.append(bundle_info)
            return bundle_info_list

        def worksheets(self):
            return _call_with_retries(lambda: self.client.list_worksheets())

        def search_worksheets(self, keywords, worksheet_uuid=None):
            return _call_with_retries(
                lambda: self.client.search_worksheets(keywords))

        def create_worksheet(self, name):
            return _call_with_retries(
                lambda: self.client.new_worksheet(name, None))

        def get_worksheet_uuid(self, spec):
            uuid = None
            spec = smart_str(spec)  # generic clean up just in case
            try:
                if (
                        spec_util.UUID_REGEX.match(spec)
                ):  # generic function sometimes get uuid already just return it.
                    uuid = spec
                else:
                    uuid = worksheet_util.get_worksheet_uuid(
                        self.client, None, spec)
            except UsageError, e:
                #TODO handle Found multiple worksheets with name
                raise e
            return uuid
예제 #8
0
파일: bundles.py 프로젝트: TPNguyen/codalab
    class BundleService():
        def __init__(self, user=None):
            self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
                                             lambda command: get_user_token(user), verbose=1)

        def items(self):
            return _call_with_retries(lambda: self.client.search())

        def get_bundle_info(self, uuid):
            ## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False):
            bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))
            # format permission data
            bundle_info['permission_str'] = permission_str(bundle_info['permission'])
            # format each groups as well
            for group_permission in bundle_info['group_permissions']:
                group_permission['permission_str'] = permission_str(group_permission['permission'])

            metadata = bundle_info['metadata']

            cls = get_bundle_subclass(bundle_info['bundle_type'])
            # format based on specs from the cli
            for spec in cls.METADATA_SPECS:
                key = spec.key
                if key not in metadata:
                    continue
                if metadata[key] == '' or metadata[key] == []:
                    continue
                value = worksheet_util.apply_func(spec.formatting, metadata.get(key))
                # if isinstance(value, list):
                #     value = ', '.join(value)
                metadata[key] = value

            bundle_info['metadata'] = metadata

            return bundle_info

        def head_target(self, target, maxlines=100):
            return self.client.head_target(target, maxlines)

        def search_bundles(self, keywords, worksheet_uuid=None):
            bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords)
            bundle_infos = self.client.get_bundle_infos(bundle_uuids)
            return bundle_infos

        def worksheets(self):
            return _call_with_retries(lambda: self.client.list_worksheets())

        def search_worksheets(self, keywords, worksheet_uuid=None):
            return _call_with_retries(lambda: self.client.search_worksheets(keywords))

        def create_worksheet(self, name):
            return _call_with_retries(lambda: self.client.new_worksheet(name, None))

        def worksheet(self, uuid, interpreted=False):
            try:
                worksheet_info  = self.client.get_worksheet_info(
                                            uuid,
                                            True,  #fetch_items
                                            True,  # get_permissions

                                )
            except PermissionError:
                raise UsageError # forces a not found
            worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info)
            # set permissions
            worksheet_info['edit_permission'] = False
            if worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL:
                worksheet_info['edit_permission'] = True

            worksheet_info['permission_str'] = permission_str(worksheet_info['permission'])
            # format each groups as well
            for group_permission in worksheet_info['group_permissions']:
                group_permission['permission_str'] = permission_str(group_permission['permission'])


            if interpreted:
                interpreted_items = worksheet_util.interpret_items(
                                    worksheet_util.get_default_schemas(),
                                    worksheet_info['items']
                                )
                worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items'])
                return worksheet_info
            else:
                return worksheet_info

        def create_run_bundle(self, args, worksheet_uuid):
            cli = self._create_cli(worksheet_uuid)
            parser = cli.create_parser('run')
            parser.add_argument('target_spec', help=cli.TARGET_SPEC_FORMAT, nargs='*')
            parser.add_argument('command', help='Command-line')
            metadata_util.add_arguments(RunBundle, set(), parser)
            metadata_util.add_edit_argument(parser)
            args = parser.parse_args(args)
            metadata = metadata_util.request_missing_metadata(RunBundle, args)
            targets = cli.parse_key_targets(self.client, worksheet_uuid, args.target_spec)

            new_bundle_uuid = self.client.derive_bundle('run', targets, args.command, metadata, worksheet_uuid)
            return new_bundle_uuid


        def upload_bundle_url(self, url, info, worksheet_uuid):
            file_name = url.split("/")[-1]
            info = {
                'bundle_type': 'dataset',
                'metadata': {
                    'description': 'Upload %s' % url,
                    'tags': [],
                    'name': '%s' % file_name,
                    'license': '',
                    'source_url': '%s' % url,
                }
            }
            new_bundle_uuid = self.client.upload_bundle_url(url, info, worksheet_uuid, True)
            return new_bundle_uuid

        def add_worksheet_item(self, worksheet_uuid, bundle_uuid):
            self.client.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle_uuid))

        def parse_and_update_worksheet(self, uuid, lines):
            worksheet_info = self.client.get_worksheet_info(uuid, True)
            new_items, commands = worksheet_util.parse_worksheet_form(lines, self.client, worksheet_info['uuid'])
            self.client.update_worksheet(
                                worksheet_info,
                                new_items
                        )


        def get_target_info(self, target, depth=1):
            return _call_with_retries(lambda: self.client.get_target_info(target, depth))

        def resolve_interpreted_items(self, interpreted_items):
            return _call_with_retries(lambda: self.client.resolve_interpreted_items(('test', 'test')))

        def get_worksheet_info(self):
            return _call_with_retries(lambda: self.client.get_worksheet_info())

        def delete_worksheet(self, worksheet_uuid):
            return _call_with_retries(lambda: self.client.delete_worksheet(worksheet_uuid))

        # Create an instance of a CLI.
        def _create_cli(self, worksheet_uuid):
            manager = CodaLabManager(temporary=True, clients={settings.BUNDLE_SERVICE_URL: self.client})
            manager.set_current_worksheet_uuid(self.client, worksheet_uuid)
            cli = bundle_cli.BundleCLI(manager, headless=True)
            return cli

        def general_command(self, worksheet_uuid, command):
            cli = self._create_cli(worksheet_uuid)
            args = worksheet_util.string_to_tokens(command)
            def do_command():
                from cStringIO import StringIO
                import sys
                real_stdout = sys.stdout
                sys.stdout = StringIO()
                stdout_str = None

                #real_stderr = sys.stderr
                #sys.stderr = StringIO()
                stderr_str = None

                exception = None
                try:
                    cli.do_command(args)
                    success = True
                except BaseException as e:  # To capture SystemExit
                    exception = e
                    success = False
                stdout_str = sys.stdout.getvalue()
                sys.stdout.close()
                sys.stdout = real_stdout

                #stderr_str = sys.stderr.getvalue()
                #sys.stderr.close()
                #sys.stderr = real_stderr

                print '>>> general_command on worksheet %s: %s' % (worksheet_uuid, command)
                print stdout_str
                print stderr_str
                return {'stdout': stdout_str, 'stderr': stderr_str, 'exception': str(exception) if exception else None}
            return _call_with_retries(do_command)

        MAX_BYTES = 1024*1024
        def read_file(self, uuid, path):
            fid = self.client.open_target((uuid, path))
            try:
                while True:
                    bytes = self.client.read_file(fid, BundleService.MAX_BYTES)
                    yield bytes.data
                    if len(bytes.data) < BundleService.MAX_BYTES:
                        break
            finally:
                self.client.close_file(fid)

        def download_target(self, uuid, return_zip=False):
            target = (uuid, '')
            result_path, container_path = self.client.download_target(target=target, follow_symlinks=True, return_zip=return_zip)
            return (result_path, container_path)

        def http_status_from_exception(self, ex):
            # This is brittle. See https://github.com/codalab/codalab/issues/345.
            if type(ex) == UsageError:
                return 404
            return 500

        def update_bundle_metadata(self, uuid, new_metadata):
            self.client.update_bundle_metadata(uuid, new_metadata)
            return
예제 #9
0
    class BundleService():
        def __init__(self, user=None):
            self.client = RemoteBundleClient(
                settings.BUNDLE_SERVICE_URL,
                lambda command: get_user_token(user),
                verbose=1)

        def items(self):
            return _call_with_retries(lambda: self.client.search())

        def get_bundle_info(self, uuid):
            ## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False):
            bundle_info = _call_with_retries(
                lambda: self.client.get_bundle_info(uuid, True, True, True))
            # format permission data
            bundle_info['permission_str'] = permission_str(
                bundle_info['permission'])
            # format each groups as well
            for group_permission in bundle_info['group_permissions']:
                group_permission['permission_str'] = permission_str(
                    group_permission['permission'])

            metadata = bundle_info['metadata']

            cls = get_bundle_subclass(bundle_info['bundle_type'])
            # format based on specs from the cli
            for spec in cls.METADATA_SPECS:
                key = spec.key
                if key not in metadata:
                    continue
                if metadata[key] == '' or metadata[key] == []:
                    continue
                value = worksheet_util.apply_func(spec.formatting,
                                                  metadata.get(key))
                # if isinstance(value, list):
                #     value = ', '.join(value)
                metadata[key] = value

            bundle_info['metadata'] = metadata

            return bundle_info

        def head_target(self, target, maxlines=100):
            return self.client.head_target(target, maxlines)

        def search_bundles(self, keywords, worksheet_uuid=None):
            bundle_uuids = self.client.search_bundle_uuids(
                worksheet_uuid, keywords)
            bundle_infos = self.client.get_bundle_infos(bundle_uuids)
            return bundle_infos

        def worksheets(self):
            return _call_with_retries(lambda: self.client.list_worksheets())

        def search_worksheets(self, keywords, worksheet_uuid=None):
            return _call_with_retries(
                lambda: self.client.search_worksheets(keywords))

        def create_worksheet(self, name):
            return _call_with_retries(
                lambda: self.client.new_worksheet(name, None))

        def worksheet(self, uuid, interpreted=False):
            try:
                worksheet_info = self.client.get_worksheet_info(
                    uuid,
                    True,  #fetch_items
                    True,  # get_permissions
                )
            except PermissionError:
                raise UsageError  # forces a not found
            worksheet_info['raw'] = worksheet_util.get_worksheet_lines(
                worksheet_info)
            # set permissions
            worksheet_info['edit_permission'] = False
            if worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL:
                worksheet_info['edit_permission'] = True

            worksheet_info['permission_str'] = permission_str(
                worksheet_info['permission'])
            # format each groups as well
            for group_permission in worksheet_info['group_permissions']:
                group_permission['permission_str'] = permission_str(
                    group_permission['permission'])

            if interpreted:
                interpreted_items = worksheet_util.interpret_items(
                    worksheet_util.get_default_schemas(),
                    worksheet_info['items'])
                worksheet_info[
                    'items'] = self.client.resolve_interpreted_items(
                        interpreted_items['items'])
                # Currently, only certain fields are base64 encoded.
                for item in worksheet_info['items']:
                    if item['mode'] in ['html', 'contents']:
                        # item['name'] in ['stdout', 'stderr']
                        item['interpreted'] = map(base64.b64decode,
                                                  item['interpreted'])
                    elif 'bundle_info' in item:
                        for bundle_info in item['bundle_info']:
                            try:
                                ## sometimes bundle_info is a string. when item['mode'] is image
                                if isinstance(
                                        bundle_info, dict) and bundle_info.get(
                                            'bundle_type', None) == 'run':
                                    if 'stdout' in bundle_info.keys():
                                        bundle_info[
                                            'stdout'] = base64.b64decode(
                                                bundle_info['stdout'])
                                    if 'stderr' in bundle_info.keys():
                                        bundle_info[
                                            'stderr'] = base64.b64decode(
                                                bundle_info['stderr'])
                            except Exception, e:
                                print e
                                import ipdb
                                ipdb.set_trace()

                return worksheet_info
예제 #10
0
    class BundleService():
        def __init__(self, user=None):
            self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
                                             lambda command: get_user_token(user), verbose=1)

        def items(self):
            return _call_with_retries(lambda: self.client.search())

        def get_bundle_info(self, uuid):
            ## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False):
            bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))
            # format permission data
            bundle_info['permission_str'] = permission_str(bundle_info['permission'])
            # format each groups as well
            for group_permission in bundle_info['group_permissions']:
                group_permission['permission_str'] = permission_str(group_permission['permission'])

            metadata = bundle_info['metadata']

            cls = get_bundle_subclass(bundle_info['bundle_type'])
            # format based on specs from the cli
            for spec in cls.METADATA_SPECS:
                key = spec.key
                if key not in metadata:
                    continue
                if metadata[key] == '' or metadata[key] == []:
                    continue
                value = worksheet_util.apply_func(spec.formatting, metadata.get(key))
                # if isinstance(value, list):
                #     value = ', '.join(value)
                metadata[key] = value

            bundle_info['metadata'] = metadata

            return bundle_info

        def head_target(self, target, maxlines=100):
            return self.client.head_target(target, maxlines)

        def search_bundles(self, keywords, worksheet_uuid=None):
            bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords)
            bundle_infos = self.client.get_bundle_infos(bundle_uuids)
            return bundle_infos

        def get_worksheet_bundles(self, worksheet_uuid):
            worksheet_info = self.client.get_worksheet_info(worksheet_uuid, True, True)
            bundle_info_list = []
            for (bundle_info, subworksheet_info, value_obj, item_type) in worksheet_info['items']:
                if item_type == worksheet_util.TYPE_BUNDLE:
                    bundle_info_list.append(bundle_info)
            return bundle_info_list

        def worksheets(self):
            return _call_with_retries(lambda: self.client.list_worksheets())

        def search_worksheets(self, keywords, worksheet_uuid=None):
            return _call_with_retries(lambda: self.client.search_worksheets(keywords))

        def create_worksheet(self, name):
            return _call_with_retries(lambda: self.client.new_worksheet(name, None))

        def get_worksheet_uuid(self, spec):
            uuid = None
            spec = smart_str(spec)  # generic clean up just in case
            try:
                if(spec_util.UUID_REGEX.match(spec)): # generic function sometimes get uuid already just return it.
                    uuid = spec
                else:
                    uuid = worksheet_util.get_worksheet_uuid(self.client, None, spec)
            except UsageError, e:
                #TODO handle Found multiple worksheets with name
                raise e
            return uuid
예제 #11
0
    class BundleService():
        def __init__(self, user=None):
            self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
                                             lambda command: get_user_token(user), verbose=1)

        def items(self):
            return _call_with_retries(lambda: self.client.search())

        def get_bundle_info(self, uuid):
            bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid))

            metadata = bundle_info['metadata']

            cls = get_bundle_subclass(bundle_info['bundle_type'])
            # format based on specs from the cli
            for spec in cls.METADATA_SPECS:
                key = spec.key
                if key not in metadata:
                    continue
                if metadata[key] == '' or metadata[key] == []:
                    continue
                value = worksheet_util.apply_func(spec.formatting, metadata.get(key))
                # if isinstance(value, list):
                #     value = ', '.join(value)
                metadata[key] = value

            bundle_info['metadata'] = metadata

            return bundle_info

        def search_bundles(self, keywords, worksheet_uuid=None):
            # search_bundle_uuids(worksheet_uuid, keywords, max_results, show_counts_only)
            bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords, 30, False)
            bundle_infos = self.client.get_bundle_infos(bundle_uuids)
            return bundle_infos

        def worksheets(self):
            return _call_with_retries(lambda: self.client.list_worksheets())

        def create_worksheet(self, name):
            return _call_with_retries(lambda: self.client.new_worksheet(name))

        def worksheet(self, uuid, interpreted=False):
            try:
                worksheet_info  = self.client.get_worksheet_info(
                                            uuid,
                                            True,  #fetch_items
                                            True,  # get_permissions

                                )
            except PermissionError:
                raise UsageError # forces a not found
            worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info)
            # set permissions
            worksheet_info['edit_permission'] = False
            if worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL:
                worksheet_info['edit_permission'] = True


            if interpreted:
                interpreted_items = worksheet_util.interpret_items(
                                    worksheet_util.get_default_schemas(),
                                    worksheet_info['items']
                                )
                worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items'])
                return worksheet_info
            else:
                return worksheet_info

        def derive_bundle(self, bundle_type, targets, worksheet_uuid, command):
            name = str(slugify(command))
            metadata = {'name': name, 'tags': [], 'allowed_time': u'', 'allowed_memory': u'', 'allowed_disk': u'', 'description': ''}
            new_bundle_uuid = self.client.derive_bundle(bundle_type, targets, command, metadata, worksheet_uuid)
            return new_bundle_uuid

        def upload_bundle_url(self, url, info, worksheet_uuid):
            file_name = url.split("/")[-1]
            info = {
                'bundle_type': 'dataset',
                'metadata': {
                    'description': 'Upload %s' % url,
                    'tags': [],
                    'name': '%s' % file_name,
                    'license': '',
                    'source_url': '%s' % url,
                }
            }
            new_bundle_uuid = self.client.upload_bundle_url(url, info, worksheet_uuid, True)
            return new_bundle_uuid

        def add_worksheet_item(self, worksheet_uuid, bundle_uuid):
            self.client.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle_uuid))

        def parse_and_update_worksheet(self, uuid, lines):
            worksheet_info = self.client.get_worksheet_info(uuid, True)
            new_items, commands = worksheet_util.parse_worksheet_form(lines, self.client, worksheet_info['uuid'])
            self.client.update_worksheet(
                                worksheet_info,
                                new_items
                        )


        def get_target_info(self, target, depth=1):
            return _call_with_retries(lambda: self.client.get_target_info(target, depth))

        def resolve_interpreted_items(self, interpreted_items):
            return _call_with_retries(lambda: self.client.resolve_interpreted_items(('test', 'test')))

        def get_worksheet_info(self):
            return _call_with_retries(lambda: self.client.get_worksheet_info())

        def delete_worksheet(self, worksheet_uuid):
            return _call_with_retries(lambda: self.client.delete_worksheet(worksheet_uuid))

        MAX_BYTES = 1024*1024
        def read_file(self, uuid, path):
            fid = self.client.open_target((uuid, path))
            try:
                while True:
                    bytes = self.client.read_file(fid, BundleService.MAX_BYTES)
                    yield bytes.data
                    if len(bytes.data) < BundleService.MAX_BYTES:
                        break
            finally:
                self.client.close_file(fid)

        def download_target(self, uuid, return_zip=False):
            target = (uuid, '')
            result_path, container_path = self.client.download_target(target=target, follow_symlinks=True, return_zip=return_zip)
            return (result_path, container_path)

        def http_status_from_exception(self, ex):
            # This is brittle. See https://github.com/codalab/codalab/issues/345.
            if type(ex) == UsageError:
                return 404
            return 500

        def update_bundle_metadata(self, uuid, new_metadata):
            self.client.update_bundle_metadata(uuid, new_metadata)
            return
예제 #12
0
    class BundleService():
        # Maximum number of lines of files to show
        HEAD_MAX_LINES = 100

        def __init__(self, user=None):
            self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
                                             lambda command: get_user_token(user), verbose=1)

        def items(self):
            return _call_with_retries(lambda: self.client.search())

        def get_bundle_info(self, uuid):
            bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))

            # Set permissions
            bundle_info['edit_permission'] = (bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
            # Format permissions into strings
            bundle_info['permission_str'] = permission_str(bundle_info['permission'])
            for group_permission in bundle_info['group_permissions']:
                group_permission['permission_str'] = permission_str(group_permission['permission'])

            metadata = bundle_info['metadata']

            cls = get_bundle_subclass(bundle_info['bundle_type'])
            for key, value in worksheet_util.get_formatted_metadata(cls, metadata):
                metadata[key] = value

            bundle_info['metadata'] = metadata
            bundle_info['editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields(cls, metadata)

            return bundle_info

        def head_target(self, target, maxlines=100):
            return self.client.head_target(target, maxlines)

        def search_bundles(self, keywords, worksheet_uuid=None):
            bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords)
            bundle_infos = self.client.get_bundle_infos(bundle_uuids)
            return bundle_infos

        def get_worksheet_bundles(self, worksheet_uuid):
            worksheet_info = self.client.get_worksheet_info(worksheet_uuid, True, True)
            bundle_info_list = []
            for (bundle_info, subworksheet_info, value_obj, item_type) in worksheet_info['items']:
                if item_type == worksheet_util.TYPE_BUNDLE:
                    bundle_info_list.append(bundle_info)
            return bundle_info_list

        def worksheets(self):
            return _call_with_retries(lambda: self.client.list_worksheets())

        def search_worksheets(self, keywords, worksheet_uuid=None):
            return _call_with_retries(lambda: self.client.search_worksheets(keywords))

        def create_worksheet(self, name):
            return _call_with_retries(lambda: self.client.new_worksheet(name, None))

        def get_worksheet_uuid(self, spec):
            uuid = None
            spec = smart_str(spec)  # generic clean up just in case
            try:
                if(spec_util.UUID_REGEX.match(spec)): # generic function sometimes get uuid already just return it.
                    uuid = spec
                else:
                    uuid = worksheet_util.get_worksheet_uuid(self.client, None, spec)
            except UsageError, e:
                #TODO handle Found multiple worksheets with name
                raise e
            return uuid