Esempio n. 1
0
    class BundleService():

        def __init__(self):
            self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL)

        def items(self):
            return _call_with_retries(lambda: self.client.search())

        def item(self, uuid):
            return _call_with_retries(lambda: self.client.info(uuid))

        def worksheets(self):
            return _call_with_retries(lambda: self.client.list_worksheets())

        def worksheet(self, uuid):
            return _call_with_retries(lambda: self.client.worksheet_info(uuid))

        def ls(self, uuid, path):
            return _call_with_retries(lambda: self.client.ls((uuid, path)))

        def http_status_from_exception(self, ex):
            # This is brittle. See https://github.com/codalab/codalab/issues/345.
            if type(ex) == UsageError:
                return 404
            return 500
Esempio n. 2
0
    class BundleService():

        def __init__(self):
            self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL)

        def items(self):
            results = self.client.search()
            return results

        def item(self, uuid):
            result = self.client.info(uuid)
            return result
    def authenticate(self, address):
        '''
        Authenticate with the given address. This will prompt user for password
        unless valid credentials are already available. Client state will be
        updated if new tokens are generated.

        Returns an access token.
        '''
        def _cache_token(token_info, username=None):
            '''
            Helper to update state with new token info and optional username.
            Returns the latest access token.
            '''
            token_info['expires_at'] = time.time() + float(token_info['expires_in']) - 60.0
            del token_info['expires_in']
            auth['token_info'] = token_info
            if username is not None:
                auth['username'] = username
            self.save_state()
            return token_info['access_token']

        # Check the cache for a valid token
        from codalab.client.remote_bundle_client import RemoteBundleClient
        auth_info = self.state['auth'].get(address, {})
        if 'token_info' in auth_info:
            token_info = auth_info['token_info']
            expires_at = token_info.get('expires_at', 0.0)
            if expires_at > time.time():
                # Token is usable but check if it's nearing expiration
                if expires_at >= (time.time() + 900.0):
                    return token_info['access_token']
                # Try to refresh token
                remote_client = RemoteBundleClient(address, lambda command: None)
                token_info = remote_client.login('refresh_token',
                                                 token_info['refresh_token'],
                                                 auth_info['username'])
                if token_info is not None:
                    return _cache_token(token_info)

        # If we get here, a valid token is not already available.
        auth = self.state['auth'][address] = {}
        print 'Requesting access at %s' % address
        print 'Username: '******'credentials', username, password)
        if token_info is None:
            raise UsageError("Invalid username or password")
        return _cache_token(token_info, username)
Esempio n. 4
0
    def client(self, address, is_cli=True):
        '''
        Return a client given the address.  Note that this can either be called
        by the CLI (is_cli=True) or the server (is_cli=False).
        If called by the CLI, we don't need to authenticate.
        Cache the Client if necessary.
        '''
        if address in self.clients:
            return self.clients[address]
        # if local force mockauth or if locl server use correct auth
        if is_local_address(address):
            bundle_store = self.bundle_store()
            model = self.model()
            auth_handler = self.auth_handler(mock=is_cli)

            from codalab.client.local_bundle_client import LocalBundleClient
            client = LocalBundleClient(address, bundle_store, model,
                                       auth_handler, self.cli_verbose)
            self.clients[address] = client
            if is_cli:
                # Set current user
                access_token = self._authenticate(client)
                auth_handler.validate_token(access_token)
        else:
            from codalab.client.remote_bundle_client import RemoteBundleClient
            client = RemoteBundleClient(
                address, lambda a_client: self._authenticate(a_client),
                self.cli_verbose())
            self.clients[address] = client
            self._authenticate(client)
        return client
Esempio n. 5
0
    class BundleService():
        def __init__(self):
            self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
                                             lambda command: "")  #TODO

        def items(self):
            return _call_with_retries(lambda: self.client.search())

        def item(self, uuid):
            return _call_with_retries(lambda: self.client.info(uuid))

        def worksheets(self):
            return _call_with_retries(lambda: self.client.list_worksheets())

        def create_worksheet(self, name):
            return _call_with_retries(lambda: self.client.new_worksheet(name))

        def worksheet(self, uuid):
            return _call_with_retries(lambda: self.client.worksheet_info(uuid))

        def ls(self, uuid, path):
            strm = self.read_file(uuid, 'bat.txt')
            return _call_with_retries(lambda: self.client.ls((uuid, path)))

        MAX_BYTES = 1024 * 1024

        def read_file(self, uuid, path):
            fid = self.client.open_target((uuid, path))
            try:
                while True:
                    bytes = self.client.read_file(fid, BundleService.MAX_BYTES)
                    yield bytes.data
                    if len(bytes.data) < BundleService.MAX_BYTES:
                        break
            finally:
                self.client.close_file(fid)

        def http_status_from_exception(self, ex):
            # This is brittle. See https://github.com/codalab/codalab/issues/345.
            if type(ex) == UsageError:
                return 404
            return 500
Esempio n. 6
0
    class BundleService():

        def __init__(self):
            self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
                                             lambda command: "") #TODO

        def items(self):
            return _call_with_retries(lambda: self.client.search())

        def item(self, uuid):
            return _call_with_retries(lambda: self.client.info(uuid))

        def worksheets(self):
            return _call_with_retries(lambda: self.client.list_worksheets())

        def create_worksheet(self, name):
            return _call_with_retries(lambda: self.client.new_worksheet(name))

        def worksheet(self, uuid):
            return _call_with_retries(lambda: self.client.worksheet_info(uuid))

        def ls(self, uuid, path):
            strm = self.read_file(uuid, 'bat.txt')
            return _call_with_retries(lambda: self.client.ls((uuid, path)))

        MAX_BYTES = 1024*1024
        def read_file(self, uuid, path):
            fid = self.client.open_target((uuid, path))
            try:
                while True:
                    bytes = self.client.read_file(fid, BundleService.MAX_BYTES)
                    yield bytes.data
                    if len(bytes.data) < BundleService.MAX_BYTES:
                        break
            finally:
                self.client.close_file(fid)

        def http_status_from_exception(self, ex):
            # This is brittle. See https://github.com/codalab/codalab/issues/345.
            if type(ex) == UsageError:
                return 404
            return 500
Esempio n. 7
0
 def __init__(self):
     self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL)
Esempio n. 8
0
    class BundleService():
        # Maximum number of lines of files to show
        HEAD_MAX_LINES = 100

        def __init__(self, user=None):
            self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
                                             lambda command: get_user_token(user), verbose=1)

        def items(self):
            return _call_with_retries(lambda: self.client.search())

        def get_bundle_info(self, uuid):
            bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))

            # Set permissions
            bundle_info['edit_permission'] = (bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
            # Format permissions into strings
            bundle_info['permission_str'] = permission_str(bundle_info['permission'])
            for group_permission in bundle_info['group_permissions']:
                group_permission['permission_str'] = permission_str(group_permission['permission'])

            metadata = bundle_info['metadata']

            cls = get_bundle_subclass(bundle_info['bundle_type'])
            for key, value in worksheet_util.get_formatted_metadata(cls, metadata):
                metadata[key] = value

            bundle_info['metadata'] = metadata
            bundle_info['editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields(cls, metadata)

            return bundle_info

        def head_target(self, target, maxlines=100):
            return self.client.head_target(target, maxlines)

        def search_bundles(self, keywords, worksheet_uuid=None):
            bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords)
            bundle_infos = self.client.get_bundle_infos(bundle_uuids)
            return bundle_infos

        def get_worksheet_bundles(self, worksheet_uuid):
            worksheet_info = self.client.get_worksheet_info(worksheet_uuid, True, True)
            bundle_info_list = []
            for (bundle_info, subworksheet_info, value_obj, item_type) in worksheet_info['items']:
                if item_type == worksheet_util.TYPE_BUNDLE:
                    bundle_info_list.append(bundle_info)
            return bundle_info_list

        def worksheets(self):
            return _call_with_retries(lambda: self.client.list_worksheets())

        def search_worksheets(self, keywords, worksheet_uuid=None):
            return _call_with_retries(lambda: self.client.search_worksheets(keywords))

        def create_worksheet(self, name):
            return _call_with_retries(lambda: self.client.new_worksheet(name, None))

        def get_worksheet_uuid(self, spec):
            uuid = None
            spec = smart_str(spec)  # generic clean up just in case
            try:
                if(spec_util.UUID_REGEX.match(spec)): # generic function sometimes get uuid already just return it.
                    uuid = spec
                else:
                    uuid = worksheet_util.get_worksheet_uuid(self.client, None, spec)
            except UsageError, e:
                #TODO handle Found multiple worksheets with name
                raise e
            return uuid
Esempio n. 9
0
    class BundleService():
        def __init__(self, user=None):
            self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
                                             lambda command: get_user_token(user), verbose=1)

        def items(self):
            return _call_with_retries(lambda: self.client.search())

        def get_bundle_info(self, uuid):
            ## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False):
            bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))
            # format permission data
            bundle_info['permission_str'] = permission_str(bundle_info['permission'])
            # format each groups as well
            for group_permission in bundle_info['group_permissions']:
                group_permission['permission_str'] = permission_str(group_permission['permission'])

            metadata = bundle_info['metadata']

            cls = get_bundle_subclass(bundle_info['bundle_type'])
            # format based on specs from the cli
            for spec in cls.METADATA_SPECS:
                key = spec.key
                if key not in metadata:
                    continue
                if metadata[key] == '' or metadata[key] == []:
                    continue
                value = worksheet_util.apply_func(spec.formatting, metadata.get(key))
                # if isinstance(value, list):
                #     value = ', '.join(value)
                metadata[key] = value

            bundle_info['metadata'] = metadata

            return bundle_info

        def head_target(self, target, maxlines=100):
            return self.client.head_target(target, maxlines)

        def search_bundles(self, keywords, worksheet_uuid=None):
            bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords)
            bundle_infos = self.client.get_bundle_infos(bundle_uuids)
            return bundle_infos

        def worksheets(self):
            return _call_with_retries(lambda: self.client.list_worksheets())

        def search_worksheets(self, keywords, worksheet_uuid=None):
            return _call_with_retries(lambda: self.client.search_worksheets(keywords))

        def create_worksheet(self, name):
            return _call_with_retries(lambda: self.client.new_worksheet(name, None))

        def worksheet(self, uuid, interpreted=False):
            try:
                worksheet_info  = self.client.get_worksheet_info(
                                            uuid,
                                            True,  #fetch_items
                                            True,  # get_permissions
                                )
            except PermissionError:
                raise UsageError # forces a not found
            worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info)
            # set permissions
            worksheet_info['edit_permission'] = False
            if worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL:
                worksheet_info['edit_permission'] = True

            worksheet_info['permission_str'] = permission_str(worksheet_info['permission'])
            # format each groups as well
            for group_permission in worksheet_info['group_permissions']:
                group_permission['permission_str'] = permission_str(group_permission['permission'])


            if interpreted:
                interpreted_items = worksheet_util.interpret_items(
                                    worksheet_util.get_default_schemas(),
                                    worksheet_info['items']
                                )
                worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items'])
                # Currently, only certain fields are base64 encoded.
                for item in worksheet_info['items']:
                    if item['mode'] in ['html', 'contents']:
                        # item['name'] in ['stdout', 'stderr']
                        item['interpreted'] = map(base64.b64decode, item['interpreted'])
                    elif 'bundle_info' in item:
                        for bundle_info in item['bundle_info']:
                            try:
                                ## sometimes bundle_info is a string. when item['mode'] is image
                                if isinstance(bundle_info, dict) and bundle_info.get('bundle_type', None) == 'run':
                                    if 'stdout' in bundle_info.keys():
                                        bundle_info['stdout'] = base64.b64decode(bundle_info['stdout'])
                                    if 'stderr' in bundle_info.keys():
                                        bundle_info['stderr'] = base64.b64decode(bundle_info['stderr'])
                            except Exception, e:
                                print e
                                import ipdb; ipdb.set_trace()


                return worksheet_info
Esempio n. 10
0
 def __init__(self):
     self.client = RemoteBundleClient(
         self._cli_url(), lambda command: self._get_user_token(), verbose=0)
Esempio n. 11
0
class RemoteBundleService(object):
    '''
    Adapts the RemoteBundleClient for REST calls.
    TODO(klopyrev): This version should eventually go away once the file upload
    logic is cleaned up. See below where this class is used for more information.
    '''
    def __init__(self):
        self.client = RemoteBundleClient(
            self._cli_url(), lambda command: self._get_user_token(), verbose=0)

    def _cli_url(self):
        return 'http://' + local.config['server']['host'] + ':' + str(
            local.config['server']['port'])

    def _get_user_token(self):
        """
        Returns an access token for the user. This function facilitates interactions
        with the bundle service.
        """
        CLIENT_ID = 'codalab_cli_client'

        if request.user is None:
            return None

        # Try to find an existing token that will work.
        token = local.model.find_oauth2_token(
            CLIENT_ID, request.user.user_id,
            datetime.utcnow() + timedelta(minutes=5))
        if token is not None:
            return token.access_token

        # Otherwise, generate a new one.
        token = OAuth2Token(
            local.model,
            access_token=generate_token(),
            refresh_token=None,
            scopes='',
            expires=datetime.utcnow() + timedelta(hours=10),
            client_id=CLIENT_ID,
            user_id=request.user.user_id,
        )
        local.model.save_oauth2_token(token)

        return token.access_token

    def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
        """
        Upload |source_file| (a stream) to |worksheet_uuid|.
        """
        # Construct info for creating the bundle.
        bundle_subclass = get_bundle_subclass(bundle_type)  # program or data
        metadata = metadata_util.fill_missing_metadata(
            bundle_subclass, {},
            initial_metadata={
                'name': source_file.filename,
                'description': 'Upload ' + source_file.filename
            })
        info = {'bundle_type': bundle_type, 'metadata': metadata}

        # Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
        remote_file_uuid = self.client.open_temp_file(metadata['name'])
        try:
            with closing(RPCFileHandle(remote_file_uuid,
                                       self.client.proxy)) as dest:
                file_util.copy(source_file.file,
                               dest,
                               autoflush=False,
                               print_status='Uploading %s' % metadata['name'])

            pack = False  # For now, always unpack (note: do this after set remote_file_uuid, which needs the extension)
            if not pack and zip_util.path_is_archive(metadata['name']):
                metadata['name'] = zip_util.strip_archive_ext(metadata['name'])

            # Then tell the client that the uploaded file handle is there.
            new_bundle_uuid = self.client.finish_upload_bundle(
                [remote_file_uuid],
                not pack,  # unpack
                info,
                worksheet_uuid,
                True)  # add_to_worksheet
        except:
            self.client.finalize_file(remote_file_uuid)
            raise
        return new_bundle_uuid
Esempio n. 12
0
    class BundleService():
        def __init__(self, user=None):
            self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
                                             lambda command: get_user_token(user), verbose=1)

        def items(self):
            return _call_with_retries(lambda: self.client.search())

        def get_bundle_info(self, uuid):
            ## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False):
            bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))
            # format permission data
            bundle_info['permission_str'] = permission_str(bundle_info['permission'])
            # format each groups as well
            for group_permission in bundle_info['group_permissions']:
                group_permission['permission_str'] = permission_str(group_permission['permission'])

            metadata = bundle_info['metadata']

            cls = get_bundle_subclass(bundle_info['bundle_type'])
            # format based on specs from the cli
            for spec in cls.METADATA_SPECS:
                key = spec.key
                if key not in metadata:
                    continue
                if metadata[key] == '' or metadata[key] == []:
                    continue
                value = worksheet_util.apply_func(spec.formatting, metadata.get(key))
                # if isinstance(value, list):
                #     value = ', '.join(value)
                metadata[key] = value

            bundle_info['metadata'] = metadata

            return bundle_info

        def head_target(self, target, maxlines=100):
            return self.client.head_target(target, maxlines)

        def search_bundles(self, keywords, worksheet_uuid=None):
            bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords)
            bundle_infos = self.client.get_bundle_infos(bundle_uuids)
            return bundle_infos

        def get_worksheet_bundles(self, worksheet_uuid):
            worksheet_info = self.client.get_worksheet_info(worksheet_uuid, True, True)
            bundle_info_list = []
            for (bundle_info, subworksheet_info, value_obj, item_type) in worksheet_info['items']:
                if item_type == worksheet_util.TYPE_BUNDLE:
                    bundle_info_list.append(bundle_info)
            return bundle_info_list

        def worksheets(self):
            return _call_with_retries(lambda: self.client.list_worksheets())

        def search_worksheets(self, keywords, worksheet_uuid=None):
            return _call_with_retries(lambda: self.client.search_worksheets(keywords))

        def create_worksheet(self, name):
            return _call_with_retries(lambda: self.client.new_worksheet(name, None))

        def get_worksheet_uuid(self, spec):
            uuid = None
            spec = smart_str(spec)  # generic clean up just in case
            try:
                if(spec_util.UUID_REGEX.match(spec)): # generic function sometimes get uuid already just return it.
                    uuid = spec
                else:
                    uuid = worksheet_util.get_worksheet_uuid(self.client, None, spec)
            except UsageError, e:
                #TODO handle Found multiple worksheets with name
                raise e
            return uuid
Esempio n. 13
0
class RemoteBundleService(object):
    '''
    Adapts the RemoteBundleClient for REST calls.
    TODO(klopyrev): This version should eventually go away once the file upload
    logic is cleaned up. See below where this class is used for more information.
    '''
    def __init__(self):
        self.client = RemoteBundleClient(self._cli_url(),
                                         lambda command: self._get_user_token(), verbose=0)

    def _cli_url(self):
        return 'http://' + local.config['server']['host'] + ':' + str(local.config['server']['port'])

    def _get_user_token(self):
        """
        Returns an access token for the user. This function facilitates interactions
        with the bundle service.
        """
        CLIENT_ID = 'codalab_cli_client'
    
        if request.user is None:
            return None
    
        # Try to find an existing token that will work.
        token = local.model.find_oauth2_token(
            CLIENT_ID,
            request.user.user_id,
            datetime.utcnow() + timedelta(minutes=5))
        if token is not None:
            return token.access_token
    
        # Otherwise, generate a new one.
        token = OAuth2Token(
            local.model,
            access_token=generate_token(),
            refresh_token=None,
            scopes='',
            expires=datetime.utcnow() + timedelta(hours=10),
            client_id=CLIENT_ID,
            user_id=request.user.user_id,
        )
        local.model.save_oauth2_token(token)
    
        return token.access_token

    def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
        """
        Upload |source_file| (a stream) to |worksheet_uuid|.
        """
        # Construct info for creating the bundle.
        bundle_subclass = get_bundle_subclass(bundle_type) # program or data
        metadata = metadata_util.fill_missing_metadata(bundle_subclass, {}, initial_metadata={'name': source_file.filename, 'description': 'Upload ' + source_file.filename})
        info = {'bundle_type': bundle_type, 'metadata': metadata}

        # Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
        remote_file_uuid = self.client.open_temp_file(metadata['name'])
        try:
            with closing(RPCFileHandle(remote_file_uuid, self.client.proxy)) as dest:
                file_util.copy(source_file.file, dest, autoflush=False, print_status='Uploading %s' % metadata['name'])
           
            pack = False  # For now, always unpack (note: do this after set remote_file_uuid, which needs the extension)
            if not pack and zip_util.path_is_archive(metadata['name']):
                metadata['name'] = zip_util.strip_archive_ext(metadata['name'])
           
            # Then tell the client that the uploaded file handle is there.
            new_bundle_uuid = self.client.finish_upload_bundle(
                [remote_file_uuid],
                not pack,  # unpack
                info,
                worksheet_uuid,
                True)  # add_to_worksheet
        except:
            self.client.finalize_file(remote_file_uuid)
            raise
        return new_bundle_uuid
Esempio n. 14
0
    class BundleService(object):
        # Maximum number of lines of files to show
        HEAD_MAX_LINES = 100

        def __init__(self, user=None):
            self.client = RemoteBundleClient(
                settings.BUNDLE_SERVICE_URL,
                lambda command: get_user_token(user),
                verbose=1)

        def items(self):
            return _call_with_retries(lambda: self.client.search())

        def get_bundle_info(self, uuid):
            bundle_info = _call_with_retries(
                lambda: self.client.get_bundle_info(uuid, True, True, True))

            if bundle_info is None:
                return None
            # Set permissions
            bundle_info['edit_permission'] = (
                bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
            # Format permissions into strings
            bundle_info['permission_str'] = permission_str(
                bundle_info['permission'])
            for group_permission in bundle_info['group_permissions']:
                group_permission['permission_str'] = permission_str(
                    group_permission['permission'])

            metadata = bundle_info['metadata']

            cls = get_bundle_subclass(bundle_info['bundle_type'])
            for key, value in worksheet_util.get_formatted_metadata(
                    cls, metadata):
                metadata[key] = value

            bundle_info['metadata'] = metadata
            bundle_info[
                'editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields(
                    cls, metadata)

            return bundle_info

        def head_target(self, target, max_num_lines=HEAD_MAX_LINES):
            return self.client.head_target(target, max_num_lines)

        def search_bundles(self, keywords, worksheet_uuid=None):
            bundle_uuids = self.client.search_bundle_uuids(
                worksheet_uuid, keywords)
            bundle_infos = self.client.get_bundle_infos(bundle_uuids)
            return bundle_infos

        def get_worksheet_bundles(self, worksheet_uuid):
            worksheet_info = self.client.get_worksheet_info(
                worksheet_uuid, True, True)
            bundle_info_list = []
            for (bundle_info, subworksheet_info, value_obj,
                 item_type) in worksheet_info['items']:
                if item_type == worksheet_util.TYPE_BUNDLE:
                    bundle_info_list.append(bundle_info)
            return bundle_info_list

        def worksheets(self):
            return _call_with_retries(lambda: self.client.list_worksheets())

        def search_worksheets(self, keywords, worksheet_uuid=None):
            return _call_with_retries(
                lambda: self.client.search_worksheets(keywords))

        def create_worksheet(self, name):
            return _call_with_retries(
                lambda: self.client.new_worksheet(name, None))

        def get_worksheet_uuid(self, spec):
            # generic clean up just in case
            spec = smart_str(spec)

            # generic function sometimes get uuid already just return it.
            if spec_util.UUID_REGEX.match(spec):
                return spec
            else:
                return worksheet_util.get_worksheet_uuid(
                    self.client, None, spec)

        def basic_worksheet(self, uuid):
            return self.worksheet(uuid,
                                  fetch_items=False,
                                  get_permissions=True,
                                  interpreted=False)

        def full_worksheet(self, uuid):
            return self.worksheet(uuid,
                                  fetch_items=True,
                                  get_permissions=True,
                                  interpreted=True)

        def worksheet(self, uuid, fetch_items, get_permissions, interpreted):
            """
            Return information about a worksheet. Calls
            - get_worksheet_info: get basic info
            - resolve_interpreted_items: get more information about a worksheet.
            In the future, for large worksheets, might want to break this up so
            that we can render something basic.
            """
            worksheet_info = self.client.get_worksheet_info(
                uuid, fetch_items, get_permissions)

            if fetch_items:
                worksheet_info['raw'] = worksheet_util.get_worksheet_lines(
                    worksheet_info)

            # Set permissions
            worksheet_info['edit_permission'] = (
                worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
            # Format permissions into strings
            worksheet_info['permission_str'] = permission_str(
                worksheet_info['permission'])
            for group_permission in worksheet_info['group_permissions']:
                group_permission['permission_str'] = permission_str(
                    group_permission['permission'])

            # Go and fetch more information about the worksheet contents by
            # resolving the interpreted items.
            if interpreted:
                try:
                    interpreted_items = worksheet_util.interpret_items(
                        worksheet_util.get_default_schemas(),
                        worksheet_info['items'])
                except UsageError, e:
                    interpreted_items = {'items': []}
                    worksheet_info['error'] = str(e)

                worksheet_info[
                    'items'] = self.client.resolve_interpreted_items(
                        interpreted_items['items'])
                # Currently, only certain fields are base64 encoded.
                for item in worksheet_info['items']:
                    if item['mode'] in ['html', 'contents']:
                        if item['interpreted'] is None:
                            item['interpreted'] = [
                                formatting.contents_str(item['interpreted'])
                            ]
                        else:
                            item['interpreted'] = map(base64.b64decode,
                                                      item['interpreted'])
                    elif item['mode'] == 'table':
                        for row_map in item['interpreted'][1]:
                            for k, v in row_map.iteritems():
                                if v is None:
                                    row_map[k] = formatting.contents_str(v)
                    elif 'bundle_info' in item:
                        infos = []
                        if isinstance(item['bundle_info'], list):
                            infos = item['bundle_info']
                        elif isinstance(item['bundle_info'], dict):
                            infos = [item['bundle_info']]
                        for bundle_info in infos:
                            try:
                                if isinstance(bundle_info, dict):
                                    worksheet_util.format_metadata(
                                        bundle_info.get('metadata'))
                            except Exception, e:
                                print e
                                import ipdb
                                ipdb.set_trace()
Esempio n. 15
0
    class BundleService():
        # Maximum number of lines of files to show
        HEAD_MAX_LINES = 100

        def __init__(self, user=None):
            self.client = RemoteBundleClient(
                settings.BUNDLE_SERVICE_URL,
                lambda command: get_user_token(user),
                verbose=1)

        def items(self):
            return _call_with_retries(lambda: self.client.search())

        def get_bundle_info(self, uuid):
            bundle_info = _call_with_retries(
                lambda: self.client.get_bundle_info(uuid, True, True, True))

            # Set permissions
            bundle_info['edit_permission'] = (
                bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
            # Format permissions into strings
            bundle_info['permission_str'] = permission_str(
                bundle_info['permission'])
            for group_permission in bundle_info['group_permissions']:
                group_permission['permission_str'] = permission_str(
                    group_permission['permission'])

            metadata = bundle_info['metadata']

            cls = get_bundle_subclass(bundle_info['bundle_type'])
            for key, value in worksheet_util.get_formatted_metadata(
                    cls, metadata):
                metadata[key] = value

            bundle_info['metadata'] = metadata
            bundle_info[
                'editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields(
                    cls, metadata)

            return bundle_info

        def head_target(self, target, maxlines=100):
            return self.client.head_target(target, maxlines)

        def search_bundles(self, keywords, worksheet_uuid=None):
            bundle_uuids = self.client.search_bundle_uuids(
                worksheet_uuid, keywords)
            bundle_infos = self.client.get_bundle_infos(bundle_uuids)
            return bundle_infos

        def get_worksheet_bundles(self, worksheet_uuid):
            worksheet_info = self.client.get_worksheet_info(
                worksheet_uuid, True, True)
            bundle_info_list = []
            for (bundle_info, subworksheet_info, value_obj,
                 item_type) in worksheet_info['items']:
                if item_type == worksheet_util.TYPE_BUNDLE:
                    bundle_info_list.append(bundle_info)
            return bundle_info_list

        def worksheets(self):
            return _call_with_retries(lambda: self.client.list_worksheets())

        def search_worksheets(self, keywords, worksheet_uuid=None):
            return _call_with_retries(
                lambda: self.client.search_worksheets(keywords))

        def create_worksheet(self, name):
            return _call_with_retries(
                lambda: self.client.new_worksheet(name, None))

        def get_worksheet_uuid(self, spec):
            uuid = None
            spec = smart_str(spec)  # generic clean up just in case
            try:
                if (
                        spec_util.UUID_REGEX.match(spec)
                ):  # generic function sometimes get uuid already just return it.
                    uuid = spec
                else:
                    uuid = worksheet_util.get_worksheet_uuid(
                        self.client, None, spec)
            except UsageError, e:
                #TODO handle Found multiple worksheets with name
                raise e
            return uuid
Esempio n. 16
0
    class BundleService():
        def __init__(self, user=None):
            self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
                                             lambda command: get_user_token(user), verbose=1)

        def items(self):
            return _call_with_retries(lambda: self.client.search())

        def get_bundle_info(self, uuid):
            ## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False):
            bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))
            # format permission data
            bundle_info['permission_str'] = permission_str(bundle_info['permission'])
            # format each groups as well
            for group_permission in bundle_info['group_permissions']:
                group_permission['permission_str'] = permission_str(group_permission['permission'])

            metadata = bundle_info['metadata']

            cls = get_bundle_subclass(bundle_info['bundle_type'])
            # format based on specs from the cli
            for spec in cls.METADATA_SPECS:
                key = spec.key
                if key not in metadata:
                    continue
                if metadata[key] == '' or metadata[key] == []:
                    continue
                value = worksheet_util.apply_func(spec.formatting, metadata.get(key))
                # if isinstance(value, list):
                #     value = ', '.join(value)
                metadata[key] = value

            bundle_info['metadata'] = metadata

            return bundle_info

        def head_target(self, target, maxlines=100):
            return self.client.head_target(target, maxlines)

        def search_bundles(self, keywords, worksheet_uuid=None):
            bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords)
            bundle_infos = self.client.get_bundle_infos(bundle_uuids)
            return bundle_infos

        def worksheets(self):
            return _call_with_retries(lambda: self.client.list_worksheets())

        def search_worksheets(self, keywords, worksheet_uuid=None):
            return _call_with_retries(lambda: self.client.search_worksheets(keywords))

        def create_worksheet(self, name):
            return _call_with_retries(lambda: self.client.new_worksheet(name, None))

        def worksheet(self, uuid, interpreted=False):
            try:
                worksheet_info  = self.client.get_worksheet_info(
                                            uuid,
                                            True,  #fetch_items
                                            True,  # get_permissions

                                )
            except PermissionError:
                raise UsageError # forces a not found
            worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info)
            # set permissions
            worksheet_info['edit_permission'] = False
            if worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL:
                worksheet_info['edit_permission'] = True

            worksheet_info['permission_str'] = permission_str(worksheet_info['permission'])
            # format each groups as well
            for group_permission in worksheet_info['group_permissions']:
                group_permission['permission_str'] = permission_str(group_permission['permission'])


            if interpreted:
                interpreted_items = worksheet_util.interpret_items(
                                    worksheet_util.get_default_schemas(),
                                    worksheet_info['items']
                                )
                worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items'])
                return worksheet_info
            else:
                return worksheet_info

        def create_run_bundle(self, args, worksheet_uuid):
            cli = self._create_cli(worksheet_uuid)
            parser = cli.create_parser('run')
            parser.add_argument('target_spec', help=cli.TARGET_SPEC_FORMAT, nargs='*')
            parser.add_argument('command', help='Command-line')
            metadata_util.add_arguments(RunBundle, set(), parser)
            metadata_util.add_edit_argument(parser)
            args = parser.parse_args(args)
            metadata = metadata_util.request_missing_metadata(RunBundle, args)
            targets = cli.parse_key_targets(self.client, worksheet_uuid, args.target_spec)

            new_bundle_uuid = self.client.derive_bundle('run', targets, args.command, metadata, worksheet_uuid)
            return new_bundle_uuid


        def upload_bundle_url(self, url, info, worksheet_uuid):
            file_name = url.split("/")[-1]
            info = {
                'bundle_type': 'dataset',
                'metadata': {
                    'description': 'Upload %s' % url,
                    'tags': [],
                    'name': '%s' % file_name,
                    'license': '',
                    'source_url': '%s' % url,
                }
            }
            new_bundle_uuid = self.client.upload_bundle_url(url, info, worksheet_uuid, True)
            return new_bundle_uuid

        def add_worksheet_item(self, worksheet_uuid, bundle_uuid):
            self.client.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle_uuid))

        def parse_and_update_worksheet(self, uuid, lines):
            worksheet_info = self.client.get_worksheet_info(uuid, True)
            new_items, commands = worksheet_util.parse_worksheet_form(lines, self.client, worksheet_info['uuid'])
            self.client.update_worksheet(
                                worksheet_info,
                                new_items
                        )


        def get_target_info(self, target, depth=1):
            return _call_with_retries(lambda: self.client.get_target_info(target, depth))

        def resolve_interpreted_items(self, interpreted_items):
            return _call_with_retries(lambda: self.client.resolve_interpreted_items(('test', 'test')))

        def get_worksheet_info(self):
            return _call_with_retries(lambda: self.client.get_worksheet_info())

        def delete_worksheet(self, worksheet_uuid):
            return _call_with_retries(lambda: self.client.delete_worksheet(worksheet_uuid))

        # Create an instance of a CLI.
        def _create_cli(self, worksheet_uuid):
            manager = CodaLabManager(temporary=True, clients={settings.BUNDLE_SERVICE_URL: self.client})
            manager.set_current_worksheet_uuid(self.client, worksheet_uuid)
            cli = bundle_cli.BundleCLI(manager, headless=True)
            return cli

        def general_command(self, worksheet_uuid, command):
            cli = self._create_cli(worksheet_uuid)
            args = worksheet_util.string_to_tokens(command)
            def do_command():
                from cStringIO import StringIO
                import sys
                real_stdout = sys.stdout
                sys.stdout = StringIO()
                stdout_str = None

                #real_stderr = sys.stderr
                #sys.stderr = StringIO()
                stderr_str = None

                exception = None
                try:
                    cli.do_command(args)
                    success = True
                except BaseException as e:  # To capture SystemExit
                    exception = e
                    success = False
                stdout_str = sys.stdout.getvalue()
                sys.stdout.close()
                sys.stdout = real_stdout

                #stderr_str = sys.stderr.getvalue()
                #sys.stderr.close()
                #sys.stderr = real_stderr

                print '>>> general_command on worksheet %s: %s' % (worksheet_uuid, command)
                print stdout_str
                print stderr_str
                return {'stdout': stdout_str, 'stderr': stderr_str, 'exception': str(exception) if exception else None}
            return _call_with_retries(do_command)

        MAX_BYTES = 1024*1024
        def read_file(self, uuid, path):
            fid = self.client.open_target((uuid, path))
            try:
                while True:
                    bytes = self.client.read_file(fid, BundleService.MAX_BYTES)
                    yield bytes.data
                    if len(bytes.data) < BundleService.MAX_BYTES:
                        break
            finally:
                self.client.close_file(fid)

        def download_target(self, uuid, return_zip=False):
            target = (uuid, '')
            result_path, container_path = self.client.download_target(target=target, follow_symlinks=True, return_zip=return_zip)
            return (result_path, container_path)

        def http_status_from_exception(self, ex):
            # This is brittle. See https://github.com/codalab/codalab/issues/345.
            if type(ex) == UsageError:
                return 404
            return 500

        def update_bundle_metadata(self, uuid, new_metadata):
            self.client.update_bundle_metadata(uuid, new_metadata)
            return
Esempio n. 17
0
    class BundleService():
        def __init__(self, user=None):
            self.client = RemoteBundleClient(
                settings.BUNDLE_SERVICE_URL,
                lambda command: get_user_token(user),
                verbose=1)

        def items(self):
            return _call_with_retries(lambda: self.client.search())

        def get_bundle_info(self, uuid):
            ## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False):
            bundle_info = _call_with_retries(
                lambda: self.client.get_bundle_info(uuid, True, True, True))
            # format permission data
            bundle_info['permission_str'] = permission_str(
                bundle_info['permission'])
            # format each groups as well
            for group_permission in bundle_info['group_permissions']:
                group_permission['permission_str'] = permission_str(
                    group_permission['permission'])

            metadata = bundle_info['metadata']

            cls = get_bundle_subclass(bundle_info['bundle_type'])
            # format based on specs from the cli
            for spec in cls.METADATA_SPECS:
                key = spec.key
                if key not in metadata:
                    continue
                if metadata[key] == '' or metadata[key] == []:
                    continue
                value = worksheet_util.apply_func(spec.formatting,
                                                  metadata.get(key))
                # if isinstance(value, list):
                #     value = ', '.join(value)
                metadata[key] = value

            bundle_info['metadata'] = metadata

            return bundle_info

        def head_target(self, target, maxlines=100):
            return self.client.head_target(target, maxlines)

        def search_bundles(self, keywords, worksheet_uuid=None):
            bundle_uuids = self.client.search_bundle_uuids(
                worksheet_uuid, keywords)
            bundle_infos = self.client.get_bundle_infos(bundle_uuids)
            return bundle_infos

        def worksheets(self):
            return _call_with_retries(lambda: self.client.list_worksheets())

        def search_worksheets(self, keywords, worksheet_uuid=None):
            return _call_with_retries(
                lambda: self.client.search_worksheets(keywords))

        def create_worksheet(self, name):
            return _call_with_retries(
                lambda: self.client.new_worksheet(name, None))

        def worksheet(self, uuid, interpreted=False):
            try:
                worksheet_info = self.client.get_worksheet_info(
                    uuid,
                    True,  #fetch_items
                    True,  # get_permissions
                )
            except PermissionError:
                raise UsageError  # forces a not found
            worksheet_info['raw'] = worksheet_util.get_worksheet_lines(
                worksheet_info)
            # set permissions
            worksheet_info['edit_permission'] = False
            if worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL:
                worksheet_info['edit_permission'] = True

            worksheet_info['permission_str'] = permission_str(
                worksheet_info['permission'])
            # format each groups as well
            for group_permission in worksheet_info['group_permissions']:
                group_permission['permission_str'] = permission_str(
                    group_permission['permission'])

            if interpreted:
                interpreted_items = worksheet_util.interpret_items(
                    worksheet_util.get_default_schemas(),
                    worksheet_info['items'])
                worksheet_info[
                    'items'] = self.client.resolve_interpreted_items(
                        interpreted_items['items'])
                # Currently, only certain fields are base64 encoded.
                for item in worksheet_info['items']:
                    if item['mode'] in ['html', 'contents']:
                        # item['name'] in ['stdout', 'stderr']
                        item['interpreted'] = map(base64.b64decode,
                                                  item['interpreted'])
                    elif 'bundle_info' in item:
                        for bundle_info in item['bundle_info']:
                            try:
                                ## sometimes bundle_info is a string. when item['mode'] is image
                                if isinstance(
                                        bundle_info, dict) and bundle_info.get(
                                            'bundle_type', None) == 'run':
                                    if 'stdout' in bundle_info.keys():
                                        bundle_info[
                                            'stdout'] = base64.b64decode(
                                                bundle_info['stdout'])
                                    if 'stderr' in bundle_info.keys():
                                        bundle_info[
                                            'stderr'] = base64.b64decode(
                                                bundle_info['stderr'])
                            except Exception, e:
                                print e
                                import ipdb
                                ipdb.set_trace()

                return worksheet_info
Esempio n. 18
0
 def __init__(self):
     self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
                                      lambda command: "") #TODO
Esempio n. 19
0
    class BundleService():

        def __init__(self, user=None):
            self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
                                             lambda command: get_user_token(user), verbose=1)

        def items(self):
            return _call_with_retries(lambda: self.client.search())

        def item(self, uuid):
            return _call_with_retries(lambda: self.client.get_bundle_info(uuid))

        def worksheets(self):
            return _call_with_retries(lambda: self.client.list_worksheets())

        def create_worksheet(self, name):
            return _call_with_retries(lambda: self.client.new_worksheet(name))

        def worksheet(self, uuid, interpreted=False):
            worksheet_info  = _call_with_retries(
                    lambda: self.client.get_worksheet_info(
                            uuid,
                            True
                    )
                )
            if interpreted:
                interpreted_items = worksheet_util.interpret_items(
                                    worksheet_util.get_default_schemas(),
                                    worksheet_info['items']
                                )
                worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items'])
                return worksheet_info
            else:
                return worksheet_info

        def ls(self, uuid, path):
            return _call_with_retries(lambda: self.client.ls((uuid, path)))

        MAX_BYTES = 1024*1024
        def read_file(self, uuid, path):
            fid = self.client.open_target((uuid, path))
            try:
                while True:
                    bytes = self.client.read_file(fid, BundleService.MAX_BYTES)
                    yield bytes.data
                    if len(bytes.data) < BundleService.MAX_BYTES:
                        break
            finally:
                self.client.close_file(fid)

        def download_target(self, uuid, return_zip=False):
            target = (uuid, '')
            result_path, container_path = self.client.download_target(target=target, follow_symlinks=True, return_zip=return_zip)
            return (result_path, container_path)

        def http_status_from_exception(self, ex):
            # This is brittle. See https://github.com/codalab/codalab/issues/345.
            if type(ex) == UsageError:
                return 404
            return 500
Esempio n. 20
0
 def __init__(self):
     self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
                                      lambda command: "")  #TODO
Esempio n. 21
0
 def __init__(self, user=None):
     self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
                                      lambda command: get_user_token(user), verbose=1)
Esempio n. 22
0
    class BundleService():
        def __init__(self, user=None):
            self.client = RemoteBundleClient(
                settings.BUNDLE_SERVICE_URL,
                lambda command: get_user_token(user),
                verbose=1)

        def items(self):
            return _call_with_retries(lambda: self.client.search())

        def item(self, uuid):
            return _call_with_retries(
                lambda: self.client.get_bundle_info(uuid))

        def worksheets(self):
            return _call_with_retries(lambda: self.client.list_worksheets())

        def create_worksheet(self, name):
            return _call_with_retries(lambda: self.client.new_worksheet(name))

        def worksheet(self, uuid, interpreted=False):
            worksheet_info = _call_with_retries(
                lambda: self.client.get_worksheet_info(uuid, True))
            if interpreted:
                interpreted_items = worksheet_util.interpret_items(
                    worksheet_util.get_default_schemas(),
                    worksheet_info['items'])
                worksheet_info[
                    'items'] = self.client.resolve_interpreted_items(
                        interpreted_items['items'])
                return worksheet_info
            else:
                return worksheet_info

        def ls(self, uuid, path):
            return _call_with_retries(lambda: self.client.ls((uuid, path)))

        MAX_BYTES = 1024 * 1024

        def read_file(self, uuid, path):
            fid = self.client.open_target((uuid, path))
            try:
                while True:
                    bytes = self.client.read_file(fid, BundleService.MAX_BYTES)
                    yield bytes.data
                    if len(bytes.data) < BundleService.MAX_BYTES:
                        break
            finally:
                self.client.close_file(fid)

        def download_target(self, uuid, return_zip=False):
            target = (uuid, '')
            result_path, container_path = self.client.download_target(
                target=target, follow_symlinks=True, return_zip=return_zip)
            return (result_path, container_path)

        def http_status_from_exception(self, ex):
            # This is brittle. See https://github.com/codalab/codalab/issues/345.
            if type(ex) == UsageError:
                return 404
            return 500
Esempio n. 23
0
 def __init__(self):
     self.client = RemoteBundleClient(self._cli_url(),
                                      lambda command: self._get_user_token(), verbose=0)
Esempio n. 24
0
    class BundleService():
        def __init__(self, user=None):
            self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
                                             lambda command: get_user_token(user), verbose=1)

        def items(self):
            return _call_with_retries(lambda: self.client.search())

        def get_bundle_info(self, uuid):
            bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid))

            metadata = bundle_info['metadata']

            cls = get_bundle_subclass(bundle_info['bundle_type'])
            # format based on specs from the cli
            for spec in cls.METADATA_SPECS:
                key = spec.key
                if key not in metadata:
                    continue
                if metadata[key] == '' or metadata[key] == []:
                    continue
                value = worksheet_util.apply_func(spec.formatting, metadata.get(key))
                # if isinstance(value, list):
                #     value = ', '.join(value)
                metadata[key] = value

            bundle_info['metadata'] = metadata

            return bundle_info

        def search_bundles(self, keywords, worksheet_uuid=None):
            # search_bundle_uuids(worksheet_uuid, keywords, max_results, show_counts_only)
            bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords, 30, False)
            bundle_infos = self.client.get_bundle_infos(bundle_uuids)
            return bundle_infos

        def worksheets(self):
            return _call_with_retries(lambda: self.client.list_worksheets())

        def create_worksheet(self, name):
            return _call_with_retries(lambda: self.client.new_worksheet(name))

        def worksheet(self, uuid, interpreted=False):
            try:
                worksheet_info  = self.client.get_worksheet_info(
                                            uuid,
                                            True,  #fetch_items
                                            True,  # get_permissions

                                )
            except PermissionError:
                raise UsageError # forces a not found
            worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info)
            # set permissions
            worksheet_info['edit_permission'] = False
            if worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL:
                worksheet_info['edit_permission'] = True


            if interpreted:
                interpreted_items = worksheet_util.interpret_items(
                                    worksheet_util.get_default_schemas(),
                                    worksheet_info['items']
                                )
                worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items'])
                return worksheet_info
            else:
                return worksheet_info

        def derive_bundle(self, bundle_type, targets, worksheet_uuid, command):
            name = str(slugify(command))
            metadata = {'name': name, 'tags': [], 'allowed_time': u'', 'allowed_memory': u'', 'allowed_disk': u'', 'description': ''}
            new_bundle_uuid = self.client.derive_bundle(bundle_type, targets, command, metadata, worksheet_uuid)
            return new_bundle_uuid

        def upload_bundle_url(self, url, info, worksheet_uuid):
            file_name = url.split("/")[-1]
            info = {
                'bundle_type': 'dataset',
                'metadata': {
                    'description': 'Upload %s' % url,
                    'tags': [],
                    'name': '%s' % file_name,
                    'license': '',
                    'source_url': '%s' % url,
                }
            }
            new_bundle_uuid = self.client.upload_bundle_url(url, info, worksheet_uuid, True)
            return new_bundle_uuid

        def add_worksheet_item(self, worksheet_uuid, bundle_uuid):
            self.client.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle_uuid))

        def parse_and_update_worksheet(self, uuid, lines):
            worksheet_info = self.client.get_worksheet_info(uuid, True)
            new_items, commands = worksheet_util.parse_worksheet_form(lines, self.client, worksheet_info['uuid'])
            self.client.update_worksheet(
                                worksheet_info,
                                new_items
                        )


        def get_target_info(self, target, depth=1):
            return _call_with_retries(lambda: self.client.get_target_info(target, depth))

        def resolve_interpreted_items(self, interpreted_items):
            return _call_with_retries(lambda: self.client.resolve_interpreted_items(('test', 'test')))

        def get_worksheet_info(self):
            return _call_with_retries(lambda: self.client.get_worksheet_info())

        def delete_worksheet(self, worksheet_uuid):
            return _call_with_retries(lambda: self.client.delete_worksheet(worksheet_uuid))

        MAX_BYTES = 1024*1024
        def read_file(self, uuid, path):
            fid = self.client.open_target((uuid, path))
            try:
                while True:
                    bytes = self.client.read_file(fid, BundleService.MAX_BYTES)
                    yield bytes.data
                    if len(bytes.data) < BundleService.MAX_BYTES:
                        break
            finally:
                self.client.close_file(fid)

        def download_target(self, uuid, return_zip=False):
            target = (uuid, '')
            result_path, container_path = self.client.download_target(target=target, follow_symlinks=True, return_zip=return_zip)
            return (result_path, container_path)

        def http_status_from_exception(self, ex):
            # This is brittle. See https://github.com/codalab/codalab/issues/345.
            if type(ex) == UsageError:
                return 404
            return 500

        def update_bundle_metadata(self, uuid, new_metadata):
            self.client.update_bundle_metadata(uuid, new_metadata)
            return
Esempio n. 25
0
 def __init__(self, user=None):
     self.client = RemoteBundleClient(
         settings.BUNDLE_SERVICE_URL,
         lambda command: get_user_token(user),
         verbose=1)
Esempio n. 26
0
class BundleService(object):
    # Maximum number of lines of files to show
    HEAD_MAX_LINES = 100

    def __init__(self, user=None):
        self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
                                         lambda command: get_user_token(user), verbose=1)

    def items(self):
        return _call_with_retries(lambda: self.client.search())

    def get_bundle_info(self, uuid):
        bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))

        if bundle_info is None:
            return None
        # Set permissions
        bundle_info['edit_permission'] = (bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
        # Format permissions into strings
        bundle_info['permission_str'] = permission_str(bundle_info['permission'])
        for group_permission in bundle_info['group_permissions']:
            group_permission['permission_str'] = permission_str(group_permission['permission'])

        metadata = bundle_info['metadata']

        cls = get_bundle_subclass(bundle_info['bundle_type'])
        for key, value in worksheet_util.get_formatted_metadata(cls, metadata):
            metadata[key] = value

        bundle_info['metadata'] = metadata
        bundle_info['editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields(cls, metadata)

        return bundle_info

    def head_target(self, target, max_num_lines=HEAD_MAX_LINES):
        return self.client.head_target(target, max_num_lines)

    def search_bundles(self, keywords, worksheet_uuid=None):
        bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords)
        bundle_infos = self.client.get_bundle_infos(bundle_uuids)
        return bundle_infos

    def get_worksheet_bundles(self, worksheet_uuid):
        worksheet_info = self.client.get_worksheet_info(worksheet_uuid, True, True)
        bundle_info_list = []
        for (bundle_info, subworksheet_info, value_obj, item_type) in worksheet_info['items']:
            if item_type == worksheet_util.TYPE_BUNDLE:
                bundle_info_list.append(bundle_info)
        return bundle_info_list

    def worksheets(self):
        return _call_with_retries(lambda: self.client.list_worksheets())

    def search_worksheets(self, keywords, worksheet_uuid=None):
        return _call_with_retries(lambda: self.client.search_worksheets(keywords))

    def create_worksheet(self, name):
        return _call_with_retries(lambda: self.client.new_worksheet(name, None))

    def get_worksheet_uuid(self, spec):
        # generic clean up just in case
        spec = smart_str(spec)

        # generic function sometimes get uuid already just return it.
        if spec_util.UUID_REGEX.match(spec):
            return spec
        else:
            return worksheet_util.get_worksheet_uuid(self.client, None, spec)

    def basic_worksheet(self, uuid):
        return self.worksheet(uuid, fetch_items=False, get_permissions=True, interpreted=False)

    def full_worksheet(self, uuid):
        return self.worksheet(uuid, fetch_items=True, get_permissions=True, interpreted=True)

    def worksheet(self, uuid, fetch_items, get_permissions, interpreted):
        """
        Return information about a worksheet. Calls
        - get_worksheet_info: get basic info
        - resolve_interpreted_items: get more information about a worksheet.
        In the future, for large worksheets, might want to break this up so
        that we can render something basic.
        """
        worksheet_info = self.client.get_worksheet_info(uuid, fetch_items, get_permissions)

        if fetch_items:
            worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info)

        # Set permissions
        worksheet_info['edit_permission'] = (worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
        # Format permissions into strings
        worksheet_info['permission_str'] = permission_str(worksheet_info['permission'])
        for group_permission in worksheet_info['group_permissions']:
            group_permission['permission_str'] = permission_str(group_permission['permission'])

        # Go and fetch more information about the worksheet contents by
        # resolving the interpreted items.
        if interpreted:
            try:
                interpreted_items = worksheet_util.interpret_items(
                                    worksheet_util.get_default_schemas(),
                                    worksheet_info['items'])
            except UsageError, e:
                interpreted_items = {'items': []}
                worksheet_info['error'] = str(e)

            worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items'])
            worksheet_info['raw_to_interpreted'] = interpreted_items['raw_to_interpreted']
            worksheet_info['interpreted_to_raw'] = interpreted_items['interpreted_to_raw']

            def decode_lines(interpreted):
                # interpreted is None or list of base64 encoded lines
                if interpreted is None:
                    return formatting.contents_str(None)
                else:
                    return map(base64.b64decode, interpreted)

            # Currently, only certain fields are base64 encoded.
            for item in worksheet_info['items']:
                if item['mode'] in ['html', 'contents']:
                    item['interpreted'] = decode_lines(item['interpreted'])
                elif item['mode'] == 'table':
                    for row_map in item['interpreted'][1]:
                        for k, v in row_map.iteritems():
                            if v is None:
                                 row_map[k] = formatting.contents_str(v)
                elif 'bundle_info' in item:
                    infos = []
                    if isinstance(item['bundle_info'], list):
                        infos = item['bundle_info']
                    elif isinstance(item['bundle_info'], dict):
                        infos = [item['bundle_info']]
                    for bundle_info in infos:
                        try:
                            if isinstance(bundle_info, dict):
                                worksheet_util.format_metadata(bundle_info.get('metadata'))
                        except Exception, e:
                            print e
                            import ipdb; ipdb.set_trace()