class BundleService(): def __init__(self): self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL) def items(self): return _call_with_retries(lambda: self.client.search()) def item(self, uuid): return _call_with_retries(lambda: self.client.info(uuid)) def worksheets(self): return _call_with_retries(lambda: self.client.list_worksheets()) def worksheet(self, uuid): return _call_with_retries(lambda: self.client.worksheet_info(uuid)) def ls(self, uuid, path): return _call_with_retries(lambda: self.client.ls((uuid, path))) def http_status_from_exception(self, ex): # This is brittle. See https://github.com/codalab/codalab/issues/345. if type(ex) == UsageError: return 404 return 500
class BundleService(): def __init__(self): self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL) def items(self): results = self.client.search() return results def item(self, uuid): result = self.client.info(uuid) return result
def authenticate(self, address): ''' Authenticate with the given address. This will prompt user for password unless valid credentials are already available. Client state will be updated if new tokens are generated. Returns an access token. ''' def _cache_token(token_info, username=None): ''' Helper to update state with new token info and optional username. Returns the latest access token. ''' token_info['expires_at'] = time.time() + float(token_info['expires_in']) - 60.0 del token_info['expires_in'] auth['token_info'] = token_info if username is not None: auth['username'] = username self.save_state() return token_info['access_token'] # Check the cache for a valid token from codalab.client.remote_bundle_client import RemoteBundleClient auth_info = self.state['auth'].get(address, {}) if 'token_info' in auth_info: token_info = auth_info['token_info'] expires_at = token_info.get('expires_at', 0.0) if expires_at > time.time(): # Token is usable but check if it's nearing expiration if expires_at >= (time.time() + 900.0): return token_info['access_token'] # Try to refresh token remote_client = RemoteBundleClient(address, lambda command: None) token_info = remote_client.login('refresh_token', token_info['refresh_token'], auth_info['username']) if token_info is not None: return _cache_token(token_info) # If we get here, a valid token is not already available. auth = self.state['auth'][address] = {} print 'Requesting access at %s' % address print 'Username: '******'credentials', username, password) if token_info is None: raise UsageError("Invalid username or password") return _cache_token(token_info, username)
def client(self, address, is_cli=True): ''' Return a client given the address. Note that this can either be called by the CLI (is_cli=True) or the server (is_cli=False). If called by the CLI, we don't need to authenticate. Cache the Client if necessary. ''' if address in self.clients: return self.clients[address] # if local force mockauth or if locl server use correct auth if is_local_address(address): bundle_store = self.bundle_store() model = self.model() auth_handler = self.auth_handler(mock=is_cli) from codalab.client.local_bundle_client import LocalBundleClient client = LocalBundleClient(address, bundle_store, model, auth_handler, self.cli_verbose) self.clients[address] = client if is_cli: # Set current user access_token = self._authenticate(client) auth_handler.validate_token(access_token) else: from codalab.client.remote_bundle_client import RemoteBundleClient client = RemoteBundleClient( address, lambda a_client: self._authenticate(a_client), self.cli_verbose()) self.clients[address] = client self._authenticate(client) return client
class BundleService(): def __init__(self): self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL, lambda command: "") #TODO def items(self): return _call_with_retries(lambda: self.client.search()) def item(self, uuid): return _call_with_retries(lambda: self.client.info(uuid)) def worksheets(self): return _call_with_retries(lambda: self.client.list_worksheets()) def create_worksheet(self, name): return _call_with_retries(lambda: self.client.new_worksheet(name)) def worksheet(self, uuid): return _call_with_retries(lambda: self.client.worksheet_info(uuid)) def ls(self, uuid, path): strm = self.read_file(uuid, 'bat.txt') return _call_with_retries(lambda: self.client.ls((uuid, path))) MAX_BYTES = 1024 * 1024 def read_file(self, uuid, path): fid = self.client.open_target((uuid, path)) try: while True: bytes = self.client.read_file(fid, BundleService.MAX_BYTES) yield bytes.data if len(bytes.data) < BundleService.MAX_BYTES: break finally: self.client.close_file(fid) def http_status_from_exception(self, ex): # This is brittle. See https://github.com/codalab/codalab/issues/345. if type(ex) == UsageError: return 404 return 500
class BundleService(): def __init__(self): self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL, lambda command: "") #TODO def items(self): return _call_with_retries(lambda: self.client.search()) def item(self, uuid): return _call_with_retries(lambda: self.client.info(uuid)) def worksheets(self): return _call_with_retries(lambda: self.client.list_worksheets()) def create_worksheet(self, name): return _call_with_retries(lambda: self.client.new_worksheet(name)) def worksheet(self, uuid): return _call_with_retries(lambda: self.client.worksheet_info(uuid)) def ls(self, uuid, path): strm = self.read_file(uuid, 'bat.txt') return _call_with_retries(lambda: self.client.ls((uuid, path))) MAX_BYTES = 1024*1024 def read_file(self, uuid, path): fid = self.client.open_target((uuid, path)) try: while True: bytes = self.client.read_file(fid, BundleService.MAX_BYTES) yield bytes.data if len(bytes.data) < BundleService.MAX_BYTES: break finally: self.client.close_file(fid) def http_status_from_exception(self, ex): # This is brittle. See https://github.com/codalab/codalab/issues/345. if type(ex) == UsageError: return 404 return 500
def __init__(self): self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL)
class BundleService(): # Maximum number of lines of files to show HEAD_MAX_LINES = 100 def __init__(self, user=None): self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL, lambda command: get_user_token(user), verbose=1) def items(self): return _call_with_retries(lambda: self.client.search()) def get_bundle_info(self, uuid): bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True)) # Set permissions bundle_info['edit_permission'] = (bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL) # Format permissions into strings bundle_info['permission_str'] = permission_str(bundle_info['permission']) for group_permission in bundle_info['group_permissions']: group_permission['permission_str'] = permission_str(group_permission['permission']) metadata = bundle_info['metadata'] cls = get_bundle_subclass(bundle_info['bundle_type']) for key, value in worksheet_util.get_formatted_metadata(cls, metadata): metadata[key] = value bundle_info['metadata'] = metadata bundle_info['editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields(cls, metadata) return bundle_info def head_target(self, target, maxlines=100): return self.client.head_target(target, maxlines) def search_bundles(self, keywords, worksheet_uuid=None): bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords) bundle_infos = self.client.get_bundle_infos(bundle_uuids) return bundle_infos def get_worksheet_bundles(self, worksheet_uuid): worksheet_info = self.client.get_worksheet_info(worksheet_uuid, True, True) bundle_info_list = [] for (bundle_info, subworksheet_info, value_obj, item_type) in worksheet_info['items']: if item_type == worksheet_util.TYPE_BUNDLE: bundle_info_list.append(bundle_info) return bundle_info_list def worksheets(self): return _call_with_retries(lambda: self.client.list_worksheets()) def search_worksheets(self, keywords, worksheet_uuid=None): return _call_with_retries(lambda: self.client.search_worksheets(keywords)) def create_worksheet(self, name): return _call_with_retries(lambda: self.client.new_worksheet(name, None)) def get_worksheet_uuid(self, spec): uuid = None spec = smart_str(spec) # generic clean up just in case try: if(spec_util.UUID_REGEX.match(spec)): # generic function sometimes get uuid already just return it. uuid = spec else: uuid = worksheet_util.get_worksheet_uuid(self.client, None, spec) except UsageError, e: #TODO handle Found multiple worksheets with name raise e return uuid
class BundleService(): def __init__(self, user=None): self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL, lambda command: get_user_token(user), verbose=1) def items(self): return _call_with_retries(lambda: self.client.search()) def get_bundle_info(self, uuid): ## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False): bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True)) # format permission data bundle_info['permission_str'] = permission_str(bundle_info['permission']) # format each groups as well for group_permission in bundle_info['group_permissions']: group_permission['permission_str'] = permission_str(group_permission['permission']) metadata = bundle_info['metadata'] cls = get_bundle_subclass(bundle_info['bundle_type']) # format based on specs from the cli for spec in cls.METADATA_SPECS: key = spec.key if key not in metadata: continue if metadata[key] == '' or metadata[key] == []: continue value = worksheet_util.apply_func(spec.formatting, metadata.get(key)) # if isinstance(value, list): # value = ', '.join(value) metadata[key] = value bundle_info['metadata'] = metadata return bundle_info def head_target(self, target, maxlines=100): return self.client.head_target(target, maxlines) def search_bundles(self, keywords, worksheet_uuid=None): bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords) bundle_infos = self.client.get_bundle_infos(bundle_uuids) return bundle_infos def worksheets(self): return _call_with_retries(lambda: self.client.list_worksheets()) def search_worksheets(self, keywords, worksheet_uuid=None): return _call_with_retries(lambda: self.client.search_worksheets(keywords)) def create_worksheet(self, name): return _call_with_retries(lambda: self.client.new_worksheet(name, None)) def worksheet(self, uuid, interpreted=False): try: worksheet_info = self.client.get_worksheet_info( uuid, True, #fetch_items True, # get_permissions ) except PermissionError: raise UsageError # forces a not found worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info) # set permissions worksheet_info['edit_permission'] = False if worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL: worksheet_info['edit_permission'] = True worksheet_info['permission_str'] = permission_str(worksheet_info['permission']) # format each groups as well for group_permission in worksheet_info['group_permissions']: group_permission['permission_str'] = permission_str(group_permission['permission']) if interpreted: interpreted_items = worksheet_util.interpret_items( worksheet_util.get_default_schemas(), worksheet_info['items'] ) worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items']) # Currently, only certain fields are base64 encoded. for item in worksheet_info['items']: if item['mode'] in ['html', 'contents']: # item['name'] in ['stdout', 'stderr'] item['interpreted'] = map(base64.b64decode, item['interpreted']) elif 'bundle_info' in item: for bundle_info in item['bundle_info']: try: ## sometimes bundle_info is a string. when item['mode'] is image if isinstance(bundle_info, dict) and bundle_info.get('bundle_type', None) == 'run': if 'stdout' in bundle_info.keys(): bundle_info['stdout'] = base64.b64decode(bundle_info['stdout']) if 'stderr' in bundle_info.keys(): bundle_info['stderr'] = base64.b64decode(bundle_info['stderr']) except Exception, e: print e import ipdb; ipdb.set_trace() return worksheet_info
def __init__(self): self.client = RemoteBundleClient( self._cli_url(), lambda command: self._get_user_token(), verbose=0)
class RemoteBundleService(object): ''' Adapts the RemoteBundleClient for REST calls. TODO(klopyrev): This version should eventually go away once the file upload logic is cleaned up. See below where this class is used for more information. ''' def __init__(self): self.client = RemoteBundleClient( self._cli_url(), lambda command: self._get_user_token(), verbose=0) def _cli_url(self): return 'http://' + local.config['server']['host'] + ':' + str( local.config['server']['port']) def _get_user_token(self): """ Returns an access token for the user. This function facilitates interactions with the bundle service. """ CLIENT_ID = 'codalab_cli_client' if request.user is None: return None # Try to find an existing token that will work. token = local.model.find_oauth2_token( CLIENT_ID, request.user.user_id, datetime.utcnow() + timedelta(minutes=5)) if token is not None: return token.access_token # Otherwise, generate a new one. token = OAuth2Token( local.model, access_token=generate_token(), refresh_token=None, scopes='', expires=datetime.utcnow() + timedelta(hours=10), client_id=CLIENT_ID, user_id=request.user.user_id, ) local.model.save_oauth2_token(token) return token.access_token def upload_bundle(self, source_file, bundle_type, worksheet_uuid): """ Upload |source_file| (a stream) to |worksheet_uuid|. """ # Construct info for creating the bundle. bundle_subclass = get_bundle_subclass(bundle_type) # program or data metadata = metadata_util.fill_missing_metadata( bundle_subclass, {}, initial_metadata={ 'name': source_file.filename, 'description': 'Upload ' + source_file.filename }) info = {'bundle_type': bundle_type, 'metadata': metadata} # Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI). remote_file_uuid = self.client.open_temp_file(metadata['name']) try: with closing(RPCFileHandle(remote_file_uuid, self.client.proxy)) as dest: file_util.copy(source_file.file, dest, autoflush=False, print_status='Uploading %s' % metadata['name']) pack = False # For now, always unpack (note: do this after set remote_file_uuid, which needs the extension) if not pack and zip_util.path_is_archive(metadata['name']): metadata['name'] = zip_util.strip_archive_ext(metadata['name']) # Then tell the client that the uploaded file handle is there. new_bundle_uuid = self.client.finish_upload_bundle( [remote_file_uuid], not pack, # unpack info, worksheet_uuid, True) # add_to_worksheet except: self.client.finalize_file(remote_file_uuid) raise return new_bundle_uuid
class BundleService(): def __init__(self, user=None): self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL, lambda command: get_user_token(user), verbose=1) def items(self): return _call_with_retries(lambda: self.client.search()) def get_bundle_info(self, uuid): ## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False): bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True)) # format permission data bundle_info['permission_str'] = permission_str(bundle_info['permission']) # format each groups as well for group_permission in bundle_info['group_permissions']: group_permission['permission_str'] = permission_str(group_permission['permission']) metadata = bundle_info['metadata'] cls = get_bundle_subclass(bundle_info['bundle_type']) # format based on specs from the cli for spec in cls.METADATA_SPECS: key = spec.key if key not in metadata: continue if metadata[key] == '' or metadata[key] == []: continue value = worksheet_util.apply_func(spec.formatting, metadata.get(key)) # if isinstance(value, list): # value = ', '.join(value) metadata[key] = value bundle_info['metadata'] = metadata return bundle_info def head_target(self, target, maxlines=100): return self.client.head_target(target, maxlines) def search_bundles(self, keywords, worksheet_uuid=None): bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords) bundle_infos = self.client.get_bundle_infos(bundle_uuids) return bundle_infos def get_worksheet_bundles(self, worksheet_uuid): worksheet_info = self.client.get_worksheet_info(worksheet_uuid, True, True) bundle_info_list = [] for (bundle_info, subworksheet_info, value_obj, item_type) in worksheet_info['items']: if item_type == worksheet_util.TYPE_BUNDLE: bundle_info_list.append(bundle_info) return bundle_info_list def worksheets(self): return _call_with_retries(lambda: self.client.list_worksheets()) def search_worksheets(self, keywords, worksheet_uuid=None): return _call_with_retries(lambda: self.client.search_worksheets(keywords)) def create_worksheet(self, name): return _call_with_retries(lambda: self.client.new_worksheet(name, None)) def get_worksheet_uuid(self, spec): uuid = None spec = smart_str(spec) # generic clean up just in case try: if(spec_util.UUID_REGEX.match(spec)): # generic function sometimes get uuid already just return it. uuid = spec else: uuid = worksheet_util.get_worksheet_uuid(self.client, None, spec) except UsageError, e: #TODO handle Found multiple worksheets with name raise e return uuid
class RemoteBundleService(object): ''' Adapts the RemoteBundleClient for REST calls. TODO(klopyrev): This version should eventually go away once the file upload logic is cleaned up. See below where this class is used for more information. ''' def __init__(self): self.client = RemoteBundleClient(self._cli_url(), lambda command: self._get_user_token(), verbose=0) def _cli_url(self): return 'http://' + local.config['server']['host'] + ':' + str(local.config['server']['port']) def _get_user_token(self): """ Returns an access token for the user. This function facilitates interactions with the bundle service. """ CLIENT_ID = 'codalab_cli_client' if request.user is None: return None # Try to find an existing token that will work. token = local.model.find_oauth2_token( CLIENT_ID, request.user.user_id, datetime.utcnow() + timedelta(minutes=5)) if token is not None: return token.access_token # Otherwise, generate a new one. token = OAuth2Token( local.model, access_token=generate_token(), refresh_token=None, scopes='', expires=datetime.utcnow() + timedelta(hours=10), client_id=CLIENT_ID, user_id=request.user.user_id, ) local.model.save_oauth2_token(token) return token.access_token def upload_bundle(self, source_file, bundle_type, worksheet_uuid): """ Upload |source_file| (a stream) to |worksheet_uuid|. """ # Construct info for creating the bundle. bundle_subclass = get_bundle_subclass(bundle_type) # program or data metadata = metadata_util.fill_missing_metadata(bundle_subclass, {}, initial_metadata={'name': source_file.filename, 'description': 'Upload ' + source_file.filename}) info = {'bundle_type': bundle_type, 'metadata': metadata} # Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI). remote_file_uuid = self.client.open_temp_file(metadata['name']) try: with closing(RPCFileHandle(remote_file_uuid, self.client.proxy)) as dest: file_util.copy(source_file.file, dest, autoflush=False, print_status='Uploading %s' % metadata['name']) pack = False # For now, always unpack (note: do this after set remote_file_uuid, which needs the extension) if not pack and zip_util.path_is_archive(metadata['name']): metadata['name'] = zip_util.strip_archive_ext(metadata['name']) # Then tell the client that the uploaded file handle is there. new_bundle_uuid = self.client.finish_upload_bundle( [remote_file_uuid], not pack, # unpack info, worksheet_uuid, True) # add_to_worksheet except: self.client.finalize_file(remote_file_uuid) raise return new_bundle_uuid
class BundleService(object): # Maximum number of lines of files to show HEAD_MAX_LINES = 100 def __init__(self, user=None): self.client = RemoteBundleClient( settings.BUNDLE_SERVICE_URL, lambda command: get_user_token(user), verbose=1) def items(self): return _call_with_retries(lambda: self.client.search()) def get_bundle_info(self, uuid): bundle_info = _call_with_retries( lambda: self.client.get_bundle_info(uuid, True, True, True)) if bundle_info is None: return None # Set permissions bundle_info['edit_permission'] = ( bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL) # Format permissions into strings bundle_info['permission_str'] = permission_str( bundle_info['permission']) for group_permission in bundle_info['group_permissions']: group_permission['permission_str'] = permission_str( group_permission['permission']) metadata = bundle_info['metadata'] cls = get_bundle_subclass(bundle_info['bundle_type']) for key, value in worksheet_util.get_formatted_metadata( cls, metadata): metadata[key] = value bundle_info['metadata'] = metadata bundle_info[ 'editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields( cls, metadata) return bundle_info def head_target(self, target, max_num_lines=HEAD_MAX_LINES): return self.client.head_target(target, max_num_lines) def search_bundles(self, keywords, worksheet_uuid=None): bundle_uuids = self.client.search_bundle_uuids( worksheet_uuid, keywords) bundle_infos = self.client.get_bundle_infos(bundle_uuids) return bundle_infos def get_worksheet_bundles(self, worksheet_uuid): worksheet_info = self.client.get_worksheet_info( worksheet_uuid, True, True) bundle_info_list = [] for (bundle_info, subworksheet_info, value_obj, item_type) in worksheet_info['items']: if item_type == worksheet_util.TYPE_BUNDLE: bundle_info_list.append(bundle_info) return bundle_info_list def worksheets(self): return _call_with_retries(lambda: self.client.list_worksheets()) def search_worksheets(self, keywords, worksheet_uuid=None): return _call_with_retries( lambda: self.client.search_worksheets(keywords)) def create_worksheet(self, name): return _call_with_retries( lambda: self.client.new_worksheet(name, None)) def get_worksheet_uuid(self, spec): # generic clean up just in case spec = smart_str(spec) # generic function sometimes get uuid already just return it. if spec_util.UUID_REGEX.match(spec): return spec else: return worksheet_util.get_worksheet_uuid( self.client, None, spec) def basic_worksheet(self, uuid): return self.worksheet(uuid, fetch_items=False, get_permissions=True, interpreted=False) def full_worksheet(self, uuid): return self.worksheet(uuid, fetch_items=True, get_permissions=True, interpreted=True) def worksheet(self, uuid, fetch_items, get_permissions, interpreted): """ Return information about a worksheet. Calls - get_worksheet_info: get basic info - resolve_interpreted_items: get more information about a worksheet. In the future, for large worksheets, might want to break this up so that we can render something basic. """ worksheet_info = self.client.get_worksheet_info( uuid, fetch_items, get_permissions) if fetch_items: worksheet_info['raw'] = worksheet_util.get_worksheet_lines( worksheet_info) # Set permissions worksheet_info['edit_permission'] = ( worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL) # Format permissions into strings worksheet_info['permission_str'] = permission_str( worksheet_info['permission']) for group_permission in worksheet_info['group_permissions']: group_permission['permission_str'] = permission_str( group_permission['permission']) # Go and fetch more information about the worksheet contents by # resolving the interpreted items. if interpreted: try: interpreted_items = worksheet_util.interpret_items( worksheet_util.get_default_schemas(), worksheet_info['items']) except UsageError, e: interpreted_items = {'items': []} worksheet_info['error'] = str(e) worksheet_info[ 'items'] = self.client.resolve_interpreted_items( interpreted_items['items']) # Currently, only certain fields are base64 encoded. for item in worksheet_info['items']: if item['mode'] in ['html', 'contents']: if item['interpreted'] is None: item['interpreted'] = [ formatting.contents_str(item['interpreted']) ] else: item['interpreted'] = map(base64.b64decode, item['interpreted']) elif item['mode'] == 'table': for row_map in item['interpreted'][1]: for k, v in row_map.iteritems(): if v is None: row_map[k] = formatting.contents_str(v) elif 'bundle_info' in item: infos = [] if isinstance(item['bundle_info'], list): infos = item['bundle_info'] elif isinstance(item['bundle_info'], dict): infos = [item['bundle_info']] for bundle_info in infos: try: if isinstance(bundle_info, dict): worksheet_util.format_metadata( bundle_info.get('metadata')) except Exception, e: print e import ipdb ipdb.set_trace()
class BundleService(): # Maximum number of lines of files to show HEAD_MAX_LINES = 100 def __init__(self, user=None): self.client = RemoteBundleClient( settings.BUNDLE_SERVICE_URL, lambda command: get_user_token(user), verbose=1) def items(self): return _call_with_retries(lambda: self.client.search()) def get_bundle_info(self, uuid): bundle_info = _call_with_retries( lambda: self.client.get_bundle_info(uuid, True, True, True)) # Set permissions bundle_info['edit_permission'] = ( bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL) # Format permissions into strings bundle_info['permission_str'] = permission_str( bundle_info['permission']) for group_permission in bundle_info['group_permissions']: group_permission['permission_str'] = permission_str( group_permission['permission']) metadata = bundle_info['metadata'] cls = get_bundle_subclass(bundle_info['bundle_type']) for key, value in worksheet_util.get_formatted_metadata( cls, metadata): metadata[key] = value bundle_info['metadata'] = metadata bundle_info[ 'editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields( cls, metadata) return bundle_info def head_target(self, target, maxlines=100): return self.client.head_target(target, maxlines) def search_bundles(self, keywords, worksheet_uuid=None): bundle_uuids = self.client.search_bundle_uuids( worksheet_uuid, keywords) bundle_infos = self.client.get_bundle_infos(bundle_uuids) return bundle_infos def get_worksheet_bundles(self, worksheet_uuid): worksheet_info = self.client.get_worksheet_info( worksheet_uuid, True, True) bundle_info_list = [] for (bundle_info, subworksheet_info, value_obj, item_type) in worksheet_info['items']: if item_type == worksheet_util.TYPE_BUNDLE: bundle_info_list.append(bundle_info) return bundle_info_list def worksheets(self): return _call_with_retries(lambda: self.client.list_worksheets()) def search_worksheets(self, keywords, worksheet_uuid=None): return _call_with_retries( lambda: self.client.search_worksheets(keywords)) def create_worksheet(self, name): return _call_with_retries( lambda: self.client.new_worksheet(name, None)) def get_worksheet_uuid(self, spec): uuid = None spec = smart_str(spec) # generic clean up just in case try: if ( spec_util.UUID_REGEX.match(spec) ): # generic function sometimes get uuid already just return it. uuid = spec else: uuid = worksheet_util.get_worksheet_uuid( self.client, None, spec) except UsageError, e: #TODO handle Found multiple worksheets with name raise e return uuid
class BundleService(): def __init__(self, user=None): self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL, lambda command: get_user_token(user), verbose=1) def items(self): return _call_with_retries(lambda: self.client.search()) def get_bundle_info(self, uuid): ## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False): bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True)) # format permission data bundle_info['permission_str'] = permission_str(bundle_info['permission']) # format each groups as well for group_permission in bundle_info['group_permissions']: group_permission['permission_str'] = permission_str(group_permission['permission']) metadata = bundle_info['metadata'] cls = get_bundle_subclass(bundle_info['bundle_type']) # format based on specs from the cli for spec in cls.METADATA_SPECS: key = spec.key if key not in metadata: continue if metadata[key] == '' or metadata[key] == []: continue value = worksheet_util.apply_func(spec.formatting, metadata.get(key)) # if isinstance(value, list): # value = ', '.join(value) metadata[key] = value bundle_info['metadata'] = metadata return bundle_info def head_target(self, target, maxlines=100): return self.client.head_target(target, maxlines) def search_bundles(self, keywords, worksheet_uuid=None): bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords) bundle_infos = self.client.get_bundle_infos(bundle_uuids) return bundle_infos def worksheets(self): return _call_with_retries(lambda: self.client.list_worksheets()) def search_worksheets(self, keywords, worksheet_uuid=None): return _call_with_retries(lambda: self.client.search_worksheets(keywords)) def create_worksheet(self, name): return _call_with_retries(lambda: self.client.new_worksheet(name, None)) def worksheet(self, uuid, interpreted=False): try: worksheet_info = self.client.get_worksheet_info( uuid, True, #fetch_items True, # get_permissions ) except PermissionError: raise UsageError # forces a not found worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info) # set permissions worksheet_info['edit_permission'] = False if worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL: worksheet_info['edit_permission'] = True worksheet_info['permission_str'] = permission_str(worksheet_info['permission']) # format each groups as well for group_permission in worksheet_info['group_permissions']: group_permission['permission_str'] = permission_str(group_permission['permission']) if interpreted: interpreted_items = worksheet_util.interpret_items( worksheet_util.get_default_schemas(), worksheet_info['items'] ) worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items']) return worksheet_info else: return worksheet_info def create_run_bundle(self, args, worksheet_uuid): cli = self._create_cli(worksheet_uuid) parser = cli.create_parser('run') parser.add_argument('target_spec', help=cli.TARGET_SPEC_FORMAT, nargs='*') parser.add_argument('command', help='Command-line') metadata_util.add_arguments(RunBundle, set(), parser) metadata_util.add_edit_argument(parser) args = parser.parse_args(args) metadata = metadata_util.request_missing_metadata(RunBundle, args) targets = cli.parse_key_targets(self.client, worksheet_uuid, args.target_spec) new_bundle_uuid = self.client.derive_bundle('run', targets, args.command, metadata, worksheet_uuid) return new_bundle_uuid def upload_bundle_url(self, url, info, worksheet_uuid): file_name = url.split("/")[-1] info = { 'bundle_type': 'dataset', 'metadata': { 'description': 'Upload %s' % url, 'tags': [], 'name': '%s' % file_name, 'license': '', 'source_url': '%s' % url, } } new_bundle_uuid = self.client.upload_bundle_url(url, info, worksheet_uuid, True) return new_bundle_uuid def add_worksheet_item(self, worksheet_uuid, bundle_uuid): self.client.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle_uuid)) def parse_and_update_worksheet(self, uuid, lines): worksheet_info = self.client.get_worksheet_info(uuid, True) new_items, commands = worksheet_util.parse_worksheet_form(lines, self.client, worksheet_info['uuid']) self.client.update_worksheet( worksheet_info, new_items ) def get_target_info(self, target, depth=1): return _call_with_retries(lambda: self.client.get_target_info(target, depth)) def resolve_interpreted_items(self, interpreted_items): return _call_with_retries(lambda: self.client.resolve_interpreted_items(('test', 'test'))) def get_worksheet_info(self): return _call_with_retries(lambda: self.client.get_worksheet_info()) def delete_worksheet(self, worksheet_uuid): return _call_with_retries(lambda: self.client.delete_worksheet(worksheet_uuid)) # Create an instance of a CLI. def _create_cli(self, worksheet_uuid): manager = CodaLabManager(temporary=True, clients={settings.BUNDLE_SERVICE_URL: self.client}) manager.set_current_worksheet_uuid(self.client, worksheet_uuid) cli = bundle_cli.BundleCLI(manager, headless=True) return cli def general_command(self, worksheet_uuid, command): cli = self._create_cli(worksheet_uuid) args = worksheet_util.string_to_tokens(command) def do_command(): from cStringIO import StringIO import sys real_stdout = sys.stdout sys.stdout = StringIO() stdout_str = None #real_stderr = sys.stderr #sys.stderr = StringIO() stderr_str = None exception = None try: cli.do_command(args) success = True except BaseException as e: # To capture SystemExit exception = e success = False stdout_str = sys.stdout.getvalue() sys.stdout.close() sys.stdout = real_stdout #stderr_str = sys.stderr.getvalue() #sys.stderr.close() #sys.stderr = real_stderr print '>>> general_command on worksheet %s: %s' % (worksheet_uuid, command) print stdout_str print stderr_str return {'stdout': stdout_str, 'stderr': stderr_str, 'exception': str(exception) if exception else None} return _call_with_retries(do_command) MAX_BYTES = 1024*1024 def read_file(self, uuid, path): fid = self.client.open_target((uuid, path)) try: while True: bytes = self.client.read_file(fid, BundleService.MAX_BYTES) yield bytes.data if len(bytes.data) < BundleService.MAX_BYTES: break finally: self.client.close_file(fid) def download_target(self, uuid, return_zip=False): target = (uuid, '') result_path, container_path = self.client.download_target(target=target, follow_symlinks=True, return_zip=return_zip) return (result_path, container_path) def http_status_from_exception(self, ex): # This is brittle. See https://github.com/codalab/codalab/issues/345. if type(ex) == UsageError: return 404 return 500 def update_bundle_metadata(self, uuid, new_metadata): self.client.update_bundle_metadata(uuid, new_metadata) return
class BundleService(): def __init__(self, user=None): self.client = RemoteBundleClient( settings.BUNDLE_SERVICE_URL, lambda command: get_user_token(user), verbose=1) def items(self): return _call_with_retries(lambda: self.client.search()) def get_bundle_info(self, uuid): ## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False): bundle_info = _call_with_retries( lambda: self.client.get_bundle_info(uuid, True, True, True)) # format permission data bundle_info['permission_str'] = permission_str( bundle_info['permission']) # format each groups as well for group_permission in bundle_info['group_permissions']: group_permission['permission_str'] = permission_str( group_permission['permission']) metadata = bundle_info['metadata'] cls = get_bundle_subclass(bundle_info['bundle_type']) # format based on specs from the cli for spec in cls.METADATA_SPECS: key = spec.key if key not in metadata: continue if metadata[key] == '' or metadata[key] == []: continue value = worksheet_util.apply_func(spec.formatting, metadata.get(key)) # if isinstance(value, list): # value = ', '.join(value) metadata[key] = value bundle_info['metadata'] = metadata return bundle_info def head_target(self, target, maxlines=100): return self.client.head_target(target, maxlines) def search_bundles(self, keywords, worksheet_uuid=None): bundle_uuids = self.client.search_bundle_uuids( worksheet_uuid, keywords) bundle_infos = self.client.get_bundle_infos(bundle_uuids) return bundle_infos def worksheets(self): return _call_with_retries(lambda: self.client.list_worksheets()) def search_worksheets(self, keywords, worksheet_uuid=None): return _call_with_retries( lambda: self.client.search_worksheets(keywords)) def create_worksheet(self, name): return _call_with_retries( lambda: self.client.new_worksheet(name, None)) def worksheet(self, uuid, interpreted=False): try: worksheet_info = self.client.get_worksheet_info( uuid, True, #fetch_items True, # get_permissions ) except PermissionError: raise UsageError # forces a not found worksheet_info['raw'] = worksheet_util.get_worksheet_lines( worksheet_info) # set permissions worksheet_info['edit_permission'] = False if worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL: worksheet_info['edit_permission'] = True worksheet_info['permission_str'] = permission_str( worksheet_info['permission']) # format each groups as well for group_permission in worksheet_info['group_permissions']: group_permission['permission_str'] = permission_str( group_permission['permission']) if interpreted: interpreted_items = worksheet_util.interpret_items( worksheet_util.get_default_schemas(), worksheet_info['items']) worksheet_info[ 'items'] = self.client.resolve_interpreted_items( interpreted_items['items']) # Currently, only certain fields are base64 encoded. for item in worksheet_info['items']: if item['mode'] in ['html', 'contents']: # item['name'] in ['stdout', 'stderr'] item['interpreted'] = map(base64.b64decode, item['interpreted']) elif 'bundle_info' in item: for bundle_info in item['bundle_info']: try: ## sometimes bundle_info is a string. when item['mode'] is image if isinstance( bundle_info, dict) and bundle_info.get( 'bundle_type', None) == 'run': if 'stdout' in bundle_info.keys(): bundle_info[ 'stdout'] = base64.b64decode( bundle_info['stdout']) if 'stderr' in bundle_info.keys(): bundle_info[ 'stderr'] = base64.b64decode( bundle_info['stderr']) except Exception, e: print e import ipdb ipdb.set_trace() return worksheet_info
def __init__(self): self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL, lambda command: "") #TODO
class BundleService(): def __init__(self, user=None): self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL, lambda command: get_user_token(user), verbose=1) def items(self): return _call_with_retries(lambda: self.client.search()) def item(self, uuid): return _call_with_retries(lambda: self.client.get_bundle_info(uuid)) def worksheets(self): return _call_with_retries(lambda: self.client.list_worksheets()) def create_worksheet(self, name): return _call_with_retries(lambda: self.client.new_worksheet(name)) def worksheet(self, uuid, interpreted=False): worksheet_info = _call_with_retries( lambda: self.client.get_worksheet_info( uuid, True ) ) if interpreted: interpreted_items = worksheet_util.interpret_items( worksheet_util.get_default_schemas(), worksheet_info['items'] ) worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items']) return worksheet_info else: return worksheet_info def ls(self, uuid, path): return _call_with_retries(lambda: self.client.ls((uuid, path))) MAX_BYTES = 1024*1024 def read_file(self, uuid, path): fid = self.client.open_target((uuid, path)) try: while True: bytes = self.client.read_file(fid, BundleService.MAX_BYTES) yield bytes.data if len(bytes.data) < BundleService.MAX_BYTES: break finally: self.client.close_file(fid) def download_target(self, uuid, return_zip=False): target = (uuid, '') result_path, container_path = self.client.download_target(target=target, follow_symlinks=True, return_zip=return_zip) return (result_path, container_path) def http_status_from_exception(self, ex): # This is brittle. See https://github.com/codalab/codalab/issues/345. if type(ex) == UsageError: return 404 return 500
def __init__(self, user=None): self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL, lambda command: get_user_token(user), verbose=1)
class BundleService(): def __init__(self, user=None): self.client = RemoteBundleClient( settings.BUNDLE_SERVICE_URL, lambda command: get_user_token(user), verbose=1) def items(self): return _call_with_retries(lambda: self.client.search()) def item(self, uuid): return _call_with_retries( lambda: self.client.get_bundle_info(uuid)) def worksheets(self): return _call_with_retries(lambda: self.client.list_worksheets()) def create_worksheet(self, name): return _call_with_retries(lambda: self.client.new_worksheet(name)) def worksheet(self, uuid, interpreted=False): worksheet_info = _call_with_retries( lambda: self.client.get_worksheet_info(uuid, True)) if interpreted: interpreted_items = worksheet_util.interpret_items( worksheet_util.get_default_schemas(), worksheet_info['items']) worksheet_info[ 'items'] = self.client.resolve_interpreted_items( interpreted_items['items']) return worksheet_info else: return worksheet_info def ls(self, uuid, path): return _call_with_retries(lambda: self.client.ls((uuid, path))) MAX_BYTES = 1024 * 1024 def read_file(self, uuid, path): fid = self.client.open_target((uuid, path)) try: while True: bytes = self.client.read_file(fid, BundleService.MAX_BYTES) yield bytes.data if len(bytes.data) < BundleService.MAX_BYTES: break finally: self.client.close_file(fid) def download_target(self, uuid, return_zip=False): target = (uuid, '') result_path, container_path = self.client.download_target( target=target, follow_symlinks=True, return_zip=return_zip) return (result_path, container_path) def http_status_from_exception(self, ex): # This is brittle. See https://github.com/codalab/codalab/issues/345. if type(ex) == UsageError: return 404 return 500
def __init__(self): self.client = RemoteBundleClient(self._cli_url(), lambda command: self._get_user_token(), verbose=0)
class BundleService(): def __init__(self, user=None): self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL, lambda command: get_user_token(user), verbose=1) def items(self): return _call_with_retries(lambda: self.client.search()) def get_bundle_info(self, uuid): bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid)) metadata = bundle_info['metadata'] cls = get_bundle_subclass(bundle_info['bundle_type']) # format based on specs from the cli for spec in cls.METADATA_SPECS: key = spec.key if key not in metadata: continue if metadata[key] == '' or metadata[key] == []: continue value = worksheet_util.apply_func(spec.formatting, metadata.get(key)) # if isinstance(value, list): # value = ', '.join(value) metadata[key] = value bundle_info['metadata'] = metadata return bundle_info def search_bundles(self, keywords, worksheet_uuid=None): # search_bundle_uuids(worksheet_uuid, keywords, max_results, show_counts_only) bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords, 30, False) bundle_infos = self.client.get_bundle_infos(bundle_uuids) return bundle_infos def worksheets(self): return _call_with_retries(lambda: self.client.list_worksheets()) def create_worksheet(self, name): return _call_with_retries(lambda: self.client.new_worksheet(name)) def worksheet(self, uuid, interpreted=False): try: worksheet_info = self.client.get_worksheet_info( uuid, True, #fetch_items True, # get_permissions ) except PermissionError: raise UsageError # forces a not found worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info) # set permissions worksheet_info['edit_permission'] = False if worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL: worksheet_info['edit_permission'] = True if interpreted: interpreted_items = worksheet_util.interpret_items( worksheet_util.get_default_schemas(), worksheet_info['items'] ) worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items']) return worksheet_info else: return worksheet_info def derive_bundle(self, bundle_type, targets, worksheet_uuid, command): name = str(slugify(command)) metadata = {'name': name, 'tags': [], 'allowed_time': u'', 'allowed_memory': u'', 'allowed_disk': u'', 'description': ''} new_bundle_uuid = self.client.derive_bundle(bundle_type, targets, command, metadata, worksheet_uuid) return new_bundle_uuid def upload_bundle_url(self, url, info, worksheet_uuid): file_name = url.split("/")[-1] info = { 'bundle_type': 'dataset', 'metadata': { 'description': 'Upload %s' % url, 'tags': [], 'name': '%s' % file_name, 'license': '', 'source_url': '%s' % url, } } new_bundle_uuid = self.client.upload_bundle_url(url, info, worksheet_uuid, True) return new_bundle_uuid def add_worksheet_item(self, worksheet_uuid, bundle_uuid): self.client.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle_uuid)) def parse_and_update_worksheet(self, uuid, lines): worksheet_info = self.client.get_worksheet_info(uuid, True) new_items, commands = worksheet_util.parse_worksheet_form(lines, self.client, worksheet_info['uuid']) self.client.update_worksheet( worksheet_info, new_items ) def get_target_info(self, target, depth=1): return _call_with_retries(lambda: self.client.get_target_info(target, depth)) def resolve_interpreted_items(self, interpreted_items): return _call_with_retries(lambda: self.client.resolve_interpreted_items(('test', 'test'))) def get_worksheet_info(self): return _call_with_retries(lambda: self.client.get_worksheet_info()) def delete_worksheet(self, worksheet_uuid): return _call_with_retries(lambda: self.client.delete_worksheet(worksheet_uuid)) MAX_BYTES = 1024*1024 def read_file(self, uuid, path): fid = self.client.open_target((uuid, path)) try: while True: bytes = self.client.read_file(fid, BundleService.MAX_BYTES) yield bytes.data if len(bytes.data) < BundleService.MAX_BYTES: break finally: self.client.close_file(fid) def download_target(self, uuid, return_zip=False): target = (uuid, '') result_path, container_path = self.client.download_target(target=target, follow_symlinks=True, return_zip=return_zip) return (result_path, container_path) def http_status_from_exception(self, ex): # This is brittle. See https://github.com/codalab/codalab/issues/345. if type(ex) == UsageError: return 404 return 500 def update_bundle_metadata(self, uuid, new_metadata): self.client.update_bundle_metadata(uuid, new_metadata) return
def __init__(self, user=None): self.client = RemoteBundleClient( settings.BUNDLE_SERVICE_URL, lambda command: get_user_token(user), verbose=1)
class BundleService(object): # Maximum number of lines of files to show HEAD_MAX_LINES = 100 def __init__(self, user=None): self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL, lambda command: get_user_token(user), verbose=1) def items(self): return _call_with_retries(lambda: self.client.search()) def get_bundle_info(self, uuid): bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True)) if bundle_info is None: return None # Set permissions bundle_info['edit_permission'] = (bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL) # Format permissions into strings bundle_info['permission_str'] = permission_str(bundle_info['permission']) for group_permission in bundle_info['group_permissions']: group_permission['permission_str'] = permission_str(group_permission['permission']) metadata = bundle_info['metadata'] cls = get_bundle_subclass(bundle_info['bundle_type']) for key, value in worksheet_util.get_formatted_metadata(cls, metadata): metadata[key] = value bundle_info['metadata'] = metadata bundle_info['editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields(cls, metadata) return bundle_info def head_target(self, target, max_num_lines=HEAD_MAX_LINES): return self.client.head_target(target, max_num_lines) def search_bundles(self, keywords, worksheet_uuid=None): bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords) bundle_infos = self.client.get_bundle_infos(bundle_uuids) return bundle_infos def get_worksheet_bundles(self, worksheet_uuid): worksheet_info = self.client.get_worksheet_info(worksheet_uuid, True, True) bundle_info_list = [] for (bundle_info, subworksheet_info, value_obj, item_type) in worksheet_info['items']: if item_type == worksheet_util.TYPE_BUNDLE: bundle_info_list.append(bundle_info) return bundle_info_list def worksheets(self): return _call_with_retries(lambda: self.client.list_worksheets()) def search_worksheets(self, keywords, worksheet_uuid=None): return _call_with_retries(lambda: self.client.search_worksheets(keywords)) def create_worksheet(self, name): return _call_with_retries(lambda: self.client.new_worksheet(name, None)) def get_worksheet_uuid(self, spec): # generic clean up just in case spec = smart_str(spec) # generic function sometimes get uuid already just return it. if spec_util.UUID_REGEX.match(spec): return spec else: return worksheet_util.get_worksheet_uuid(self.client, None, spec) def basic_worksheet(self, uuid): return self.worksheet(uuid, fetch_items=False, get_permissions=True, interpreted=False) def full_worksheet(self, uuid): return self.worksheet(uuid, fetch_items=True, get_permissions=True, interpreted=True) def worksheet(self, uuid, fetch_items, get_permissions, interpreted): """ Return information about a worksheet. Calls - get_worksheet_info: get basic info - resolve_interpreted_items: get more information about a worksheet. In the future, for large worksheets, might want to break this up so that we can render something basic. """ worksheet_info = self.client.get_worksheet_info(uuid, fetch_items, get_permissions) if fetch_items: worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info) # Set permissions worksheet_info['edit_permission'] = (worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL) # Format permissions into strings worksheet_info['permission_str'] = permission_str(worksheet_info['permission']) for group_permission in worksheet_info['group_permissions']: group_permission['permission_str'] = permission_str(group_permission['permission']) # Go and fetch more information about the worksheet contents by # resolving the interpreted items. if interpreted: try: interpreted_items = worksheet_util.interpret_items( worksheet_util.get_default_schemas(), worksheet_info['items']) except UsageError, e: interpreted_items = {'items': []} worksheet_info['error'] = str(e) worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items']) worksheet_info['raw_to_interpreted'] = interpreted_items['raw_to_interpreted'] worksheet_info['interpreted_to_raw'] = interpreted_items['interpreted_to_raw'] def decode_lines(interpreted): # interpreted is None or list of base64 encoded lines if interpreted is None: return formatting.contents_str(None) else: return map(base64.b64decode, interpreted) # Currently, only certain fields are base64 encoded. for item in worksheet_info['items']: if item['mode'] in ['html', 'contents']: item['interpreted'] = decode_lines(item['interpreted']) elif item['mode'] == 'table': for row_map in item['interpreted'][1]: for k, v in row_map.iteritems(): if v is None: row_map[k] = formatting.contents_str(v) elif 'bundle_info' in item: infos = [] if isinstance(item['bundle_info'], list): infos = item['bundle_info'] elif isinstance(item['bundle_info'], dict): infos = [item['bundle_info']] for bundle_info in infos: try: if isinstance(bundle_info, dict): worksheet_util.format_metadata(bundle_info.get('metadata')) except Exception, e: print e import ipdb; ipdb.set_trace()