class BundleService():
def __init__(self):
self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL)
def items(self):
return _call_with_retries(lambda: self.client.search())
def item(self, uuid):
return _call_with_retries(lambda: self.client.info(uuid))
def worksheets(self):
return _call_with_retries(lambda: self.client.list_worksheets())
def worksheet(self, uuid):
return _call_with_retries(lambda: self.client.worksheet_info(uuid))
def ls(self, uuid, path):
return _call_with_retries(lambda: self.client.ls((uuid, path)))
def http_status_from_exception(self, ex):
# This is brittle. See https://github.com/codalab/codalab/issues/345.
if type(ex) == UsageError:
return 404
return 500
class BundleService():
def __init__(self):
self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL)
def items(self):
results = self.client.search()
return results
def item(self, uuid):
result = self.client.info(uuid)
return result
def authenticate(self, address):
'''
Authenticate with the given address. This will prompt user for password
unless valid credentials are already available. Client state will be
updated if new tokens are generated.
Returns an access token.
'''
def _cache_token(token_info, username=None):
'''
Helper to update state with new token info and optional username.
Returns the latest access token.
'''
token_info['expires_at'] = time.time() + float(token_info['expires_in']) - 60.0
del token_info['expires_in']
auth['token_info'] = token_info
if username is not None:
auth['username'] = username
self.save_state()
return token_info['access_token']
# Check the cache for a valid token
from codalab.client.remote_bundle_client import RemoteBundleClient
auth_info = self.state['auth'].get(address, {})
if 'token_info' in auth_info:
token_info = auth_info['token_info']
expires_at = token_info.get('expires_at', 0.0)
if expires_at > time.time():
# Token is usable but check if it's nearing expiration
if expires_at >= (time.time() + 900.0):
return token_info['access_token']
# Try to refresh token
remote_client = RemoteBundleClient(address, lambda command: None)
token_info = remote_client.login('refresh_token',
token_info['refresh_token'],
auth_info['username'])
if token_info is not None:
return _cache_token(token_info)
# If we get here, a valid token is not already available.
auth = self.state['auth'][address] = {}
print 'Requesting access at %s' % address
print 'Username: '******'credentials', username, password)
if token_info is None:
raise UsageError("Invalid username or password")
return _cache_token(token_info, username)
def client(self, address, is_cli=True):
'''
Return a client given the address. Note that this can either be called
by the CLI (is_cli=True) or the server (is_cli=False).
If called by the CLI, we don't need to authenticate.
Cache the Client if necessary.
'''
if address in self.clients:
return self.clients[address]
# if local force mockauth or if locl server use correct auth
if is_local_address(address):
bundle_store = self.bundle_store()
model = self.model()
auth_handler = self.auth_handler(mock=is_cli)
from codalab.client.local_bundle_client import LocalBundleClient
client = LocalBundleClient(address, bundle_store, model,
auth_handler, self.cli_verbose)
self.clients[address] = client
if is_cli:
# Set current user
access_token = self._authenticate(client)
auth_handler.validate_token(access_token)
else:
from codalab.client.remote_bundle_client import RemoteBundleClient
client = RemoteBundleClient(
address, lambda a_client: self._authenticate(a_client),
self.cli_verbose())
self.clients[address] = client
self._authenticate(client)
return client
class BundleService():
def __init__(self):
self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
lambda command: "") #TODO
def items(self):
return _call_with_retries(lambda: self.client.search())
def item(self, uuid):
return _call_with_retries(lambda: self.client.info(uuid))
def worksheets(self):
return _call_with_retries(lambda: self.client.list_worksheets())
def create_worksheet(self, name):
return _call_with_retries(lambda: self.client.new_worksheet(name))
def worksheet(self, uuid):
return _call_with_retries(lambda: self.client.worksheet_info(uuid))
def ls(self, uuid, path):
strm = self.read_file(uuid, 'bat.txt')
return _call_with_retries(lambda: self.client.ls((uuid, path)))
MAX_BYTES = 1024 * 1024
def read_file(self, uuid, path):
fid = self.client.open_target((uuid, path))
try:
while True:
bytes = self.client.read_file(fid, BundleService.MAX_BYTES)
yield bytes.data
if len(bytes.data) < BundleService.MAX_BYTES:
break
finally:
self.client.close_file(fid)
def http_status_from_exception(self, ex):
# This is brittle. See https://github.com/codalab/codalab/issues/345.
if type(ex) == UsageError:
return 404
return 500
class BundleService():
def __init__(self):
self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
lambda command: "") #TODO
def items(self):
return _call_with_retries(lambda: self.client.search())
def item(self, uuid):
return _call_with_retries(lambda: self.client.info(uuid))
def worksheets(self):
return _call_with_retries(lambda: self.client.list_worksheets())
def create_worksheet(self, name):
return _call_with_retries(lambda: self.client.new_worksheet(name))
def worksheet(self, uuid):
return _call_with_retries(lambda: self.client.worksheet_info(uuid))
def ls(self, uuid, path):
strm = self.read_file(uuid, 'bat.txt')
return _call_with_retries(lambda: self.client.ls((uuid, path)))
MAX_BYTES = 1024*1024
def read_file(self, uuid, path):
fid = self.client.open_target((uuid, path))
try:
while True:
bytes = self.client.read_file(fid, BundleService.MAX_BYTES)
yield bytes.data
if len(bytes.data) < BundleService.MAX_BYTES:
break
finally:
self.client.close_file(fid)
def http_status_from_exception(self, ex):
# This is brittle. See https://github.com/codalab/codalab/issues/345.
if type(ex) == UsageError:
return 404
return 500
def __init__(self):
self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL)
class BundleService():
# Maximum number of lines of files to show
HEAD_MAX_LINES = 100
def __init__(self, user=None):
self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
lambda command: get_user_token(user), verbose=1)
def items(self):
return _call_with_retries(lambda: self.client.search())
def get_bundle_info(self, uuid):
bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))
# Set permissions
bundle_info['edit_permission'] = (bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
# Format permissions into strings
bundle_info['permission_str'] = permission_str(bundle_info['permission'])
for group_permission in bundle_info['group_permissions']:
group_permission['permission_str'] = permission_str(group_permission['permission'])
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
for key, value in worksheet_util.get_formatted_metadata(cls, metadata):
metadata[key] = value
bundle_info['metadata'] = metadata
bundle_info['editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields(cls, metadata)
return bundle_info
def head_target(self, target, maxlines=100):
return self.client.head_target(target, maxlines)
def search_bundles(self, keywords, worksheet_uuid=None):
bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords)
bundle_infos = self.client.get_bundle_infos(bundle_uuids)
return bundle_infos
def get_worksheet_bundles(self, worksheet_uuid):
worksheet_info = self.client.get_worksheet_info(worksheet_uuid, True, True)
bundle_info_list = []
for (bundle_info, subworksheet_info, value_obj, item_type) in worksheet_info['items']:
if item_type == worksheet_util.TYPE_BUNDLE:
bundle_info_list.append(bundle_info)
return bundle_info_list
def worksheets(self):
return _call_with_retries(lambda: self.client.list_worksheets())
def search_worksheets(self, keywords, worksheet_uuid=None):
return _call_with_retries(lambda: self.client.search_worksheets(keywords))
def create_worksheet(self, name):
return _call_with_retries(lambda: self.client.new_worksheet(name, None))
def get_worksheet_uuid(self, spec):
uuid = None
spec = smart_str(spec) # generic clean up just in case
try:
if(spec_util.UUID_REGEX.match(spec)): # generic function sometimes get uuid already just return it.
uuid = spec
else:
uuid = worksheet_util.get_worksheet_uuid(self.client, None, spec)
except UsageError, e:
#TODO handle Found multiple worksheets with name
raise e
return uuid
class BundleService():
def __init__(self, user=None):
self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
lambda command: get_user_token(user), verbose=1)
def items(self):
return _call_with_retries(lambda: self.client.search())
def get_bundle_info(self, uuid):
## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False):
bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))
# format permission data
bundle_info['permission_str'] = permission_str(bundle_info['permission'])
# format each groups as well
for group_permission in bundle_info['group_permissions']:
group_permission['permission_str'] = permission_str(group_permission['permission'])
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
# format based on specs from the cli
for spec in cls.METADATA_SPECS:
key = spec.key
if key not in metadata:
continue
if metadata[key] == '' or metadata[key] == []:
continue
value = worksheet_util.apply_func(spec.formatting, metadata.get(key))
# if isinstance(value, list):
# value = ', '.join(value)
metadata[key] = value
bundle_info['metadata'] = metadata
return bundle_info
def head_target(self, target, maxlines=100):
return self.client.head_target(target, maxlines)
def search_bundles(self, keywords, worksheet_uuid=None):
bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords)
bundle_infos = self.client.get_bundle_infos(bundle_uuids)
return bundle_infos
def worksheets(self):
return _call_with_retries(lambda: self.client.list_worksheets())
def search_worksheets(self, keywords, worksheet_uuid=None):
return _call_with_retries(lambda: self.client.search_worksheets(keywords))
def create_worksheet(self, name):
return _call_with_retries(lambda: self.client.new_worksheet(name, None))
def worksheet(self, uuid, interpreted=False):
try:
worksheet_info = self.client.get_worksheet_info(
uuid,
True, #fetch_items
True, # get_permissions
)
except PermissionError:
raise UsageError # forces a not found
worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info)
# set permissions
worksheet_info['edit_permission'] = False
if worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL:
worksheet_info['edit_permission'] = True
worksheet_info['permission_str'] = permission_str(worksheet_info['permission'])
# format each groups as well
for group_permission in worksheet_info['group_permissions']:
group_permission['permission_str'] = permission_str(group_permission['permission'])
if interpreted:
interpreted_items = worksheet_util.interpret_items(
worksheet_util.get_default_schemas(),
worksheet_info['items']
)
worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items'])
# Currently, only certain fields are base64 encoded.
for item in worksheet_info['items']:
if item['mode'] in ['html', 'contents']:
# item['name'] in ['stdout', 'stderr']
item['interpreted'] = map(base64.b64decode, item['interpreted'])
elif 'bundle_info' in item:
for bundle_info in item['bundle_info']:
try:
## sometimes bundle_info is a string. when item['mode'] is image
if isinstance(bundle_info, dict) and bundle_info.get('bundle_type', None) == 'run':
if 'stdout' in bundle_info.keys():
bundle_info['stdout'] = base64.b64decode(bundle_info['stdout'])
if 'stderr' in bundle_info.keys():
bundle_info['stderr'] = base64.b64decode(bundle_info['stderr'])
except Exception, e:
print e
import ipdb; ipdb.set_trace()
return worksheet_info
def __init__(self):
self.client = RemoteBundleClient(
self._cli_url(), lambda command: self._get_user_token(), verbose=0)
class RemoteBundleService(object):
'''
Adapts the RemoteBundleClient for REST calls.
TODO(klopyrev): This version should eventually go away once the file upload
logic is cleaned up. See below where this class is used for more information.
'''
def __init__(self):
self.client = RemoteBundleClient(
self._cli_url(), lambda command: self._get_user_token(), verbose=0)
def _cli_url(self):
return 'http://' + local.config['server']['host'] + ':' + str(
local.config['server']['port'])
def _get_user_token(self):
"""
Returns an access token for the user. This function facilitates interactions
with the bundle service.
"""
CLIENT_ID = 'codalab_cli_client'
if request.user is None:
return None
# Try to find an existing token that will work.
token = local.model.find_oauth2_token(
CLIENT_ID, request.user.user_id,
datetime.utcnow() + timedelta(minutes=5))
if token is not None:
return token.access_token
# Otherwise, generate a new one.
token = OAuth2Token(
local.model,
access_token=generate_token(),
refresh_token=None,
scopes='',
expires=datetime.utcnow() + timedelta(hours=10),
client_id=CLIENT_ID,
user_id=request.user.user_id,
)
local.model.save_oauth2_token(token)
return token.access_token
def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
"""
Upload |source_file| (a stream) to |worksheet_uuid|.
"""
# Construct info for creating the bundle.
bundle_subclass = get_bundle_subclass(bundle_type) # program or data
metadata = metadata_util.fill_missing_metadata(
bundle_subclass, {},
initial_metadata={
'name': source_file.filename,
'description': 'Upload ' + source_file.filename
})
info = {'bundle_type': bundle_type, 'metadata': metadata}
# Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
remote_file_uuid = self.client.open_temp_file(metadata['name'])
try:
with closing(RPCFileHandle(remote_file_uuid,
self.client.proxy)) as dest:
file_util.copy(source_file.file,
dest,
autoflush=False,
print_status='Uploading %s' % metadata['name'])
pack = False # For now, always unpack (note: do this after set remote_file_uuid, which needs the extension)
if not pack and zip_util.path_is_archive(metadata['name']):
metadata['name'] = zip_util.strip_archive_ext(metadata['name'])
# Then tell the client that the uploaded file handle is there.
new_bundle_uuid = self.client.finish_upload_bundle(
[remote_file_uuid],
not pack, # unpack
info,
worksheet_uuid,
True) # add_to_worksheet
except:
self.client.finalize_file(remote_file_uuid)
raise
return new_bundle_uuid
class BundleService():
def __init__(self, user=None):
self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
lambda command: get_user_token(user), verbose=1)
def items(self):
return _call_with_retries(lambda: self.client.search())
def get_bundle_info(self, uuid):
## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False):
bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))
# format permission data
bundle_info['permission_str'] = permission_str(bundle_info['permission'])
# format each groups as well
for group_permission in bundle_info['group_permissions']:
group_permission['permission_str'] = permission_str(group_permission['permission'])
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
# format based on specs from the cli
for spec in cls.METADATA_SPECS:
key = spec.key
if key not in metadata:
continue
if metadata[key] == '' or metadata[key] == []:
continue
value = worksheet_util.apply_func(spec.formatting, metadata.get(key))
# if isinstance(value, list):
# value = ', '.join(value)
metadata[key] = value
bundle_info['metadata'] = metadata
return bundle_info
def head_target(self, target, maxlines=100):
return self.client.head_target(target, maxlines)
def search_bundles(self, keywords, worksheet_uuid=None):
bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords)
bundle_infos = self.client.get_bundle_infos(bundle_uuids)
return bundle_infos
def get_worksheet_bundles(self, worksheet_uuid):
worksheet_info = self.client.get_worksheet_info(worksheet_uuid, True, True)
bundle_info_list = []
for (bundle_info, subworksheet_info, value_obj, item_type) in worksheet_info['items']:
if item_type == worksheet_util.TYPE_BUNDLE:
bundle_info_list.append(bundle_info)
return bundle_info_list
def worksheets(self):
return _call_with_retries(lambda: self.client.list_worksheets())
def search_worksheets(self, keywords, worksheet_uuid=None):
return _call_with_retries(lambda: self.client.search_worksheets(keywords))
def create_worksheet(self, name):
return _call_with_retries(lambda: self.client.new_worksheet(name, None))
def get_worksheet_uuid(self, spec):
uuid = None
spec = smart_str(spec) # generic clean up just in case
try:
if(spec_util.UUID_REGEX.match(spec)): # generic function sometimes get uuid already just return it.
uuid = spec
else:
uuid = worksheet_util.get_worksheet_uuid(self.client, None, spec)
except UsageError, e:
#TODO handle Found multiple worksheets with name
raise e
return uuid
class RemoteBundleService(object):
'''
Adapts the RemoteBundleClient for REST calls.
TODO(klopyrev): This version should eventually go away once the file upload
logic is cleaned up. See below where this class is used for more information.
'''
def __init__(self):
self.client = RemoteBundleClient(self._cli_url(),
lambda command: self._get_user_token(), verbose=0)
def _cli_url(self):
return 'http://' + local.config['server']['host'] + ':' + str(local.config['server']['port'])
def _get_user_token(self):
"""
Returns an access token for the user. This function facilitates interactions
with the bundle service.
"""
CLIENT_ID = 'codalab_cli_client'
if request.user is None:
return None
# Try to find an existing token that will work.
token = local.model.find_oauth2_token(
CLIENT_ID,
request.user.user_id,
datetime.utcnow() + timedelta(minutes=5))
if token is not None:
return token.access_token
# Otherwise, generate a new one.
token = OAuth2Token(
local.model,
access_token=generate_token(),
refresh_token=None,
scopes='',
expires=datetime.utcnow() + timedelta(hours=10),
client_id=CLIENT_ID,
user_id=request.user.user_id,
)
local.model.save_oauth2_token(token)
return token.access_token
def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
"""
Upload |source_file| (a stream) to |worksheet_uuid|.
"""
# Construct info for creating the bundle.
bundle_subclass = get_bundle_subclass(bundle_type) # program or data
metadata = metadata_util.fill_missing_metadata(bundle_subclass, {}, initial_metadata={'name': source_file.filename, 'description': 'Upload ' + source_file.filename})
info = {'bundle_type': bundle_type, 'metadata': metadata}
# Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
remote_file_uuid = self.client.open_temp_file(metadata['name'])
try:
with closing(RPCFileHandle(remote_file_uuid, self.client.proxy)) as dest:
file_util.copy(source_file.file, dest, autoflush=False, print_status='Uploading %s' % metadata['name'])
pack = False # For now, always unpack (note: do this after set remote_file_uuid, which needs the extension)
if not pack and zip_util.path_is_archive(metadata['name']):
metadata['name'] = zip_util.strip_archive_ext(metadata['name'])
# Then tell the client that the uploaded file handle is there.
new_bundle_uuid = self.client.finish_upload_bundle(
[remote_file_uuid],
not pack, # unpack
info,
worksheet_uuid,
True) # add_to_worksheet
except:
self.client.finalize_file(remote_file_uuid)
raise
return new_bundle_uuid
class BundleService(object):
# Maximum number of lines of files to show
HEAD_MAX_LINES = 100
def __init__(self, user=None):
self.client = RemoteBundleClient(
settings.BUNDLE_SERVICE_URL,
lambda command: get_user_token(user),
verbose=1)
def items(self):
return _call_with_retries(lambda: self.client.search())
def get_bundle_info(self, uuid):
bundle_info = _call_with_retries(
lambda: self.client.get_bundle_info(uuid, True, True, True))
if bundle_info is None:
return None
# Set permissions
bundle_info['edit_permission'] = (
bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
# Format permissions into strings
bundle_info['permission_str'] = permission_str(
bundle_info['permission'])
for group_permission in bundle_info['group_permissions']:
group_permission['permission_str'] = permission_str(
group_permission['permission'])
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
for key, value in worksheet_util.get_formatted_metadata(
cls, metadata):
metadata[key] = value
bundle_info['metadata'] = metadata
bundle_info[
'editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields(
cls, metadata)
return bundle_info
def head_target(self, target, max_num_lines=HEAD_MAX_LINES):
return self.client.head_target(target, max_num_lines)
def search_bundles(self, keywords, worksheet_uuid=None):
bundle_uuids = self.client.search_bundle_uuids(
worksheet_uuid, keywords)
bundle_infos = self.client.get_bundle_infos(bundle_uuids)
return bundle_infos
def get_worksheet_bundles(self, worksheet_uuid):
worksheet_info = self.client.get_worksheet_info(
worksheet_uuid, True, True)
bundle_info_list = []
for (bundle_info, subworksheet_info, value_obj,
item_type) in worksheet_info['items']:
if item_type == worksheet_util.TYPE_BUNDLE:
bundle_info_list.append(bundle_info)
return bundle_info_list
def worksheets(self):
return _call_with_retries(lambda: self.client.list_worksheets())
def search_worksheets(self, keywords, worksheet_uuid=None):
return _call_with_retries(
lambda: self.client.search_worksheets(keywords))
def create_worksheet(self, name):
return _call_with_retries(
lambda: self.client.new_worksheet(name, None))
def get_worksheet_uuid(self, spec):
# generic clean up just in case
spec = smart_str(spec)
# generic function sometimes get uuid already just return it.
if spec_util.UUID_REGEX.match(spec):
return spec
else:
return worksheet_util.get_worksheet_uuid(
self.client, None, spec)
def basic_worksheet(self, uuid):
return self.worksheet(uuid,
fetch_items=False,
get_permissions=True,
interpreted=False)
def full_worksheet(self, uuid):
return self.worksheet(uuid,
fetch_items=True,
get_permissions=True,
interpreted=True)
def worksheet(self, uuid, fetch_items, get_permissions, interpreted):
"""
Return information about a worksheet. Calls
- get_worksheet_info: get basic info
- resolve_interpreted_items: get more information about a worksheet.
In the future, for large worksheets, might want to break this up so
that we can render something basic.
"""
worksheet_info = self.client.get_worksheet_info(
uuid, fetch_items, get_permissions)
if fetch_items:
worksheet_info['raw'] = worksheet_util.get_worksheet_lines(
worksheet_info)
# Set permissions
worksheet_info['edit_permission'] = (
worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
# Format permissions into strings
worksheet_info['permission_str'] = permission_str(
worksheet_info['permission'])
for group_permission in worksheet_info['group_permissions']:
group_permission['permission_str'] = permission_str(
group_permission['permission'])
# Go and fetch more information about the worksheet contents by
# resolving the interpreted items.
if interpreted:
try:
interpreted_items = worksheet_util.interpret_items(
worksheet_util.get_default_schemas(),
worksheet_info['items'])
except UsageError, e:
interpreted_items = {'items': []}
worksheet_info['error'] = str(e)
worksheet_info[
'items'] = self.client.resolve_interpreted_items(
interpreted_items['items'])
# Currently, only certain fields are base64 encoded.
for item in worksheet_info['items']:
if item['mode'] in ['html', 'contents']:
if item['interpreted'] is None:
item['interpreted'] = [
formatting.contents_str(item['interpreted'])
]
else:
item['interpreted'] = map(base64.b64decode,
item['interpreted'])
elif item['mode'] == 'table':
for row_map in item['interpreted'][1]:
for k, v in row_map.iteritems():
if v is None:
row_map[k] = formatting.contents_str(v)
elif 'bundle_info' in item:
infos = []
if isinstance(item['bundle_info'], list):
infos = item['bundle_info']
elif isinstance(item['bundle_info'], dict):
infos = [item['bundle_info']]
for bundle_info in infos:
try:
if isinstance(bundle_info, dict):
worksheet_util.format_metadata(
bundle_info.get('metadata'))
except Exception, e:
print e
import ipdb
ipdb.set_trace()
class BundleService():
# Maximum number of lines of files to show
HEAD_MAX_LINES = 100
def __init__(self, user=None):
self.client = RemoteBundleClient(
settings.BUNDLE_SERVICE_URL,
lambda command: get_user_token(user),
verbose=1)
def items(self):
return _call_with_retries(lambda: self.client.search())
def get_bundle_info(self, uuid):
bundle_info = _call_with_retries(
lambda: self.client.get_bundle_info(uuid, True, True, True))
# Set permissions
bundle_info['edit_permission'] = (
bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
# Format permissions into strings
bundle_info['permission_str'] = permission_str(
bundle_info['permission'])
for group_permission in bundle_info['group_permissions']:
group_permission['permission_str'] = permission_str(
group_permission['permission'])
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
for key, value in worksheet_util.get_formatted_metadata(
cls, metadata):
metadata[key] = value
bundle_info['metadata'] = metadata
bundle_info[
'editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields(
cls, metadata)
return bundle_info
def head_target(self, target, maxlines=100):
return self.client.head_target(target, maxlines)
def search_bundles(self, keywords, worksheet_uuid=None):
bundle_uuids = self.client.search_bundle_uuids(
worksheet_uuid, keywords)
bundle_infos = self.client.get_bundle_infos(bundle_uuids)
return bundle_infos
def get_worksheet_bundles(self, worksheet_uuid):
worksheet_info = self.client.get_worksheet_info(
worksheet_uuid, True, True)
bundle_info_list = []
for (bundle_info, subworksheet_info, value_obj,
item_type) in worksheet_info['items']:
if item_type == worksheet_util.TYPE_BUNDLE:
bundle_info_list.append(bundle_info)
return bundle_info_list
def worksheets(self):
return _call_with_retries(lambda: self.client.list_worksheets())
def search_worksheets(self, keywords, worksheet_uuid=None):
return _call_with_retries(
lambda: self.client.search_worksheets(keywords))
def create_worksheet(self, name):
return _call_with_retries(
lambda: self.client.new_worksheet(name, None))
def get_worksheet_uuid(self, spec):
uuid = None
spec = smart_str(spec) # generic clean up just in case
try:
if (
spec_util.UUID_REGEX.match(spec)
): # generic function sometimes get uuid already just return it.
uuid = spec
else:
uuid = worksheet_util.get_worksheet_uuid(
self.client, None, spec)
except UsageError, e:
#TODO handle Found multiple worksheets with name
raise e
return uuid
class BundleService():
def __init__(self, user=None):
self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
lambda command: get_user_token(user), verbose=1)
def items(self):
return _call_with_retries(lambda: self.client.search())
def get_bundle_info(self, uuid):
## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False):
bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))
# format permission data
bundle_info['permission_str'] = permission_str(bundle_info['permission'])
# format each groups as well
for group_permission in bundle_info['group_permissions']:
group_permission['permission_str'] = permission_str(group_permission['permission'])
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
# format based on specs from the cli
for spec in cls.METADATA_SPECS:
key = spec.key
if key not in metadata:
continue
if metadata[key] == '' or metadata[key] == []:
continue
value = worksheet_util.apply_func(spec.formatting, metadata.get(key))
# if isinstance(value, list):
# value = ', '.join(value)
metadata[key] = value
bundle_info['metadata'] = metadata
return bundle_info
def head_target(self, target, maxlines=100):
return self.client.head_target(target, maxlines)
def search_bundles(self, keywords, worksheet_uuid=None):
bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords)
bundle_infos = self.client.get_bundle_infos(bundle_uuids)
return bundle_infos
def worksheets(self):
return _call_with_retries(lambda: self.client.list_worksheets())
def search_worksheets(self, keywords, worksheet_uuid=None):
return _call_with_retries(lambda: self.client.search_worksheets(keywords))
def create_worksheet(self, name):
return _call_with_retries(lambda: self.client.new_worksheet(name, None))
def worksheet(self, uuid, interpreted=False):
try:
worksheet_info = self.client.get_worksheet_info(
uuid,
True, #fetch_items
True, # get_permissions
)
except PermissionError:
raise UsageError # forces a not found
worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info)
# set permissions
worksheet_info['edit_permission'] = False
if worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL:
worksheet_info['edit_permission'] = True
worksheet_info['permission_str'] = permission_str(worksheet_info['permission'])
# format each groups as well
for group_permission in worksheet_info['group_permissions']:
group_permission['permission_str'] = permission_str(group_permission['permission'])
if interpreted:
interpreted_items = worksheet_util.interpret_items(
worksheet_util.get_default_schemas(),
worksheet_info['items']
)
worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items'])
return worksheet_info
else:
return worksheet_info
def create_run_bundle(self, args, worksheet_uuid):
cli = self._create_cli(worksheet_uuid)
parser = cli.create_parser('run')
parser.add_argument('target_spec', help=cli.TARGET_SPEC_FORMAT, nargs='*')
parser.add_argument('command', help='Command-line')
metadata_util.add_arguments(RunBundle, set(), parser)
metadata_util.add_edit_argument(parser)
args = parser.parse_args(args)
metadata = metadata_util.request_missing_metadata(RunBundle, args)
targets = cli.parse_key_targets(self.client, worksheet_uuid, args.target_spec)
new_bundle_uuid = self.client.derive_bundle('run', targets, args.command, metadata, worksheet_uuid)
return new_bundle_uuid
def upload_bundle_url(self, url, info, worksheet_uuid):
file_name = url.split("/")[-1]
info = {
'bundle_type': 'dataset',
'metadata': {
'description': 'Upload %s' % url,
'tags': [],
'name': '%s' % file_name,
'license': '',
'source_url': '%s' % url,
}
}
new_bundle_uuid = self.client.upload_bundle_url(url, info, worksheet_uuid, True)
return new_bundle_uuid
def add_worksheet_item(self, worksheet_uuid, bundle_uuid):
self.client.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle_uuid))
def parse_and_update_worksheet(self, uuid, lines):
worksheet_info = self.client.get_worksheet_info(uuid, True)
new_items, commands = worksheet_util.parse_worksheet_form(lines, self.client, worksheet_info['uuid'])
self.client.update_worksheet(
worksheet_info,
new_items
)
def get_target_info(self, target, depth=1):
return _call_with_retries(lambda: self.client.get_target_info(target, depth))
def resolve_interpreted_items(self, interpreted_items):
return _call_with_retries(lambda: self.client.resolve_interpreted_items(('test', 'test')))
def get_worksheet_info(self):
return _call_with_retries(lambda: self.client.get_worksheet_info())
def delete_worksheet(self, worksheet_uuid):
return _call_with_retries(lambda: self.client.delete_worksheet(worksheet_uuid))
# Create an instance of a CLI.
def _create_cli(self, worksheet_uuid):
manager = CodaLabManager(temporary=True, clients={settings.BUNDLE_SERVICE_URL: self.client})
manager.set_current_worksheet_uuid(self.client, worksheet_uuid)
cli = bundle_cli.BundleCLI(manager, headless=True)
return cli
def general_command(self, worksheet_uuid, command):
cli = self._create_cli(worksheet_uuid)
args = worksheet_util.string_to_tokens(command)
def do_command():
from cStringIO import StringIO
import sys
real_stdout = sys.stdout
sys.stdout = StringIO()
stdout_str = None
#real_stderr = sys.stderr
#sys.stderr = StringIO()
stderr_str = None
exception = None
try:
cli.do_command(args)
success = True
except BaseException as e: # To capture SystemExit
exception = e
success = False
stdout_str = sys.stdout.getvalue()
sys.stdout.close()
sys.stdout = real_stdout
#stderr_str = sys.stderr.getvalue()
#sys.stderr.close()
#sys.stderr = real_stderr
print '>>> general_command on worksheet %s: %s' % (worksheet_uuid, command)
print stdout_str
print stderr_str
return {'stdout': stdout_str, 'stderr': stderr_str, 'exception': str(exception) if exception else None}
return _call_with_retries(do_command)
MAX_BYTES = 1024*1024
def read_file(self, uuid, path):
fid = self.client.open_target((uuid, path))
try:
while True:
bytes = self.client.read_file(fid, BundleService.MAX_BYTES)
yield bytes.data
if len(bytes.data) < BundleService.MAX_BYTES:
break
finally:
self.client.close_file(fid)
def download_target(self, uuid, return_zip=False):
target = (uuid, '')
result_path, container_path = self.client.download_target(target=target, follow_symlinks=True, return_zip=return_zip)
return (result_path, container_path)
def http_status_from_exception(self, ex):
# This is brittle. See https://github.com/codalab/codalab/issues/345.
if type(ex) == UsageError:
return 404
return 500
def update_bundle_metadata(self, uuid, new_metadata):
self.client.update_bundle_metadata(uuid, new_metadata)
return
class BundleService():
def __init__(self, user=None):
self.client = RemoteBundleClient(
settings.BUNDLE_SERVICE_URL,
lambda command: get_user_token(user),
verbose=1)
def items(self):
return _call_with_retries(lambda: self.client.search())
def get_bundle_info(self, uuid):
## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False):
bundle_info = _call_with_retries(
lambda: self.client.get_bundle_info(uuid, True, True, True))
# format permission data
bundle_info['permission_str'] = permission_str(
bundle_info['permission'])
# format each groups as well
for group_permission in bundle_info['group_permissions']:
group_permission['permission_str'] = permission_str(
group_permission['permission'])
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
# format based on specs from the cli
for spec in cls.METADATA_SPECS:
key = spec.key
if key not in metadata:
continue
if metadata[key] == '' or metadata[key] == []:
continue
value = worksheet_util.apply_func(spec.formatting,
metadata.get(key))
# if isinstance(value, list):
# value = ', '.join(value)
metadata[key] = value
bundle_info['metadata'] = metadata
return bundle_info
def head_target(self, target, maxlines=100):
return self.client.head_target(target, maxlines)
def search_bundles(self, keywords, worksheet_uuid=None):
bundle_uuids = self.client.search_bundle_uuids(
worksheet_uuid, keywords)
bundle_infos = self.client.get_bundle_infos(bundle_uuids)
return bundle_infos
def worksheets(self):
return _call_with_retries(lambda: self.client.list_worksheets())
def search_worksheets(self, keywords, worksheet_uuid=None):
return _call_with_retries(
lambda: self.client.search_worksheets(keywords))
def create_worksheet(self, name):
return _call_with_retries(
lambda: self.client.new_worksheet(name, None))
def worksheet(self, uuid, interpreted=False):
try:
worksheet_info = self.client.get_worksheet_info(
uuid,
True, #fetch_items
True, # get_permissions
)
except PermissionError:
raise UsageError # forces a not found
worksheet_info['raw'] = worksheet_util.get_worksheet_lines(
worksheet_info)
# set permissions
worksheet_info['edit_permission'] = False
if worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL:
worksheet_info['edit_permission'] = True
worksheet_info['permission_str'] = permission_str(
worksheet_info['permission'])
# format each groups as well
for group_permission in worksheet_info['group_permissions']:
group_permission['permission_str'] = permission_str(
group_permission['permission'])
if interpreted:
interpreted_items = worksheet_util.interpret_items(
worksheet_util.get_default_schemas(),
worksheet_info['items'])
worksheet_info[
'items'] = self.client.resolve_interpreted_items(
interpreted_items['items'])
# Currently, only certain fields are base64 encoded.
for item in worksheet_info['items']:
if item['mode'] in ['html', 'contents']:
# item['name'] in ['stdout', 'stderr']
item['interpreted'] = map(base64.b64decode,
item['interpreted'])
elif 'bundle_info' in item:
for bundle_info in item['bundle_info']:
try:
## sometimes bundle_info is a string. when item['mode'] is image
if isinstance(
bundle_info, dict) and bundle_info.get(
'bundle_type', None) == 'run':
if 'stdout' in bundle_info.keys():
bundle_info[
'stdout'] = base64.b64decode(
bundle_info['stdout'])
if 'stderr' in bundle_info.keys():
bundle_info[
'stderr'] = base64.b64decode(
bundle_info['stderr'])
except Exception, e:
print e
import ipdb
ipdb.set_trace()
return worksheet_info
def __init__(self):
self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
lambda command: "") #TODO
class BundleService():
def __init__(self, user=None):
self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
lambda command: get_user_token(user), verbose=1)
def items(self):
return _call_with_retries(lambda: self.client.search())
def item(self, uuid):
return _call_with_retries(lambda: self.client.get_bundle_info(uuid))
def worksheets(self):
return _call_with_retries(lambda: self.client.list_worksheets())
def create_worksheet(self, name):
return _call_with_retries(lambda: self.client.new_worksheet(name))
def worksheet(self, uuid, interpreted=False):
worksheet_info = _call_with_retries(
lambda: self.client.get_worksheet_info(
uuid,
True
)
)
if interpreted:
interpreted_items = worksheet_util.interpret_items(
worksheet_util.get_default_schemas(),
worksheet_info['items']
)
worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items'])
return worksheet_info
else:
return worksheet_info
def ls(self, uuid, path):
return _call_with_retries(lambda: self.client.ls((uuid, path)))
MAX_BYTES = 1024*1024
def read_file(self, uuid, path):
fid = self.client.open_target((uuid, path))
try:
while True:
bytes = self.client.read_file(fid, BundleService.MAX_BYTES)
yield bytes.data
if len(bytes.data) < BundleService.MAX_BYTES:
break
finally:
self.client.close_file(fid)
def download_target(self, uuid, return_zip=False):
target = (uuid, '')
result_path, container_path = self.client.download_target(target=target, follow_symlinks=True, return_zip=return_zip)
return (result_path, container_path)
def http_status_from_exception(self, ex):
# This is brittle. See https://github.com/codalab/codalab/issues/345.
if type(ex) == UsageError:
return 404
return 500
def __init__(self):
self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
lambda command: "") #TODO
def __init__(self, user=None):
self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
lambda command: get_user_token(user), verbose=1)
class BundleService():
def __init__(self, user=None):
self.client = RemoteBundleClient(
settings.BUNDLE_SERVICE_URL,
lambda command: get_user_token(user),
verbose=1)
def items(self):
return _call_with_retries(lambda: self.client.search())
def item(self, uuid):
return _call_with_retries(
lambda: self.client.get_bundle_info(uuid))
def worksheets(self):
return _call_with_retries(lambda: self.client.list_worksheets())
def create_worksheet(self, name):
return _call_with_retries(lambda: self.client.new_worksheet(name))
def worksheet(self, uuid, interpreted=False):
worksheet_info = _call_with_retries(
lambda: self.client.get_worksheet_info(uuid, True))
if interpreted:
interpreted_items = worksheet_util.interpret_items(
worksheet_util.get_default_schemas(),
worksheet_info['items'])
worksheet_info[
'items'] = self.client.resolve_interpreted_items(
interpreted_items['items'])
return worksheet_info
else:
return worksheet_info
def ls(self, uuid, path):
return _call_with_retries(lambda: self.client.ls((uuid, path)))
MAX_BYTES = 1024 * 1024
def read_file(self, uuid, path):
fid = self.client.open_target((uuid, path))
try:
while True:
bytes = self.client.read_file(fid, BundleService.MAX_BYTES)
yield bytes.data
if len(bytes.data) < BundleService.MAX_BYTES:
break
finally:
self.client.close_file(fid)
def download_target(self, uuid, return_zip=False):
target = (uuid, '')
result_path, container_path = self.client.download_target(
target=target, follow_symlinks=True, return_zip=return_zip)
return (result_path, container_path)
def http_status_from_exception(self, ex):
# This is brittle. See https://github.com/codalab/codalab/issues/345.
if type(ex) == UsageError:
return 404
return 500
def __init__(self):
self.client = RemoteBundleClient(self._cli_url(),
lambda command: self._get_user_token(), verbose=0)
class BundleService():
def __init__(self, user=None):
self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
lambda command: get_user_token(user), verbose=1)
def items(self):
return _call_with_retries(lambda: self.client.search())
def get_bundle_info(self, uuid):
bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid))
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
# format based on specs from the cli
for spec in cls.METADATA_SPECS:
key = spec.key
if key not in metadata:
continue
if metadata[key] == '' or metadata[key] == []:
continue
value = worksheet_util.apply_func(spec.formatting, metadata.get(key))
# if isinstance(value, list):
# value = ', '.join(value)
metadata[key] = value
bundle_info['metadata'] = metadata
return bundle_info
def search_bundles(self, keywords, worksheet_uuid=None):
# search_bundle_uuids(worksheet_uuid, keywords, max_results, show_counts_only)
bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords, 30, False)
bundle_infos = self.client.get_bundle_infos(bundle_uuids)
return bundle_infos
def worksheets(self):
return _call_with_retries(lambda: self.client.list_worksheets())
def create_worksheet(self, name):
return _call_with_retries(lambda: self.client.new_worksheet(name))
def worksheet(self, uuid, interpreted=False):
try:
worksheet_info = self.client.get_worksheet_info(
uuid,
True, #fetch_items
True, # get_permissions
)
except PermissionError:
raise UsageError # forces a not found
worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info)
# set permissions
worksheet_info['edit_permission'] = False
if worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL:
worksheet_info['edit_permission'] = True
if interpreted:
interpreted_items = worksheet_util.interpret_items(
worksheet_util.get_default_schemas(),
worksheet_info['items']
)
worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items'])
return worksheet_info
else:
return worksheet_info
def derive_bundle(self, bundle_type, targets, worksheet_uuid, command):
name = str(slugify(command))
metadata = {'name': name, 'tags': [], 'allowed_time': u'', 'allowed_memory': u'', 'allowed_disk': u'', 'description': ''}
new_bundle_uuid = self.client.derive_bundle(bundle_type, targets, command, metadata, worksheet_uuid)
return new_bundle_uuid
def upload_bundle_url(self, url, info, worksheet_uuid):
file_name = url.split("/")[-1]
info = {
'bundle_type': 'dataset',
'metadata': {
'description': 'Upload %s' % url,
'tags': [],
'name': '%s' % file_name,
'license': '',
'source_url': '%s' % url,
}
}
new_bundle_uuid = self.client.upload_bundle_url(url, info, worksheet_uuid, True)
return new_bundle_uuid
def add_worksheet_item(self, worksheet_uuid, bundle_uuid):
self.client.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle_uuid))
def parse_and_update_worksheet(self, uuid, lines):
worksheet_info = self.client.get_worksheet_info(uuid, True)
new_items, commands = worksheet_util.parse_worksheet_form(lines, self.client, worksheet_info['uuid'])
self.client.update_worksheet(
worksheet_info,
new_items
)
def get_target_info(self, target, depth=1):
return _call_with_retries(lambda: self.client.get_target_info(target, depth))
def resolve_interpreted_items(self, interpreted_items):
return _call_with_retries(lambda: self.client.resolve_interpreted_items(('test', 'test')))
def get_worksheet_info(self):
return _call_with_retries(lambda: self.client.get_worksheet_info())
def delete_worksheet(self, worksheet_uuid):
return _call_with_retries(lambda: self.client.delete_worksheet(worksheet_uuid))
MAX_BYTES = 1024*1024
def read_file(self, uuid, path):
fid = self.client.open_target((uuid, path))
try:
while True:
bytes = self.client.read_file(fid, BundleService.MAX_BYTES)
yield bytes.data
if len(bytes.data) < BundleService.MAX_BYTES:
break
finally:
self.client.close_file(fid)
def download_target(self, uuid, return_zip=False):
target = (uuid, '')
result_path, container_path = self.client.download_target(target=target, follow_symlinks=True, return_zip=return_zip)
return (result_path, container_path)
def http_status_from_exception(self, ex):
# This is brittle. See https://github.com/codalab/codalab/issues/345.
if type(ex) == UsageError:
return 404
return 500
def update_bundle_metadata(self, uuid, new_metadata):
self.client.update_bundle_metadata(uuid, new_metadata)
return
def __init__(self, user=None):
self.client = RemoteBundleClient(
settings.BUNDLE_SERVICE_URL,
lambda command: get_user_token(user),
verbose=1)
class BundleService(object):
# Maximum number of lines of files to show
HEAD_MAX_LINES = 100
def __init__(self, user=None):
self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
lambda command: get_user_token(user), verbose=1)
def items(self):
return _call_with_retries(lambda: self.client.search())
def get_bundle_info(self, uuid):
bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))
if bundle_info is None:
return None
# Set permissions
bundle_info['edit_permission'] = (bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
# Format permissions into strings
bundle_info['permission_str'] = permission_str(bundle_info['permission'])
for group_permission in bundle_info['group_permissions']:
group_permission['permission_str'] = permission_str(group_permission['permission'])
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
for key, value in worksheet_util.get_formatted_metadata(cls, metadata):
metadata[key] = value
bundle_info['metadata'] = metadata
bundle_info['editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields(cls, metadata)
return bundle_info
def head_target(self, target, max_num_lines=HEAD_MAX_LINES):
return self.client.head_target(target, max_num_lines)
def search_bundles(self, keywords, worksheet_uuid=None):
bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords)
bundle_infos = self.client.get_bundle_infos(bundle_uuids)
return bundle_infos
def get_worksheet_bundles(self, worksheet_uuid):
worksheet_info = self.client.get_worksheet_info(worksheet_uuid, True, True)
bundle_info_list = []
for (bundle_info, subworksheet_info, value_obj, item_type) in worksheet_info['items']:
if item_type == worksheet_util.TYPE_BUNDLE:
bundle_info_list.append(bundle_info)
return bundle_info_list
def worksheets(self):
return _call_with_retries(lambda: self.client.list_worksheets())
def search_worksheets(self, keywords, worksheet_uuid=None):
return _call_with_retries(lambda: self.client.search_worksheets(keywords))
def create_worksheet(self, name):
return _call_with_retries(lambda: self.client.new_worksheet(name, None))
def get_worksheet_uuid(self, spec):
# generic clean up just in case
spec = smart_str(spec)
# generic function sometimes get uuid already just return it.
if spec_util.UUID_REGEX.match(spec):
return spec
else:
return worksheet_util.get_worksheet_uuid(self.client, None, spec)
def basic_worksheet(self, uuid):
return self.worksheet(uuid, fetch_items=False, get_permissions=True, interpreted=False)
def full_worksheet(self, uuid):
return self.worksheet(uuid, fetch_items=True, get_permissions=True, interpreted=True)
def worksheet(self, uuid, fetch_items, get_permissions, interpreted):
"""
Return information about a worksheet. Calls
- get_worksheet_info: get basic info
- resolve_interpreted_items: get more information about a worksheet.
In the future, for large worksheets, might want to break this up so
that we can render something basic.
"""
worksheet_info = self.client.get_worksheet_info(uuid, fetch_items, get_permissions)
if fetch_items:
worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info)
# Set permissions
worksheet_info['edit_permission'] = (worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
# Format permissions into strings
worksheet_info['permission_str'] = permission_str(worksheet_info['permission'])
for group_permission in worksheet_info['group_permissions']:
group_permission['permission_str'] = permission_str(group_permission['permission'])
# Go and fetch more information about the worksheet contents by
# resolving the interpreted items.
if interpreted:
try:
interpreted_items = worksheet_util.interpret_items(
worksheet_util.get_default_schemas(),
worksheet_info['items'])
except UsageError, e:
interpreted_items = {'items': []}
worksheet_info['error'] = str(e)
worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items'])
worksheet_info['raw_to_interpreted'] = interpreted_items['raw_to_interpreted']
worksheet_info['interpreted_to_raw'] = interpreted_items['interpreted_to_raw']
def decode_lines(interpreted):
# interpreted is None or list of base64 encoded lines
if interpreted is None:
return formatting.contents_str(None)
else:
return map(base64.b64decode, interpreted)
# Currently, only certain fields are base64 encoded.
for item in worksheet_info['items']:
if item['mode'] in ['html', 'contents']:
item['interpreted'] = decode_lines(item['interpreted'])
elif item['mode'] == 'table':
for row_map in item['interpreted'][1]:
for k, v in row_map.iteritems():
if v is None:
row_map[k] = formatting.contents_str(v)
elif 'bundle_info' in item:
infos = []
if isinstance(item['bundle_info'], list):
infos = item['bundle_info']
elif isinstance(item['bundle_info'], dict):
infos = [item['bundle_info']]
for bundle_info in infos:
try:
if isinstance(bundle_info, dict):
worksheet_util.format_metadata(bundle_info.get('metadata'))
except Exception, e:
print e
import ipdb; ipdb.set_trace()