class BundleService():
def __init__(self, user=None):
self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
lambda command: get_user_token(user), verbose=1)
def items(self):
return _call_with_retries(lambda: self.client.search())
def item(self, uuid):
return _call_with_retries(lambda: self.client.get_bundle_info(uuid))
def worksheets(self):
return _call_with_retries(lambda: self.client.list_worksheets())
def create_worksheet(self, name):
return _call_with_retries(lambda: self.client.new_worksheet(name))
def worksheet(self, uuid, interpreted=False):
worksheet_info = _call_with_retries(
lambda: self.client.get_worksheet_info(
uuid,
True
)
)
if interpreted:
interpreted_items = worksheet_util.interpret_items(
worksheet_util.get_default_schemas(),
worksheet_info['items']
)
worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items'])
return worksheet_info
else:
return worksheet_info
def ls(self, uuid, path):
return _call_with_retries(lambda: self.client.ls((uuid, path)))
MAX_BYTES = 1024*1024
def read_file(self, uuid, path):
fid = self.client.open_target((uuid, path))
try:
while True:
bytes = self.client.read_file(fid, BundleService.MAX_BYTES)
yield bytes.data
if len(bytes.data) < BundleService.MAX_BYTES:
break
finally:
self.client.close_file(fid)
def download_target(self, uuid, return_zip=False):
target = (uuid, '')
result_path, container_path = self.client.download_target(target=target, follow_symlinks=True, return_zip=return_zip)
return (result_path, container_path)
def http_status_from_exception(self, ex):
# This is brittle. See https://github.com/codalab/codalab/issues/345.
if type(ex) == UsageError:
return 404
return 500
class BundleService():
def __init__(self, user=None):
self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
lambda command: get_user_token(user), verbose=1)
def items(self):
return _call_with_retries(lambda: self.client.search())
def item(self, uuid):
return _call_with_retries(lambda: self.client.get_bundle_info(uuid))
def worksheets(self):
return _call_with_retries(lambda: self.client.list_worksheets())
def create_worksheet(self, name):
return _call_with_retries(lambda: self.client.new_worksheet(name))
def worksheet(self, uuid):
return _call_with_retries(lambda: self.client.get_worksheet_info(uuid))
def ls(self, uuid, path):
return _call_with_retries(lambda: self.client.ls((uuid, path)))
MAX_BYTES = 1024*1024
def read_file(self, uuid, path):
fid = self.client.open_target((uuid, path))
try:
while True:
bytes = self.client.read_file(fid, BundleService.MAX_BYTES)
yield bytes.data
if len(bytes.data) < BundleService.MAX_BYTES:
break
finally:
self.client.close_file(fid)
def http_status_from_exception(self, ex):
# This is brittle. See https://github.com/codalab/codalab/issues/345.
if type(ex) == UsageError:
return 404
return 500
class BundleService(object):
# Maximum number of lines of files to show
HEAD_MAX_LINES = 100
def __init__(self, user=None):
self.client = RemoteBundleClient(
settings.BUNDLE_SERVICE_URL,
lambda command: get_user_token(user),
verbose=1)
def items(self):
return _call_with_retries(lambda: self.client.search())
def get_bundle_info(self, uuid):
bundle_info = _call_with_retries(
lambda: self.client.get_bundle_info(uuid, True, True, True))
if bundle_info is None:
return None
# Set permissions
bundle_info['edit_permission'] = (
bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
# Format permissions into strings
bundle_info['permission_str'] = permission_str(
bundle_info['permission'])
for group_permission in bundle_info['group_permissions']:
group_permission['permission_str'] = permission_str(
group_permission['permission'])
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
for key, value in worksheet_util.get_formatted_metadata(
cls, metadata):
metadata[key] = value
bundle_info['metadata'] = metadata
bundle_info[
'editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields(
cls, metadata)
return bundle_info
def head_target(self, target, max_num_lines=HEAD_MAX_LINES):
return self.client.head_target(target, max_num_lines)
def search_bundles(self, keywords, worksheet_uuid=None):
bundle_uuids = self.client.search_bundle_uuids(
worksheet_uuid, keywords)
bundle_infos = self.client.get_bundle_infos(bundle_uuids)
return bundle_infos
def get_worksheet_bundles(self, worksheet_uuid):
worksheet_info = self.client.get_worksheet_info(
worksheet_uuid, True, True)
bundle_info_list = []
for (bundle_info, subworksheet_info, value_obj,
item_type) in worksheet_info['items']:
if item_type == worksheet_util.TYPE_BUNDLE:
bundle_info_list.append(bundle_info)
return bundle_info_list
def worksheets(self):
return _call_with_retries(lambda: self.client.list_worksheets())
def search_worksheets(self, keywords, worksheet_uuid=None):
return _call_with_retries(
lambda: self.client.search_worksheets(keywords))
def create_worksheet(self, name):
return _call_with_retries(
lambda: self.client.new_worksheet(name, None))
def get_worksheet_uuid(self, spec):
# generic clean up just in case
spec = smart_str(spec)
# generic function sometimes get uuid already just return it.
if spec_util.UUID_REGEX.match(spec):
return spec
else:
return worksheet_util.get_worksheet_uuid(
self.client, None, spec)
def basic_worksheet(self, uuid):
return self.worksheet(uuid,
fetch_items=False,
get_permissions=True,
interpreted=False)
def full_worksheet(self, uuid):
return self.worksheet(uuid,
fetch_items=True,
get_permissions=True,
interpreted=True)
def worksheet(self, uuid, fetch_items, get_permissions, interpreted):
"""
Return information about a worksheet. Calls
- get_worksheet_info: get basic info
- resolve_interpreted_items: get more information about a worksheet.
In the future, for large worksheets, might want to break this up so
that we can render something basic.
"""
worksheet_info = self.client.get_worksheet_info(
uuid, fetch_items, get_permissions)
if fetch_items:
worksheet_info['raw'] = worksheet_util.get_worksheet_lines(
worksheet_info)
# Set permissions
worksheet_info['edit_permission'] = (
worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
# Format permissions into strings
worksheet_info['permission_str'] = permission_str(
worksheet_info['permission'])
for group_permission in worksheet_info['group_permissions']:
group_permission['permission_str'] = permission_str(
group_permission['permission'])
# Go and fetch more information about the worksheet contents by
# resolving the interpreted items.
if interpreted:
try:
interpreted_items = worksheet_util.interpret_items(
worksheet_util.get_default_schemas(),
worksheet_info['items'])
except UsageError, e:
interpreted_items = {'items': []}
worksheet_info['error'] = str(e)
worksheet_info[
'items'] = self.client.resolve_interpreted_items(
interpreted_items['items'])
# Currently, only certain fields are base64 encoded.
for item in worksheet_info['items']:
if item['mode'] in ['html', 'contents']:
if item['interpreted'] is None:
item['interpreted'] = [
formatting.contents_str(item['interpreted'])
]
else:
item['interpreted'] = map(base64.b64decode,
item['interpreted'])
elif item['mode'] == 'table':
for row_map in item['interpreted'][1]:
for k, v in row_map.iteritems():
if v is None:
row_map[k] = formatting.contents_str(v)
elif 'bundle_info' in item:
infos = []
if isinstance(item['bundle_info'], list):
infos = item['bundle_info']
elif isinstance(item['bundle_info'], dict):
infos = [item['bundle_info']]
for bundle_info in infos:
try:
if isinstance(bundle_info, dict):
worksheet_util.format_metadata(
bundle_info.get('metadata'))
except Exception, e:
print e
import ipdb
ipdb.set_trace()
class BundleService():
def __init__(self, user=None):
self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
lambda command: get_user_token(user), verbose=1)
def items(self):
return _call_with_retries(lambda: self.client.search())
def get_bundle_info(self, uuid):
## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False):
bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))
# format permission data
bundle_info['permission_str'] = permission_str(bundle_info['permission'])
# format each groups as well
for group_permission in bundle_info['group_permissions']:
group_permission['permission_str'] = permission_str(group_permission['permission'])
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
# format based on specs from the cli
for spec in cls.METADATA_SPECS:
key = spec.key
if key not in metadata:
continue
if metadata[key] == '' or metadata[key] == []:
continue
value = worksheet_util.apply_func(spec.formatting, metadata.get(key))
# if isinstance(value, list):
# value = ', '.join(value)
metadata[key] = value
bundle_info['metadata'] = metadata
return bundle_info
def head_target(self, target, maxlines=100):
return self.client.head_target(target, maxlines)
def search_bundles(self, keywords, worksheet_uuid=None):
bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords)
bundle_infos = self.client.get_bundle_infos(bundle_uuids)
return bundle_infos
def worksheets(self):
return _call_with_retries(lambda: self.client.list_worksheets())
def search_worksheets(self, keywords, worksheet_uuid=None):
return _call_with_retries(lambda: self.client.search_worksheets(keywords))
def create_worksheet(self, name):
return _call_with_retries(lambda: self.client.new_worksheet(name, None))
def worksheet(self, uuid, interpreted=False):
try:
worksheet_info = self.client.get_worksheet_info(
uuid,
True, #fetch_items
True, # get_permissions
)
except PermissionError:
raise UsageError # forces a not found
worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info)
# set permissions
worksheet_info['edit_permission'] = False
if worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL:
worksheet_info['edit_permission'] = True
worksheet_info['permission_str'] = permission_str(worksheet_info['permission'])
# format each groups as well
for group_permission in worksheet_info['group_permissions']:
group_permission['permission_str'] = permission_str(group_permission['permission'])
if interpreted:
interpreted_items = worksheet_util.interpret_items(
worksheet_util.get_default_schemas(),
worksheet_info['items']
)
worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items'])
# Currently, only certain fields are base64 encoded.
for item in worksheet_info['items']:
if item['mode'] in ['html', 'contents']:
# item['name'] in ['stdout', 'stderr']
item['interpreted'] = map(base64.b64decode, item['interpreted'])
elif 'bundle_info' in item:
for bundle_info in item['bundle_info']:
try:
## sometimes bundle_info is a string. when item['mode'] is image
if isinstance(bundle_info, dict) and bundle_info.get('bundle_type', None) == 'run':
if 'stdout' in bundle_info.keys():
bundle_info['stdout'] = base64.b64decode(bundle_info['stdout'])
if 'stderr' in bundle_info.keys():
bundle_info['stderr'] = base64.b64decode(bundle_info['stderr'])
except Exception, e:
print e
import ipdb; ipdb.set_trace()
return worksheet_info
class BundleService():
def __init__(self, user=None):
self.client = RemoteBundleClient(
settings.BUNDLE_SERVICE_URL,
lambda command: get_user_token(user),
verbose=1)
def items(self):
return _call_with_retries(lambda: self.client.search())
def item(self, uuid):
return _call_with_retries(
lambda: self.client.get_bundle_info(uuid))
def worksheets(self):
return _call_with_retries(lambda: self.client.list_worksheets())
def create_worksheet(self, name):
return _call_with_retries(lambda: self.client.new_worksheet(name))
def worksheet(self, uuid, interpreted=False):
worksheet_info = _call_with_retries(
lambda: self.client.get_worksheet_info(uuid, True))
if interpreted:
interpreted_items = worksheet_util.interpret_items(
worksheet_util.get_default_schemas(),
worksheet_info['items'])
worksheet_info[
'items'] = self.client.resolve_interpreted_items(
interpreted_items['items'])
return worksheet_info
else:
return worksheet_info
def ls(self, uuid, path):
return _call_with_retries(lambda: self.client.ls((uuid, path)))
MAX_BYTES = 1024 * 1024
def read_file(self, uuid, path):
fid = self.client.open_target((uuid, path))
try:
while True:
bytes = self.client.read_file(fid, BundleService.MAX_BYTES)
yield bytes.data
if len(bytes.data) < BundleService.MAX_BYTES:
break
finally:
self.client.close_file(fid)
def download_target(self, uuid, return_zip=False):
target = (uuid, '')
result_path, container_path = self.client.download_target(
target=target, follow_symlinks=True, return_zip=return_zip)
return (result_path, container_path)
def http_status_from_exception(self, ex):
# This is brittle. See https://github.com/codalab/codalab/issues/345.
if type(ex) == UsageError:
return 404
return 500
class BundleService(object):
# Maximum number of lines of files to show
HEAD_MAX_LINES = 100
def __init__(self, user=None):
self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
lambda command: get_user_token(user), verbose=1)
def items(self):
return _call_with_retries(lambda: self.client.search())
def get_bundle_info(self, uuid):
bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))
if bundle_info is None:
return None
# Set permissions
bundle_info['edit_permission'] = (bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
# Format permissions into strings
bundle_info['permission_str'] = permission_str(bundle_info['permission'])
for group_permission in bundle_info['group_permissions']:
group_permission['permission_str'] = permission_str(group_permission['permission'])
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
for key, value in worksheet_util.get_formatted_metadata(cls, metadata):
metadata[key] = value
bundle_info['metadata'] = metadata
bundle_info['editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields(cls, metadata)
return bundle_info
def head_target(self, target, max_num_lines=HEAD_MAX_LINES):
return self.client.head_target(target, max_num_lines)
def search_bundles(self, keywords, worksheet_uuid=None):
bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords)
bundle_infos = self.client.get_bundle_infos(bundle_uuids)
return bundle_infos
def get_worksheet_bundles(self, worksheet_uuid):
worksheet_info = self.client.get_worksheet_info(worksheet_uuid, True, True)
bundle_info_list = []
for (bundle_info, subworksheet_info, value_obj, item_type) in worksheet_info['items']:
if item_type == worksheet_util.TYPE_BUNDLE:
bundle_info_list.append(bundle_info)
return bundle_info_list
def worksheets(self):
return _call_with_retries(lambda: self.client.list_worksheets())
def search_worksheets(self, keywords, worksheet_uuid=None):
return _call_with_retries(lambda: self.client.search_worksheets(keywords))
def create_worksheet(self, name):
return _call_with_retries(lambda: self.client.new_worksheet(name, None))
def get_worksheet_uuid(self, spec):
# generic clean up just in case
spec = smart_str(spec)
# generic function sometimes get uuid already just return it.
if spec_util.UUID_REGEX.match(spec):
return spec
else:
return worksheet_util.get_worksheet_uuid(self.client, None, spec)
def basic_worksheet(self, uuid):
return self.worksheet(uuid, fetch_items=False, get_permissions=True, interpreted=False)
def full_worksheet(self, uuid):
return self.worksheet(uuid, fetch_items=True, get_permissions=True, interpreted=True)
def worksheet(self, uuid, fetch_items, get_permissions, interpreted):
"""
Return information about a worksheet. Calls
- get_worksheet_info: get basic info
- resolve_interpreted_items: get more information about a worksheet.
In the future, for large worksheets, might want to break this up so
that we can render something basic.
"""
worksheet_info = self.client.get_worksheet_info(uuid, fetch_items, get_permissions)
if fetch_items:
worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info)
# Set permissions
worksheet_info['edit_permission'] = (worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
# Format permissions into strings
worksheet_info['permission_str'] = permission_str(worksheet_info['permission'])
for group_permission in worksheet_info['group_permissions']:
group_permission['permission_str'] = permission_str(group_permission['permission'])
# Go and fetch more information about the worksheet contents by
# resolving the interpreted items.
if interpreted:
try:
interpreted_items = worksheet_util.interpret_items(
worksheet_util.get_default_schemas(),
worksheet_info['items'])
except UsageError, e:
interpreted_items = {'items': []}
worksheet_info['error'] = str(e)
worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items'])
worksheet_info['raw_to_interpreted'] = interpreted_items['raw_to_interpreted']
worksheet_info['interpreted_to_raw'] = interpreted_items['interpreted_to_raw']
def decode_lines(interpreted):
# interpreted is None or list of base64 encoded lines
if interpreted is None:
return formatting.contents_str(None)
else:
return map(base64.b64decode, interpreted)
# Currently, only certain fields are base64 encoded.
for item in worksheet_info['items']:
if item['mode'] in ['html', 'contents']:
item['interpreted'] = decode_lines(item['interpreted'])
elif item['mode'] == 'table':
for row_map in item['interpreted'][1]:
for k, v in row_map.iteritems():
if v is None:
row_map[k] = formatting.contents_str(v)
elif 'bundle_info' in item:
infos = []
if isinstance(item['bundle_info'], list):
infos = item['bundle_info']
elif isinstance(item['bundle_info'], dict):
infos = [item['bundle_info']]
for bundle_info in infos:
try:
if isinstance(bundle_info, dict):
worksheet_util.format_metadata(bundle_info.get('metadata'))
except Exception, e:
print e
import ipdb; ipdb.set_trace()
class BundleService():
# Maximum number of lines of files to show
HEAD_MAX_LINES = 100
def __init__(self, user=None):
self.client = RemoteBundleClient(
settings.BUNDLE_SERVICE_URL,
lambda command: get_user_token(user),
verbose=1)
def items(self):
return _call_with_retries(lambda: self.client.search())
def get_bundle_info(self, uuid):
bundle_info = _call_with_retries(
lambda: self.client.get_bundle_info(uuid, True, True, True))
# Set permissions
bundle_info['edit_permission'] = (
bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
# Format permissions into strings
bundle_info['permission_str'] = permission_str(
bundle_info['permission'])
for group_permission in bundle_info['group_permissions']:
group_permission['permission_str'] = permission_str(
group_permission['permission'])
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
for key, value in worksheet_util.get_formatted_metadata(
cls, metadata):
metadata[key] = value
bundle_info['metadata'] = metadata
bundle_info[
'editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields(
cls, metadata)
return bundle_info
def head_target(self, target, maxlines=100):
return self.client.head_target(target, maxlines)
def search_bundles(self, keywords, worksheet_uuid=None):
bundle_uuids = self.client.search_bundle_uuids(
worksheet_uuid, keywords)
bundle_infos = self.client.get_bundle_infos(bundle_uuids)
return bundle_infos
def get_worksheet_bundles(self, worksheet_uuid):
worksheet_info = self.client.get_worksheet_info(
worksheet_uuid, True, True)
bundle_info_list = []
for (bundle_info, subworksheet_info, value_obj,
item_type) in worksheet_info['items']:
if item_type == worksheet_util.TYPE_BUNDLE:
bundle_info_list.append(bundle_info)
return bundle_info_list
def worksheets(self):
return _call_with_retries(lambda: self.client.list_worksheets())
def search_worksheets(self, keywords, worksheet_uuid=None):
return _call_with_retries(
lambda: self.client.search_worksheets(keywords))
def create_worksheet(self, name):
return _call_with_retries(
lambda: self.client.new_worksheet(name, None))
def get_worksheet_uuid(self, spec):
uuid = None
spec = smart_str(spec) # generic clean up just in case
try:
if (
spec_util.UUID_REGEX.match(spec)
): # generic function sometimes get uuid already just return it.
uuid = spec
else:
uuid = worksheet_util.get_worksheet_uuid(
self.client, None, spec)
except UsageError, e:
#TODO handle Found multiple worksheets with name
raise e
return uuid
class BundleService():
def __init__(self, user=None):
self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
lambda command: get_user_token(user), verbose=1)
def items(self):
return _call_with_retries(lambda: self.client.search())
def get_bundle_info(self, uuid):
## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False):
bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))
# format permission data
bundle_info['permission_str'] = permission_str(bundle_info['permission'])
# format each groups as well
for group_permission in bundle_info['group_permissions']:
group_permission['permission_str'] = permission_str(group_permission['permission'])
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
# format based on specs from the cli
for spec in cls.METADATA_SPECS:
key = spec.key
if key not in metadata:
continue
if metadata[key] == '' or metadata[key] == []:
continue
value = worksheet_util.apply_func(spec.formatting, metadata.get(key))
# if isinstance(value, list):
# value = ', '.join(value)
metadata[key] = value
bundle_info['metadata'] = metadata
return bundle_info
def head_target(self, target, maxlines=100):
return self.client.head_target(target, maxlines)
def search_bundles(self, keywords, worksheet_uuid=None):
bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords)
bundle_infos = self.client.get_bundle_infos(bundle_uuids)
return bundle_infos
def worksheets(self):
return _call_with_retries(lambda: self.client.list_worksheets())
def search_worksheets(self, keywords, worksheet_uuid=None):
return _call_with_retries(lambda: self.client.search_worksheets(keywords))
def create_worksheet(self, name):
return _call_with_retries(lambda: self.client.new_worksheet(name, None))
def worksheet(self, uuid, interpreted=False):
try:
worksheet_info = self.client.get_worksheet_info(
uuid,
True, #fetch_items
True, # get_permissions
)
except PermissionError:
raise UsageError # forces a not found
worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info)
# set permissions
worksheet_info['edit_permission'] = False
if worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL:
worksheet_info['edit_permission'] = True
worksheet_info['permission_str'] = permission_str(worksheet_info['permission'])
# format each groups as well
for group_permission in worksheet_info['group_permissions']:
group_permission['permission_str'] = permission_str(group_permission['permission'])
if interpreted:
interpreted_items = worksheet_util.interpret_items(
worksheet_util.get_default_schemas(),
worksheet_info['items']
)
worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items'])
return worksheet_info
else:
return worksheet_info
def create_run_bundle(self, args, worksheet_uuid):
cli = self._create_cli(worksheet_uuid)
parser = cli.create_parser('run')
parser.add_argument('target_spec', help=cli.TARGET_SPEC_FORMAT, nargs='*')
parser.add_argument('command', help='Command-line')
metadata_util.add_arguments(RunBundle, set(), parser)
metadata_util.add_edit_argument(parser)
args = parser.parse_args(args)
metadata = metadata_util.request_missing_metadata(RunBundle, args)
targets = cli.parse_key_targets(self.client, worksheet_uuid, args.target_spec)
new_bundle_uuid = self.client.derive_bundle('run', targets, args.command, metadata, worksheet_uuid)
return new_bundle_uuid
def upload_bundle_url(self, url, info, worksheet_uuid):
file_name = url.split("/")[-1]
info = {
'bundle_type': 'dataset',
'metadata': {
'description': 'Upload %s' % url,
'tags': [],
'name': '%s' % file_name,
'license': '',
'source_url': '%s' % url,
}
}
new_bundle_uuid = self.client.upload_bundle_url(url, info, worksheet_uuid, True)
return new_bundle_uuid
def add_worksheet_item(self, worksheet_uuid, bundle_uuid):
self.client.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle_uuid))
def parse_and_update_worksheet(self, uuid, lines):
worksheet_info = self.client.get_worksheet_info(uuid, True)
new_items, commands = worksheet_util.parse_worksheet_form(lines, self.client, worksheet_info['uuid'])
self.client.update_worksheet(
worksheet_info,
new_items
)
def get_target_info(self, target, depth=1):
return _call_with_retries(lambda: self.client.get_target_info(target, depth))
def resolve_interpreted_items(self, interpreted_items):
return _call_with_retries(lambda: self.client.resolve_interpreted_items(('test', 'test')))
def get_worksheet_info(self):
return _call_with_retries(lambda: self.client.get_worksheet_info())
def delete_worksheet(self, worksheet_uuid):
return _call_with_retries(lambda: self.client.delete_worksheet(worksheet_uuid))
# Create an instance of a CLI.
def _create_cli(self, worksheet_uuid):
manager = CodaLabManager(temporary=True, clients={settings.BUNDLE_SERVICE_URL: self.client})
manager.set_current_worksheet_uuid(self.client, worksheet_uuid)
cli = bundle_cli.BundleCLI(manager, headless=True)
return cli
def general_command(self, worksheet_uuid, command):
cli = self._create_cli(worksheet_uuid)
args = worksheet_util.string_to_tokens(command)
def do_command():
from cStringIO import StringIO
import sys
real_stdout = sys.stdout
sys.stdout = StringIO()
stdout_str = None
#real_stderr = sys.stderr
#sys.stderr = StringIO()
stderr_str = None
exception = None
try:
cli.do_command(args)
success = True
except BaseException as e: # To capture SystemExit
exception = e
success = False
stdout_str = sys.stdout.getvalue()
sys.stdout.close()
sys.stdout = real_stdout
#stderr_str = sys.stderr.getvalue()
#sys.stderr.close()
#sys.stderr = real_stderr
print '>>> general_command on worksheet %s: %s' % (worksheet_uuid, command)
print stdout_str
print stderr_str
return {'stdout': stdout_str, 'stderr': stderr_str, 'exception': str(exception) if exception else None}
return _call_with_retries(do_command)
MAX_BYTES = 1024*1024
def read_file(self, uuid, path):
fid = self.client.open_target((uuid, path))
try:
while True:
bytes = self.client.read_file(fid, BundleService.MAX_BYTES)
yield bytes.data
if len(bytes.data) < BundleService.MAX_BYTES:
break
finally:
self.client.close_file(fid)
def download_target(self, uuid, return_zip=False):
target = (uuid, '')
result_path, container_path = self.client.download_target(target=target, follow_symlinks=True, return_zip=return_zip)
return (result_path, container_path)
def http_status_from_exception(self, ex):
# This is brittle. See https://github.com/codalab/codalab/issues/345.
if type(ex) == UsageError:
return 404
return 500
def update_bundle_metadata(self, uuid, new_metadata):
self.client.update_bundle_metadata(uuid, new_metadata)
return
class BundleService():
def __init__(self, user=None):
self.client = RemoteBundleClient(
settings.BUNDLE_SERVICE_URL,
lambda command: get_user_token(user),
verbose=1)
def items(self):
return _call_with_retries(lambda: self.client.search())
def get_bundle_info(self, uuid):
## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False):
bundle_info = _call_with_retries(
lambda: self.client.get_bundle_info(uuid, True, True, True))
# format permission data
bundle_info['permission_str'] = permission_str(
bundle_info['permission'])
# format each groups as well
for group_permission in bundle_info['group_permissions']:
group_permission['permission_str'] = permission_str(
group_permission['permission'])
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
# format based on specs from the cli
for spec in cls.METADATA_SPECS:
key = spec.key
if key not in metadata:
continue
if metadata[key] == '' or metadata[key] == []:
continue
value = worksheet_util.apply_func(spec.formatting,
metadata.get(key))
# if isinstance(value, list):
# value = ', '.join(value)
metadata[key] = value
bundle_info['metadata'] = metadata
return bundle_info
def head_target(self, target, maxlines=100):
return self.client.head_target(target, maxlines)
def search_bundles(self, keywords, worksheet_uuid=None):
bundle_uuids = self.client.search_bundle_uuids(
worksheet_uuid, keywords)
bundle_infos = self.client.get_bundle_infos(bundle_uuids)
return bundle_infos
def worksheets(self):
return _call_with_retries(lambda: self.client.list_worksheets())
def search_worksheets(self, keywords, worksheet_uuid=None):
return _call_with_retries(
lambda: self.client.search_worksheets(keywords))
def create_worksheet(self, name):
return _call_with_retries(
lambda: self.client.new_worksheet(name, None))
def worksheet(self, uuid, interpreted=False):
try:
worksheet_info = self.client.get_worksheet_info(
uuid,
True, #fetch_items
True, # get_permissions
)
except PermissionError:
raise UsageError # forces a not found
worksheet_info['raw'] = worksheet_util.get_worksheet_lines(
worksheet_info)
# set permissions
worksheet_info['edit_permission'] = False
if worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL:
worksheet_info['edit_permission'] = True
worksheet_info['permission_str'] = permission_str(
worksheet_info['permission'])
# format each groups as well
for group_permission in worksheet_info['group_permissions']:
group_permission['permission_str'] = permission_str(
group_permission['permission'])
if interpreted:
interpreted_items = worksheet_util.interpret_items(
worksheet_util.get_default_schemas(),
worksheet_info['items'])
worksheet_info[
'items'] = self.client.resolve_interpreted_items(
interpreted_items['items'])
# Currently, only certain fields are base64 encoded.
for item in worksheet_info['items']:
if item['mode'] in ['html', 'contents']:
# item['name'] in ['stdout', 'stderr']
item['interpreted'] = map(base64.b64decode,
item['interpreted'])
elif 'bundle_info' in item:
for bundle_info in item['bundle_info']:
try:
## sometimes bundle_info is a string. when item['mode'] is image
if isinstance(
bundle_info, dict) and bundle_info.get(
'bundle_type', None) == 'run':
if 'stdout' in bundle_info.keys():
bundle_info[
'stdout'] = base64.b64decode(
bundle_info['stdout'])
if 'stderr' in bundle_info.keys():
bundle_info[
'stderr'] = base64.b64decode(
bundle_info['stderr'])
except Exception, e:
print e
import ipdb
ipdb.set_trace()
return worksheet_info
class BundleService():
def __init__(self, user=None):
self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
lambda command: get_user_token(user), verbose=1)
def items(self):
return _call_with_retries(lambda: self.client.search())
def get_bundle_info(self, uuid):
## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False):
bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))
# format permission data
bundle_info['permission_str'] = permission_str(bundle_info['permission'])
# format each groups as well
for group_permission in bundle_info['group_permissions']:
group_permission['permission_str'] = permission_str(group_permission['permission'])
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
# format based on specs from the cli
for spec in cls.METADATA_SPECS:
key = spec.key
if key not in metadata:
continue
if metadata[key] == '' or metadata[key] == []:
continue
value = worksheet_util.apply_func(spec.formatting, metadata.get(key))
# if isinstance(value, list):
# value = ', '.join(value)
metadata[key] = value
bundle_info['metadata'] = metadata
return bundle_info
def head_target(self, target, maxlines=100):
return self.client.head_target(target, maxlines)
def search_bundles(self, keywords, worksheet_uuid=None):
bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords)
bundle_infos = self.client.get_bundle_infos(bundle_uuids)
return bundle_infos
def get_worksheet_bundles(self, worksheet_uuid):
worksheet_info = self.client.get_worksheet_info(worksheet_uuid, True, True)
bundle_info_list = []
for (bundle_info, subworksheet_info, value_obj, item_type) in worksheet_info['items']:
if item_type == worksheet_util.TYPE_BUNDLE:
bundle_info_list.append(bundle_info)
return bundle_info_list
def worksheets(self):
return _call_with_retries(lambda: self.client.list_worksheets())
def search_worksheets(self, keywords, worksheet_uuid=None):
return _call_with_retries(lambda: self.client.search_worksheets(keywords))
def create_worksheet(self, name):
return _call_with_retries(lambda: self.client.new_worksheet(name, None))
def get_worksheet_uuid(self, spec):
uuid = None
spec = smart_str(spec) # generic clean up just in case
try:
if(spec_util.UUID_REGEX.match(spec)): # generic function sometimes get uuid already just return it.
uuid = spec
else:
uuid = worksheet_util.get_worksheet_uuid(self.client, None, spec)
except UsageError, e:
#TODO handle Found multiple worksheets with name
raise e
return uuid
class BundleService():
def __init__(self, user=None):
self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
lambda command: get_user_token(user), verbose=1)
def items(self):
return _call_with_retries(lambda: self.client.search())
def get_bundle_info(self, uuid):
bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid))
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
# format based on specs from the cli
for spec in cls.METADATA_SPECS:
key = spec.key
if key not in metadata:
continue
if metadata[key] == '' or metadata[key] == []:
continue
value = worksheet_util.apply_func(spec.formatting, metadata.get(key))
# if isinstance(value, list):
# value = ', '.join(value)
metadata[key] = value
bundle_info['metadata'] = metadata
return bundle_info
def search_bundles(self, keywords, worksheet_uuid=None):
# search_bundle_uuids(worksheet_uuid, keywords, max_results, show_counts_only)
bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords, 30, False)
bundle_infos = self.client.get_bundle_infos(bundle_uuids)
return bundle_infos
def worksheets(self):
return _call_with_retries(lambda: self.client.list_worksheets())
def create_worksheet(self, name):
return _call_with_retries(lambda: self.client.new_worksheet(name))
def worksheet(self, uuid, interpreted=False):
try:
worksheet_info = self.client.get_worksheet_info(
uuid,
True, #fetch_items
True, # get_permissions
)
except PermissionError:
raise UsageError # forces a not found
worksheet_info['raw'] = worksheet_util.get_worksheet_lines(worksheet_info)
# set permissions
worksheet_info['edit_permission'] = False
if worksheet_info['permission'] == GROUP_OBJECT_PERMISSION_ALL:
worksheet_info['edit_permission'] = True
if interpreted:
interpreted_items = worksheet_util.interpret_items(
worksheet_util.get_default_schemas(),
worksheet_info['items']
)
worksheet_info['items'] = self.client.resolve_interpreted_items(interpreted_items['items'])
return worksheet_info
else:
return worksheet_info
def derive_bundle(self, bundle_type, targets, worksheet_uuid, command):
name = str(slugify(command))
metadata = {'name': name, 'tags': [], 'allowed_time': u'', 'allowed_memory': u'', 'allowed_disk': u'', 'description': ''}
new_bundle_uuid = self.client.derive_bundle(bundle_type, targets, command, metadata, worksheet_uuid)
return new_bundle_uuid
def upload_bundle_url(self, url, info, worksheet_uuid):
file_name = url.split("/")[-1]
info = {
'bundle_type': 'dataset',
'metadata': {
'description': 'Upload %s' % url,
'tags': [],
'name': '%s' % file_name,
'license': '',
'source_url': '%s' % url,
}
}
new_bundle_uuid = self.client.upload_bundle_url(url, info, worksheet_uuid, True)
return new_bundle_uuid
def add_worksheet_item(self, worksheet_uuid, bundle_uuid):
self.client.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle_uuid))
def parse_and_update_worksheet(self, uuid, lines):
worksheet_info = self.client.get_worksheet_info(uuid, True)
new_items, commands = worksheet_util.parse_worksheet_form(lines, self.client, worksheet_info['uuid'])
self.client.update_worksheet(
worksheet_info,
new_items
)
def get_target_info(self, target, depth=1):
return _call_with_retries(lambda: self.client.get_target_info(target, depth))
def resolve_interpreted_items(self, interpreted_items):
return _call_with_retries(lambda: self.client.resolve_interpreted_items(('test', 'test')))
def get_worksheet_info(self):
return _call_with_retries(lambda: self.client.get_worksheet_info())
def delete_worksheet(self, worksheet_uuid):
return _call_with_retries(lambda: self.client.delete_worksheet(worksheet_uuid))
MAX_BYTES = 1024*1024
def read_file(self, uuid, path):
fid = self.client.open_target((uuid, path))
try:
while True:
bytes = self.client.read_file(fid, BundleService.MAX_BYTES)
yield bytes.data
if len(bytes.data) < BundleService.MAX_BYTES:
break
finally:
self.client.close_file(fid)
def download_target(self, uuid, return_zip=False):
target = (uuid, '')
result_path, container_path = self.client.download_target(target=target, follow_symlinks=True, return_zip=return_zip)
return (result_path, container_path)
def http_status_from_exception(self, ex):
# This is brittle. See https://github.com/codalab/codalab/issues/345.
if type(ex) == UsageError:
return 404
return 500
def update_bundle_metadata(self, uuid, new_metadata):
self.client.update_bundle_metadata(uuid, new_metadata)
return
class BundleService():
# Maximum number of lines of files to show
HEAD_MAX_LINES = 100
def __init__(self, user=None):
self.client = RemoteBundleClient(settings.BUNDLE_SERVICE_URL,
lambda command: get_user_token(user), verbose=1)
def items(self):
return _call_with_retries(lambda: self.client.search())
def get_bundle_info(self, uuid):
bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))
# Set permissions
bundle_info['edit_permission'] = (bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
# Format permissions into strings
bundle_info['permission_str'] = permission_str(bundle_info['permission'])
for group_permission in bundle_info['group_permissions']:
group_permission['permission_str'] = permission_str(group_permission['permission'])
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
for key, value in worksheet_util.get_formatted_metadata(cls, metadata):
metadata[key] = value
bundle_info['metadata'] = metadata
bundle_info['editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields(cls, metadata)
return bundle_info
def head_target(self, target, maxlines=100):
return self.client.head_target(target, maxlines)
def search_bundles(self, keywords, worksheet_uuid=None):
bundle_uuids = self.client.search_bundle_uuids(worksheet_uuid, keywords)
bundle_infos = self.client.get_bundle_infos(bundle_uuids)
return bundle_infos
def get_worksheet_bundles(self, worksheet_uuid):
worksheet_info = self.client.get_worksheet_info(worksheet_uuid, True, True)
bundle_info_list = []
for (bundle_info, subworksheet_info, value_obj, item_type) in worksheet_info['items']:
if item_type == worksheet_util.TYPE_BUNDLE:
bundle_info_list.append(bundle_info)
return bundle_info_list
def worksheets(self):
return _call_with_retries(lambda: self.client.list_worksheets())
def search_worksheets(self, keywords, worksheet_uuid=None):
return _call_with_retries(lambda: self.client.search_worksheets(keywords))
def create_worksheet(self, name):
return _call_with_retries(lambda: self.client.new_worksheet(name, None))
def get_worksheet_uuid(self, spec):
uuid = None
spec = smart_str(spec) # generic clean up just in case
try:
if(spec_util.UUID_REGEX.match(spec)): # generic function sometimes get uuid already just return it.
uuid = spec
else:
uuid = worksheet_util.get_worksheet_uuid(self.client, None, spec)
except UsageError, e:
#TODO handle Found multiple worksheets with name
raise e
return uuid