def get_target_info(self, uuid, path, depth):
"""
Returns information about an individual target inside the bundle, or
None if the target doesn't exist.
For information about the format of the return value, see
worker.download_util.get_target_info.
"""
if self._bundle_model.get_bundle_state(uuid) != State.RUNNING:
bundle_path = self._bundle_store.get_bundle_location(uuid)
try:
return download_util.get_target_info(bundle_path, uuid, path, depth)
except download_util.PathException as e:
raise UsageError(e.message)
else:
# get_target_info calls are sent to the worker even on a shared file
# system since 1) due to NFS caching the worker has more up to date
# information on directory contents, and 2) the logic of hiding
# the dependency paths doesn't need to be re-implemented here.
worker = self._worker_model.get_bundle_worker(uuid)
response_socket_id = self._worker_model.allocate_socket(worker['user_id'], worker['worker_id'])
try:
read_args = {
'type': 'get_target_info',
'depth': depth,
}
self._send_read_message(worker, response_socket_id, uuid, path, read_args)
with closing(self._worker_model.start_listening(response_socket_id)) as sock:
result = self._worker_model.get_json_message(sock, 60)
precondition(result is not None, 'Unable to reach worker')
if 'error_code' in result:
raise http_error_to_exception(result['error_code'], result['error_message'])
return result['target_info']
finally:
self._worker_model.deallocate_socket(response_socket_id)
def wrapper(*args, **kwargs):
try:
return f(*args, **kwargs)
except urllib2.HTTPError as e:
# Translate known errors to the standard CodaLab errors
error_body = e.read()
exc = http_error_to_exception(e.code, error_body)
# All standard CodaLab errors are subclasses of UsageError
if isinstance(exc, UsageError):
raise exc.__class__, exc, sys.exc_info()[2]
else:
# Shunt other exceptions into one class
raise JsonApiException, \
JsonApiException(message.format(*args, **kwargs) +
': ' + httplib.responses[e.code] +
' - ' + error_body,
400 <= e.code < 500), \
sys.exc_info()[2]
except RestClientException as e:
raise JsonApiException, \
JsonApiException(message.format(*args, **kwargs) +
': ' + e.message, e.client_error), \
sys.exc_info()[2]
except (urllib2.URLError, httplib.HTTPException, socket.error) as e:
raise JsonApiException, \
JsonApiException(message.format(*args, **kwargs) +
': ' + str(e), False), \
sys.exc_info()[2]
def wrapper(*args, **kwargs):
try:
return f(*args, **kwargs)
except urllib2.HTTPError as e:
# Translate known errors to the standard CodaLab errors
exc = http_error_to_exception(e.code, e.read())
# All standard CodaLab errors are subclasses of UsageError
if isinstance(exc, UsageError):
raise exc.__class__, exc, sys.exc_info()[2]
else:
# Shunt other exceptions into one class
raise JsonApiException, \
JsonApiException(message.format(*args, **kwargs) +
': ' + httplib.responses[e.code] +
' - ' + e.read(),
400 <= e.code < 500), \
sys.exc_info()[2]
except RestClientException as e:
raise JsonApiException, \
JsonApiException(message.format(*args, **kwargs) +
': ' + e.message, e.client_error), \
sys.exc_info()[2]
except (urllib2.URLError, httplib.HTTPException, socket.error) as e:
raise JsonApiException, \
JsonApiException(message.format(*args, **kwargs) +
': ' + str(e), False), \
sys.exc_info()[2]
def _get_target_info_within_bundle(self, target, depth):
"""
Helper for get_target_info that only checks for the target info within that bundle
without considering the path might be pointing to one of the dependencies.
Raises NotFoundError if the path is not found.
"""
bundle_state = self._bundle_model.get_bundle_state(target.bundle_uuid)
bundle_link_url = self._bundle_model.get_bundle_metadata(
[target.bundle_uuid], "link_url").get(target.bundle_uuid)
if bundle_link_url:
bundle_link_url = self._transform_link_path(bundle_link_url)
# Raises NotFoundException if uuid is invalid
if bundle_state == State.PREPARING:
raise NotFoundError(
"Bundle {} hasn't started running yet, files not available".
format(target.bundle_uuid))
elif bundle_state != State.RUNNING:
bundle_path = bundle_link_url or self._bundle_store.get_bundle_location(
target.bundle_uuid)
try:
return download_util.get_target_info(bundle_path, target,
depth)
except download_util.PathException as err:
raise NotFoundError(str(err))
else:
# get_target_info calls are sent to the worker even on a shared file
# system since 1) due to NFS caching the worker has more up to date
# information on directory contents, and 2) the logic of hiding
# the dependency paths doesn't need to be re-implemented here.
worker = self._bundle_model.get_bundle_worker(target.bundle_uuid)
response_socket_id = self._worker_model.allocate_socket(
worker['user_id'], worker['worker_id'])
try:
read_args = {'type': 'get_target_info', 'depth': depth}
self._send_read_message(worker, response_socket_id, target,
read_args)
with closing(
self._worker_model.start_listening(
response_socket_id)) as sock:
result = self._worker_model.get_json_message(sock, 60)
if result is None: # dead workers are a fact of life now
logging.info(
'Unable to reach worker, bundle state {}'.format(
bundle_state))
raise NotFoundError(
'Unable to reach worker of running bundle with bundle state {}'
.format(bundle_state))
elif 'error_code' in result:
raise http_error_to_exception(result['error_code'],
result['error_message'])
target_info = result['target_info']
# Deserialize dict response sent over JSON
target_info[
'resolved_target'] = download_util.BundleTarget.from_dict(
target_info['resolved_target'])
return target_info
finally:
self._worker_model.deallocate_socket(response_socket_id)
def _get_read_response_stream(self, response_socket_id):
with closing(self._worker_model.start_listening(response_socket_id)) as sock:
header_message = self._worker_model.get_json_message(sock, 60)
precondition(header_message is not None, 'Unable to reach worker')
if 'error_code' in header_message:
raise http_error_to_exception(header_message['error_code'], header_message['error_message'])
fileobj = self._worker_model.get_stream(sock, 60)
precondition(fileobj is not None, 'Unable to reach worker')
return fileobj
def _get_read_response_stream(self, response_socket_id):
with closing(self._worker_model.start_listening(response_socket_id)) as sock:
header_message = self._worker_model.get_json_message(sock, 60)
precondition(header_message is not None, 'Unable to reach worker')
if 'error_code' in header_message:
raise http_error_to_exception(header_message['error_code'], header_message['error_message'])
fileobj = self._worker_model.get_stream(sock, 60)
precondition(fileobj is not None, 'Unable to reach worker')
return fileobj
def _get_target_info(self, uuid, path, depth):
"""
Returns information about an individual target inside the bundle, or
None if the target or bundle doesn't exist.
For information about the format of the return value, see
worker.download_util.get_target_info.
"""
try:
bundle_state = self.get_bundle_state(uuid)
except NotFoundError:
bundle_state = None
# Return None if invalid bundle reference
if bundle_state is None:
return None
if self._is_available_locally(uuid):
bundle_path = self.get_bundle_location(uuid)
try:
key = (bundle_path, uuid, path, depth)
return self._cached_if_stable(
uuid, 'target_info', key,
lambda: download_util.get_target_info(*key))
except download_util.PathException as e:
raise UsageError(e.message)
else:
# TODO: validate that this is a reasonable approach
worker = self.get_bundle_worker(uuid)
response_socket_id = self._worker_model.allocate_socket(
worker['user_id'], worker['worker_id'])
try:
read_args = {
'type': 'get_target_info',
'depth': depth,
}
self._send_read_message(worker, response_socket_id, uuid, path,
read_args)
with closing(
self._worker_model.start_listening(
response_socket_id)) as sock:
result = self._worker_model.get_json_message(sock, 60)
if result is None: # dead workers are a fact of life now
logging.info(
'Unable to reach worker, bundle state {}'.format(
bundle_state))
return None
elif 'error_code' in result:
raise http_error_to_exception(result['error_code'],
result['error_message'])
return result['target_info']
finally:
self._worker_model.deallocate_socket(response_socket_id)
def get_target_info(self, uuid, path, depth):
"""
Returns information about an individual target inside the bundle, or
None if the target or bundle doesn't exist.
For information about the format of the return value, see
worker.download_util.get_target_info.
"""
try:
bundle_state = self._bundle_model.get_bundle_state(uuid)
except NotFoundError:
bundle_state = None
# Return None if invalid bundle reference
if bundle_state is None:
return None
elif bundle_state != State.RUNNING:
bundle_path = self._bundle_store.get_bundle_location(uuid)
try:
return download_util.get_target_info(bundle_path, uuid, path,
depth)
except download_util.PathException as e:
raise UsageError(e.message)
else:
# get_target_info calls are sent to the worker even on a shared file
# system since 1) due to NFS caching the worker has more up to date
# information on directory contents, and 2) the logic of hiding
# the dependency paths doesn't need to be re-implemented here.
worker = self._worker_model.get_bundle_worker(uuid)
response_socket_id = self._worker_model.allocate_socket(
worker['user_id'], worker['worker_id'])
try:
read_args = {
'type': 'get_target_info',
'depth': depth,
}
self._send_read_message(worker, response_socket_id, uuid, path,
read_args)
with closing(
self._worker_model.start_listening(
response_socket_id)) as sock:
result = self._worker_model.get_json_message(sock, 60)
if result is None: # dead workers are a fact of life now
logging.info(
'Unable to reach worker, bundle state {}'.format(
bundle_state))
return None
elif 'error_code' in result:
raise http_error_to_exception(result['error_code'],
result['error_message'])
return result['target_info']
finally:
self._worker_model.deallocate_socket(response_socket_id)
def get_target_info(self, uuid, path, depth):
"""
Returns information about an individual target inside the bundle,
Raises NotFoundError if the bundle or the path within the bundle is not found
For information about the format of the return value, see
worker.download_util.get_target_info.
"""
bundle_state = self._bundle_model.get_bundle_state(uuid)
# Raises NotFoundException if uuid is invalid
if bundle_state == State.PREPARING:
raise NotFoundError(
"Bundle {} hasn't started running yet, files not available".format(uuid)
)
elif bundle_state != State.RUNNING:
bundle_path = self._bundle_store.get_bundle_location(uuid)
try:
return download_util.get_target_info(bundle_path, uuid, path, depth)
except download_util.PathException as e:
raise NotFoundError(e.message)
else:
# get_target_info calls are sent to the worker even on a shared file
# system since 1) due to NFS caching the worker has more up to date
# information on directory contents, and 2) the logic of hiding
# the dependency paths doesn't need to be re-implemented here.
worker = self._worker_model.get_bundle_worker(uuid)
response_socket_id = self._worker_model.allocate_socket(
worker['user_id'], worker['worker_id']
)
try:
read_args = {'type': 'get_target_info', 'depth': depth}
self._send_read_message(worker, response_socket_id, uuid, path, read_args)
with closing(self._worker_model.start_listening(response_socket_id)) as sock:
result = self._worker_model.get_json_message(sock, 60)
if result is None: # dead workers are a fact of life now
logging.info('Unable to reach worker, bundle state {}'.format(bundle_state))
raise NotFoundError(
'Unable to reach worker of running bundle with bundle state {}'.format(
bundle_state
)
)
elif 'error_code' in result:
raise http_error_to_exception(result['error_code'], result['error_message'])
return result['target_info']
finally:
self._worker_model.deallocate_socket(response_socket_id)
def wrapper(*args, **kwargs):
# Always pop out the 'client' kwarg
client = kwargs.pop('client', None)
try:
# Test to see if request context is initialized
_ = request.user.user_id
except (AttributeError, RuntimeError):
# Request context not initialized: we are NOT in a Bottle app
# Fabricate a thread-local context for LocalBundleClient
if client is not None:
user_id = client._current_user_id()
# User will be None if not logged in (a 'public' user)
from codalab.objects.user import PUBLIC_USER
if user_id is None:
user = PUBLIC_USER
else:
user = client.model.get_user(user_id=user_id)
local_bundle_client_context.local = client
local_bundle_client_context.request = DummyRequest(user)
precondition((hasattr(local_bundle_client_context, 'local') and
hasattr(local_bundle_client_context, 'request')),
'LocalBundleClient environment failed to initialize')
try:
# Shim in local and request
return f(local_bundle_client_context.local,
local_bundle_client_context.request,
*args, **kwargs)
except HTTPError as e:
# Translate HTTP errors back to CodaLab exceptions
raise http_error_to_exception(e.status_code, e.message)
finally:
# Clean up when this request is done, thread may be recycled
# But should only do this on the root call, where 'client'
# was passed as a kwarg: all recursive calls of REST methods
# that derive from the original call need to use the same
# context.
if client is not None:
delattr(local_bundle_client_context, 'local')
delattr(local_bundle_client_context, 'request')
else:
# We are in the Bottle app, all is good
return f(local, request, *args, **kwargs)
def wrapper(*args, **kwargs):
try:
return f(*args, **kwargs)
except urllib.error.HTTPError as e:
# Standardize the data type of error_body generated from different objects (e.g. bottle.HTTPError or
# urllib.error.HTTPError) to string. The data type of error message generated by
# urllib.error.HTTPError is string by default. The one generated by bottle.HTTPError is bytes.
error_body = e.read().decode() if isinstance(
e.read(), bytes) else e.read()
# Translate known errors to the standard CodaLab errors
exc = http_error_to_exception(e.code, error_body)
# All standard CodaLab errors are subclasses of UsageError
if isinstance(exc, UsageError):
six.reraise(exc.__class__, exc, sys.exc_info()[2])
else:
# Shunt other exceptions into one class
six.reraise(
JsonApiException,
JsonApiException(
message.format(*args, **kwargs) + ': ' +
http.client.responses[e.code] + ' - ' + error_body,
400 <= e.code < 500,
),
sys.exc_info()[2],
)
except RestClientException as e:
six.reraise(
JsonApiException,
JsonApiException(
message.format(*args, **kwargs) + ': ' + str(e),
e.client_error),
sys.exc_info()[2],
)
except (urllib.error.URLError, http.client.HTTPException,
socket.error) as e:
six.reraise(
JsonApiException,
JsonApiException(
message.format(*args, **kwargs) + ': ' + str(e),
False),
sys.exc_info()[2],
)
