def get_bundle_title_js(uuid):
"""
Returns Javascript code that updates the bundle title.
"""
bundle = local.model.get_bundle(uuid)
check_bundles_have_read_permission(local.model, request.user, [uuid])
return template('title_setter_js', title=bundle.metadata.name)
def _netcurl_bundle(uuid, port, path=''):
"""
Forward an HTTP request into the specified port of the running bundle with uuid.
Return the HTTP response from this bundle.
"""
check_bundles_have_read_permission(local.model, request.user, [uuid])
bundle = local.model.get_bundle(uuid)
if bundle.state in State.FINAL_STATES:
abort(http.client.FORBIDDEN, 'Cannot netcurl bundle, bundle already finalized.')
try:
request.path_shift(4) # shift away the routing parts of the URL
headers_string = [
'{}: {}'.format(h, request.headers.get(h)) for h in request.headers.keys()
]
message = "{} {} HTTP/1.1\r\n".format(request.method, request.path)
message += "\r\n".join(headers_string) + "\r\n"
message += "\r\n"
message += request.body.read()
info = local.download_manager.netcat(uuid, port, message)
except Exception:
print("{}".format(request.environ), file=sys.stderr)
raise
finally:
request.path_shift(-4) # restore the URL
return info
def _fetch_bundle_contents_info(uuid, path=''):
depth = query_get_type(int, 'depth', default=0)
if depth < 0:
abort(httplib.BAD_REQUEST, "Depth must be at least 0")
check_bundles_have_read_permission(local.model, request.user, [uuid])
return {'data': local.download_manager.get_target_info(uuid, path, depth)}
def head_target(self, target, num_lines):
check_bundles_have_read_permission(self.model, self._current_user(), [target[0]])
path = self.get_target_path(target)
lines = path_util.read_lines(path, num_lines)
if lines == None: return None
import base64
return map(base64.b64encode, lines)
def get_blob(uuid, path=''):
"""
API to download the contents of a bundle or a subpath within a bundle.
For directories this method always returns a tarred and gzipped archive of
the directory.
For files, if the request has an Accept-Encoding header containing gzip,
then the returned file is gzipped.
"""
check_bundles_have_read_permission(local.model, request.user, [uuid])
bundle = local.model.get_bundle(uuid)
target_info = local.download_manager.get_target_info(uuid, path, 0)
if target_info is None:
abort(httplib.NOT_FOUND, 'Not found.')
# Figure out the file name.
if not path and bundle.metadata.name:
filename = bundle.metadata.name
else:
filename = target_info['name']
if target_info['type'] == 'directory':
# Always tar and gzip directories.
filename = filename + '.tar.gz'
fileobj = local.download_manager.stream_tarred_gzipped_directory(
uuid, path)
elif target_info['type'] == 'file':
if not zip_util.path_is_archive(
filename) and request_accepts_gzip_encoding():
# Let's gzip to save bandwidth. The browser will transparently decode
# the file.
filename = filename + '.gz'
fileobj = local.download_manager.stream_file(uuid,
path,
gzipped=True)
else:
fileobj = local.download_manager.stream_file(uuid,
path,
gzipped=False)
else:
# Symlinks.
abort(httplib.FORBIDDEN, 'Cannot download files of this type.')
# Set headers.
mimetype, _ = mimetypes.guess_type(filename, strict=False)
response.set_header('Content-Type', mimetype or 'text/plain')
if zip_util.get_archive_ext(
filename) == '.gz' and request_accepts_gzip_encoding():
filename = zip_util.strip_archive_ext(filename)
response.set_header('Content-Encoding', 'gzip')
else:
response.set_header('Content-Encoding', 'identity')
response.set_header('Content-Disposition', 'filename="%s"' % filename)
return fileobj
def _netcat_bundle(uuid, port):
"""
Send a raw bytestring into the specified port of the running bundle with uuid.
Return the response from this bundle.
"""
check_bundles_have_read_permission(local.model, request.user, [uuid])
bundle = local.model.get_bundle(uuid)
if bundle.state in State.FINAL_STATES:
abort(httplib.FORBIDDEN, 'Cannot netcat bundle, bundle already finalized.')
info = local.download_manager.netcat(uuid, port, request.json['message'])
return {'data': info}
def _netcat_bundle(uuid, port):
"""
Send a raw bytestring into the specified port of the running bundle with uuid.
Return the response from this bundle.
"""
check_bundles_have_read_permission(local.model, request.user, [uuid])
bundle = local.model.get_bundle(uuid)
print(bundle.state)
if bundle.state != State.RUNNING:
abort(http.client.FORBIDDEN, 'Cannot netcat bundle, bundle not running.')
info = local.download_manager.netcat(uuid, port, request.json['message'])
return {'data': info}
def get_blob(uuid, path=''):
"""
API to download the contents of a bundle or a subpath within a bundle.
For directories this method always returns a tarred and gzipped archive of
the directory.
For files, if the request has an Accept-Encoding header containing gzip,
then the returned file is gzipped.
"""
check_bundles_have_read_permission(local.model, request.user, [uuid])
bundle = local.model.get_bundle(uuid)
target_info = local.download_manager.get_target_info(uuid, path, 0)
if target_info is None:
abort(httplib.NOT_FOUND, 'Not found.')
# Figure out the file name.
if not path and bundle.metadata.name:
filename = bundle.metadata.name
else:
filename = target_info['name']
if target_info['type'] == 'directory':
# Always tar and gzip directories.
filename = filename + '.tar.gz'
fileobj = local.download_manager.stream_tarred_gzipped_directory(uuid, path)
elif target_info['type'] == 'file':
if not zip_util.path_is_archive(filename) and request_accepts_gzip_encoding():
# Let's gzip to save bandwidth. The browser will transparently decode
# the file.
filename = filename + '.gz'
fileobj = local.download_manager.stream_file(uuid, path, gzipped=True)
else:
fileobj = local.download_manager.stream_file(uuid, path, gzipped=False)
else:
# Symlinks.
abort(httplib.FORBIDDEN, 'Cannot download files of this type.')
# Set headers.
mimetype, _ = mimetypes.guess_type(filename, strict=False)
response.set_header('Content-Type', mimetype or 'text/plain')
if zip_util.get_archive_ext(filename) == '.gz' and request_accepts_gzip_encoding():
filename = zip_util.strip_archive_ext(filename)
response.set_header('Content-Encoding', 'gzip')
else:
response.set_header('Content-Encoding', 'identity')
response.set_header('Content-Disposition', 'filename="%s"' % filename)
return fileobj
def _netcurl_bundle(uuid, port, path=''):
"""
Forward an HTTP request into the specified port of the running bundle with uuid.
Return the HTTP response from this bundle.
"""
check_bundles_have_read_permission(local.model, request.user, [uuid])
bundle = local.model.get_bundle(uuid)
if bundle.state in State.FINAL_STATES:
abort(http.client.FORBIDDEN,
'Cannot netcurl bundle, bundle already finalized.')
try:
request.path_shift(4) # shift away the routing parts of the URL
# Put the request headers into the message
headers_string = [
'{}: {}'.format(h, request.headers.get(h))
for h in request.headers.keys()
]
message = "{} {} HTTP/1.1\r\n".format(request.method, request.path)
message += "\r\n".join(headers_string) + "\r\n"
message += "\r\n"
message += request.body.read().decode(
) # Assume bytes can be decoded to string
bytestring = local.download_manager.netcat(uuid, port, message)
# Parse the response
class FakeSocket:
def __init__(self, bytestring):
self._file = BytesIO(bytestring)
def makefile(self, *args, **kwargs):
return self._file
new_response = HTTPResponse(FakeSocket(bytestring))
new_response.begin()
# Copy the headers over
for k in new_response.headers:
response.headers[k] = new_response.headers[k]
# Return the response (which sends back the body)
return new_response
except Exception:
logger.exception(
"Failed to forward HTTP request for the bundle with uuid: {} for environment: {}"
.format(uuid, request.environ))
raise
finally:
request.path_shift(-4) # restore the URL
def _netcat_bundle(uuid, port):
"""
Send a raw bytestring into the specified port of the running bundle with uuid.
Return the response from this bundle.
"""
# Note that read permission is enough to hit the port (same for netcurl).
# This allows users to host demos that the public can play with. In
# general, this is not safe, since hitting a port can mutate what's
# happening in the bundle. In the future, we might want to make people
# explicitly expose ports to the world.
check_bundles_have_read_permission(local.model, request.user, [uuid])
bundle = local.model.get_bundle(uuid)
if bundle.state != State.RUNNING:
abort(http.client.FORBIDDEN, 'Cannot netcat bundle, bundle not running.')
return local.download_manager.netcat(uuid, port, request.json['message'])
def _fetch_bundle_contents_info(uuid, path=''):
"""
Fetch metadata of the bundle contents or a subpath within the bundle.
Query parameters:
- `depth`: recursively fetch subdirectory info up to this depth.
Default is 0.
Response format:
```
{
"data": {
"name": "<name of file or directory>",
"link": "<string representing target if file is a symbolic link>",
"type": "<file|directory|link>",
"size": <size of file in bytes>,
"perm": <unix permission integer>,
"contents": [
{
"name": ...,
<each file of directory represented recursively with the same schema>
},
...
]
}
}
```
"""
depth = query_get_type(int, 'depth', default=0)
target = BundleTarget(uuid, path)
if depth < 0:
abort(http.client.BAD_REQUEST, "Depth must be at least 0")
check_bundles_have_read_permission(local.model, request.user, [uuid])
try:
info = local.download_manager.get_target_info(target, depth)
# Object is not JSON serializable so submit its dict in API response
# The client is responsible for deserializing it
info['resolved_target'] = info['resolved_target'].__dict__
except NotFoundError as e:
abort(http.client.NOT_FOUND, str(e))
except Exception as e:
abort(http.client.BAD_REQUEST, str(e))
return {'data': info}
def _fetch_bundle_contents_info(uuid, path=''):
"""
Fetch metadata of the bundle contents or a subpath within the bundle.
Query parameters:
- `depth`: recursively fetch subdirectory info up to this depth.
Default is 0.
Response format:
```
{
"data": {
"name": "<name of file or directory>",
"link": "<string representing target if file is a symbolic link>",
"type": "<file|directory|link>",
"size": <size of file in bytes>,
"perm": <unix permission integer>,
"contents": [
{
"name": ...,
<each file of directory represented recursively with the same schema>
},
...
]
}
}
```
"""
depth = query_get_type(int, 'depth', default=0)
if depth < 0:
abort(httplib.BAD_REQUEST, "Depth must be at least 0")
check_bundles_have_read_permission(local.model, request.user, [uuid])
info = local.download_manager.get_target_info(uuid, path, depth)
if info is None:
abort(httplib.NOT_FOUND, 'Bundle not found')
return {
'data': info
}
def get_target_info(self, target, depth):
check_bundles_have_read_permission(self.model, self._current_user(), [target[0]])
path = self.get_target_path(target)
return path_util.get_info(path, depth)
def _fetch_bundle_contents_blob(uuid, path=''):
"""
API to download the contents of a bundle or a subpath within a bundle.
For directories this method always returns a tarred and gzipped archive of
the directory.
For files, if the request has an Accept-Encoding header containing gzip,
then the returned file is gzipped.
"""
byte_range = get_request_range()
head_lines = query_get_type(int, 'head', default=0)
tail_lines = query_get_type(int, 'tail', default=0)
max_line_length = query_get_type(int, 'max_line_length', default=128)
check_bundles_have_read_permission(local.model, request.user, [uuid])
bundle = local.model.get_bundle(uuid)
target_info = local.download_manager.get_target_info(uuid, path, 0)
if target_info is None:
abort(httplib.NOT_FOUND, 'Not found.')
# Figure out the file name.
if not path and bundle.metadata.name:
filename = bundle.metadata.name
else:
filename = target_info['name']
if target_info['type'] == 'directory':
if byte_range:
abort(httplib.BAD_REQUEST,
'Range not supported for directory blobs.')
if head_lines:
abort(httplib.BAD_REQUEST,
'Head not supported for directory blobs.')
# Always tar and gzip directories.
filename = filename + '.tar.gz'
fileobj = local.download_manager.stream_tarred_gzipped_directory(
uuid, path)
elif target_info['type'] == 'file':
gzipped = False
if not zip_util.path_is_archive(
filename) and request_accepts_gzip_encoding():
# Let's gzip to save bandwidth. The browser will transparently decode
# the file.
filename = filename + '.gz'
gzipped = True
if byte_range and (head_lines or tail_lines):
abort(httplib.BAD_REQUEST,
'Head and range not supported on the same request.')
elif byte_range:
start, end = byte_range
fileobj = local.download_manager.read_file_section(
uuid, path, start, end - start + 1, gzipped)
elif head_lines or tail_lines:
fileobj = local.download_manager.summarize_file(
uuid, path, head_lines, tail_lines, max_line_length, None,
gzipped)
else:
fileobj = local.download_manager.stream_file(uuid, path, gzipped)
else:
# Symlinks.
abort(httplib.FORBIDDEN, 'Cannot download files of this type.')
# Set headers.
mimetype, _ = mimetypes.guess_type(filename, strict=False)
response.set_header('Content-Type', mimetype or 'text/plain')
if zip_util.get_archive_ext(
filename) == '.gz' and request_accepts_gzip_encoding():
filename = zip_util.strip_archive_ext(filename)
response.set_header('Content-Encoding', 'gzip')
else:
response.set_header('Content-Encoding', 'identity')
response.set_header('Content-Disposition', 'filename="%s"' % filename)
return fileobj
def open_target(self, target):
check_bundles_have_read_permission(self.model, self._current_user(), [target[0]])
path = self.get_target_path(target)
path_util.check_isfile(path, 'open_target')
return open(path)
def mimic(self, old_inputs, old_output, new_inputs, new_output_name, worksheet_uuid, depth, shadow, dry_run):
'''
old_inputs: list of bundle uuids
old_output: bundle uuid that we produced
new_inputs: list of bundle uuids that are analogous to old_inputs
new_output_name: name of the bundle to create to be analogous to old_output (possibly None)
worksheet_uuid: add newly created bundles to this worksheet
depth: how far to do a BFS up from old_output.
shadow: whether to add the new inputs right after all occurrences of the old inputs in worksheets.
'''
check_worksheet_has_all_permission(self.model, self._current_user(), self.model.get_worksheet(worksheet_uuid, fetch_items=False))
#print 'old_inputs: %s, new_inputs: %s, old_output: %s, new_output_name: %s' % (old_inputs, new_inputs, old_output, new_output_name)
# Build the graph (get all the infos).
# If old_output is given, look at ancestors of old_output until we
# reached some depth. If it's not given, we first get all the
# descendants first, and then get their ancestors.
infos = {} # uuid -> bundle info
if old_output:
bundle_uuids = [old_output]
else:
bundle_uuids = self.model.get_self_and_descendants(old_inputs, depth=depth)
all_bundle_uuids = list(bundle_uuids) # should be infos.keys() in order
for _ in range(depth):
new_bundle_uuids = []
for bundle_uuid in bundle_uuids:
if bundle_uuid in infos: continue # Already visited
info = infos[bundle_uuid] = self.get_bundle_info(bundle_uuid)
for dep in info['dependencies']:
parent_uuid = dep['parent_uuid']
if parent_uuid not in infos:
new_bundle_uuids.append(parent_uuid)
all_bundle_uuids = new_bundle_uuids + all_bundle_uuids
bundle_uuids = new_bundle_uuids
# Make sure we have read access to all the bundles involved here.
check_bundles_have_read_permission(self.model, self._current_user(), list(infos.keys()))
# Now go recursively create the bundles.
old_to_new = {} # old_uuid -> new_uuid
downstream = set() # old_uuid -> whether we're downstream of an input (and actually needs to be mapped onto a new uuid)
created_uuids = set() # set of uuids which were newly created
plan = [] # sequence of (old, new) bundle infos to make
for old, new in zip(old_inputs, new_inputs):
old_to_new[old] = new
downstream.add(old)
# Return corresponding new_bundle_uuid
def recurse(old_bundle_uuid):
if old_bundle_uuid in old_to_new:
#print old_bundle_uuid, 'cached'
return old_to_new[old_bundle_uuid]
# Don't have any more information (because we probably hit the maximum depth)
if old_bundle_uuid not in infos:
#print old_bundle_uuid, 'no information'
return old_bundle_uuid
# Get information about the old bundle.
info = infos[old_bundle_uuid]
new_dependencies = [{
'parent_uuid': recurse(dep['parent_uuid']),
'parent_path': dep['parent_path'],
'child_uuid': dep['child_uuid'], # This is just a placeholder to do the equality test
'child_path': dep['child_path']
} for dep in info['dependencies']]
# If there are no inputs or if we're downstream of any inputs, we need to make a new bundle.
lone_output = (len(old_inputs) == 0 and old_bundle_uuid == old_output)
downstream_of_inputs = any(dep['parent_uuid'] in downstream for dep in info['dependencies'])
if lone_output or downstream_of_inputs:
# Now create a new bundle that mimics the old bundle.
# Only change the name if the output name is supplied.
old_bundle_name = info['metadata']['name']
new_info = copy.deepcopy(info)
new_metadata = new_info['metadata']
if new_output_name:
if old_bundle_uuid == old_output:
new_metadata['name'] = new_output_name
else:
# Just make up a name heuristically
new_metadata['name'] = new_output_name + '-' + info['metadata']['name']
# Remove all the automatically generated keys
cls = get_bundle_subclass(new_info['bundle_type'])
for spec in cls.METADATA_SPECS:
if spec.generated and spec.key in new_metadata:
new_metadata.pop(spec.key)
# Set the targets
targets = [(dep['child_path'], (dep['parent_uuid'], dep['parent_path'])) for dep in new_dependencies]
if dry_run:
new_bundle_uuid = None
else:
if new_info['bundle_type'] not in ('make', 'run'):
raise UsageError('Can\'t mimic %s since it is not make or run' % old_bundle_uuid)
new_bundle_uuid = self._derive_bundle(new_info['bundle_type'], \
targets, new_info['command'], new_metadata, worksheet_uuid)
new_info['uuid'] = new_bundle_uuid
plan.append((info, new_info))
downstream.add(old_bundle_uuid)
created_uuids.add(new_bundle_uuid)
else:
new_bundle_uuid = old_bundle_uuid
old_to_new[old_bundle_uuid] = new_bundle_uuid # Cache it
return new_bundle_uuid
if old_output:
recurse(old_output)
else:
# Don't have a particular output we're targetting, so just create
# new versions of all the uuids.
for uuid in all_bundle_uuids:
recurse(uuid)
# Add to worksheet
if not dry_run:
if shadow:
# Add each new bundle in the "shadow" of the old_bundle (right after it).
for old_bundle_uuid, new_bundle_uuid in old_to_new.items():
if new_bundle_uuid in created_uuids: # Only add novel bundles
self.model.add_shadow_worksheet_items(old_bundle_uuid, new_bundle_uuid)
else:
def newline():
self.model.add_worksheet_item(worksheet_uuid, worksheet_util.markup_item(''))
# A prelude of a bundle on a worksheet is the set of items that occur right before it (markup, directives, etc.)
# Let W be the first worksheet containing the old_inputs[0].
# Add all items on that worksheet that appear in old_to_new along with their preludes.
# For items not on this worksheet, add them at the end (instead of making them floating).
if old_output:
anchor_uuid = old_output
elif len(old_inputs) > 0:
anchor_uuid = old_inputs[0]
host_worksheet_uuids = self.model.get_host_worksheet_uuids([anchor_uuid])[anchor_uuid]
new_bundle_uuids_added = set()
skipped = True # Whether there were items that we didn't include in the prelude (in which case we want to put '')
if len(host_worksheet_uuids) > 0:
# Choose a single worksheet.
if worksheet_uuid in host_worksheet_uuids:
# If current worksheet is one of them, favor that one.
host_worksheet_uuid = worksheet_uuid
else:
# Choose an arbitrary one (in the future, have a better way of canonicalizing).
host_worksheet_uuid = host_worksheet_uuids[0]
# Fetch the worksheet
worksheet_info = self.get_worksheet_info(host_worksheet_uuid, fetch_items=True)
prelude_items = [] # The prelude that we're building up
for item in worksheet_info['items']:
(bundle_info, subworkheet_info, value_obj, item_type) = item
just_added = False
if item_type == worksheet_util.TYPE_BUNDLE:
old_bundle_uuid = bundle_info['uuid']
if old_bundle_uuid in old_to_new:
# Flush the prelude gathered so far.
new_bundle_uuid = old_to_new[old_bundle_uuid]
if new_bundle_uuid in created_uuids: # Only add novel bundles
# Stand in for things skipped (this is important so directives have proper extent).
if skipped:
newline()
# Add prelude and items
for item2 in prelude_items:
self.add_worksheet_item(worksheet_uuid, worksheet_util.convert_item_to_db(item2))
self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(new_bundle_uuid))
new_bundle_uuids_added.add(new_bundle_uuid)
just_added = True
if (item_type == worksheet_util.TYPE_MARKUP and value_obj != '') or item_type == worksheet_util.TYPE_DIRECTIVE:
prelude_items.append(item) # Include in prelude
skipped = False
else:
prelude_items = [] # Reset
skipped = not just_added
# Add the bundles that haven't been added yet
for info, new_info in plan:
new_bundle_uuid = new_info['uuid']
if new_bundle_uuid not in new_bundle_uuids_added:
if skipped:
newline()
skipped = False
self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(new_bundle_uuid))
return plan
def download_target(self, target, follow_symlinks):
check_bundles_have_read_permission(self.model, self._current_user(), [target[0]])
# Don't need to download anything because it's already local.
# Note that we can't really enforce follow_symlinks, but this is okay,
# because we will follow them when we copy it from the target path.
return (self.get_target_path(target), None)
def open_target_handle(self, target):
check_bundles_have_read_permission(self.model, self._current_user(), [target[0]])
path = self.get_target_path(target)
return open(path) if path and os.path.exists(path) else None
def check_target_has_read_permission(target):
check_bundles_have_read_permission(local.model, request.user, [target[0]])
def _fetch_bundle_contents_blob(uuid, path=''):
"""
API to download the contents of a bundle or a subpath within a bundle.
For directories, this method always returns a tarred and gzipped archive of
the directory.
For files, if the request has an Accept-Encoding header containing gzip,
then the returned file is gzipped. Otherwise, the file is returned as-is.
HTTP Request headers:
- `Range: bytes=<start>-<end>`: fetch bytes from the range
`[<start>, <end>)`.
- `Accept-Encoding: <encoding>`: indicate that the client can accept
encoding `<encoding>`. Currently only `gzip` encoding is supported.
Query parameters:
- `head`: number of lines to fetch from the beginning of the file.
Default is 0, meaning to fetch the entire file.
- `tail`: number of lines to fetch from the end of the file.
Default is 0, meaning to fetch the entire file.
- `max_line_length`: maximum number of characters to fetch from each line,
if either `head` or `tail` is specified. Default is 128.
HTTP Response headers (for single-file targets):
- `Content-Disposition: inline; filename=<bundle name or target filename>`
- `Content-Type: <guess of mimetype based on file extension>`
- `Content-Encoding: [gzip|identity]`
- `Target-Type: file`
HTTP Response headers (for directories):
- `Content-Disposition: attachment; filename=<bundle or directory name>.tar.gz`
- `Content-Type: application/gzip`
- `Content-Encoding: identity`
- `Target-Type: directory`
"""
byte_range = get_request_range()
head_lines = query_get_type(int, 'head', default=0)
tail_lines = query_get_type(int, 'tail', default=0)
truncation_text = query_get_type(str, 'truncation_text', default='')
max_line_length = query_get_type(int, 'max_line_length', default=128)
check_bundles_have_read_permission(local.model, request.user, [uuid])
target = BundleTarget(uuid, path)
try:
target_info = local.download_manager.get_target_info(target, 0)
if target_info['resolved_target'] != target:
check_bundles_have_read_permission(
local.model, request.user,
[target_info['resolved_target'].bundle_uuid])
target = target_info['resolved_target']
except NotFoundError as e:
abort(http.client.NOT_FOUND, str(e))
except Exception as e:
abort(http.client.BAD_REQUEST, str(e))
# Figure out the file name.
bundle_name = local.model.get_bundle(target.bundle_uuid).metadata.name
if not path and bundle_name:
filename = bundle_name
else:
filename = target_info['name']
if target_info['type'] == 'directory':
if byte_range:
abort(http.client.BAD_REQUEST,
'Range not supported for directory blobs.')
if head_lines or tail_lines:
abort(http.client.BAD_REQUEST,
'Head and tail not supported for directory blobs.')
# Always tar and gzip directories
gzipped_stream = False # but don't set the encoding to 'gzip'
mimetype = 'application/gzip'
filename += '.tar.gz'
fileobj = local.download_manager.stream_tarred_gzipped_directory(
target)
elif target_info['type'] == 'file':
# Let's gzip to save bandwidth.
# For simplicity, we do this even if the file is already a packed
# archive (which should be relatively rare).
# The browser will transparently decode the file.
gzipped_stream = request_accepts_gzip_encoding()
# Since guess_type() will interpret '.tar.gz' as an 'application/x-tar' file
# with 'gzip' encoding, which would usually go into the Content-Encoding
# header. But if the bundle contents is actually a packed archive, we don't
# want the client to automatically decompress the file, so we don't want to
# set the Content-Encoding header. Instead, if guess_type() detects an
# archive, we just set mimetype to indicate an arbitrary binary file.
mimetype, encoding = mimetypes.guess_type(filename, strict=False)
if encoding is not None:
mimetype = 'application/octet-stream'
if byte_range and (head_lines or tail_lines):
abort(http.client.BAD_REQUEST,
'Head and range not supported on the same request.')
elif byte_range:
start, end = byte_range
fileobj = local.download_manager.read_file_section(
target, start, end - start + 1, gzipped_stream)
elif head_lines or tail_lines:
fileobj = local.download_manager.summarize_file(
target, head_lines, tail_lines, max_line_length,
truncation_text, gzipped_stream)
else:
fileobj = local.download_manager.stream_file(
target, gzipped_stream)
else:
# Symlinks.
abort(http.client.FORBIDDEN,
'Cannot download files of this type (%s).' % target_info['type'])
# Set headers.
response.set_header('Content-Type', mimetype or 'text/plain')
response.set_header('Content-Encoding',
'gzip' if gzipped_stream else 'identity')
if target_info['type'] == 'file':
response.set_header('Content-Disposition',
'inline; filename="%s"' % filename)
else:
response.set_header('Content-Disposition',
'attachment; filename="%s"' % filename)
response.set_header('Target-Type', target_info['type'])
return fileobj
def cat_target(self, target, out):
check_bundles_have_read_permission(self.model, self._current_user(), [target[0]])
path = self.get_target_path(target)
path_util.cat(path, out)
def _stage_bundles(self):
"""
Stages bundles by:
1) Failing any bundles that have any missing or failed dependencies.
2) Staging any bundles that have all ready dependencies.
"""
bundles = self._model.batch_get_bundles(state=State.CREATED)
parent_uuids = set(
dep.parent_uuid for bundle in bundles for dep in bundle.dependencies)
parents = self._model.batch_get_bundles(uuid=parent_uuids)
all_parent_states = {parent.uuid: parent.state for parent in parents}
all_parent_uuids = set(all_parent_states)
bundles_to_fail = []
bundles_to_stage = []
for bundle in bundles:
parent_uuids = set(dep.parent_uuid for dep in bundle.dependencies)
try:
check_bundles_have_read_permission(self._model, self._model.get_user(bundle.owner_id), parent_uuids)
except PermissionError as e:
bundles_to_fail.append(
(bundle, str(e))
)
continue
missing_uuids = parent_uuids - all_parent_uuids
if missing_uuids:
bundles_to_fail.append(
(bundle,
'Missing parent bundles: %s' % ', '.join(missing_uuids)))
continue
parent_states = {uuid: all_parent_states[uuid]
for uuid in parent_uuids}
acceptable_states = [State.READY]
if bundle.metadata.allow_failed_dependencies:
acceptable_states.append(State.FAILED)
acceptable_states.append(State.KILLED)
else:
failed_uuids = [
uuid for uuid, state in parent_states.iteritems()
if state == State.FAILED]
if failed_uuids:
bundles_to_fail.append(
(bundle,
'Parent bundles failed: %s' % ', '.join(failed_uuids)))
continue
if all(state in acceptable_states for state in parent_states.itervalues()):
bundles_to_stage.append(bundle)
for bundle, failure_message in bundles_to_fail:
logger.info('Failing bundle %s: %s', bundle.uuid, failure_message)
self._model.update_bundle(
bundle, {'state': State.FAILED,
'metadata': {'failure_message': failure_message}})
for bundle in bundles_to_stage:
logger.info('Staging %s', bundle.uuid)
self._model.update_bundle(
bundle, {'state': State.STAGED})
def _stage_bundles(self):
"""
Stages bundles by:
1) Failing any bundles that have any missing or failed dependencies.
2) Staging any bundles that have all ready dependencies.
"""
bundles = self._model.batch_get_bundles(state=State.CREATED)
parent_uuids = set(dep.parent_uuid for bundle in bundles for dep in bundle.dependencies)
parents = self._model.batch_get_bundles(uuid=parent_uuids)
all_parent_states = {parent.uuid: parent.state for parent in parents}
all_parent_uuids = set(all_parent_states)
bundles_to_fail = []
bundles_to_stage = []
for bundle in bundles:
parent_uuids = set(dep.parent_uuid for dep in bundle.dependencies)
try:
check_bundles_have_read_permission(
self._model, self._model.get_user(bundle.owner_id), parent_uuids
)
except PermissionError as e:
bundles_to_fail.append((bundle, str(e)))
continue
missing_uuids = parent_uuids - all_parent_uuids
if missing_uuids:
bundles_to_fail.append(
(bundle, 'Missing parent bundles: %s' % ', '.join(missing_uuids))
)
continue
parent_states = {uuid: all_parent_states[uuid] for uuid in parent_uuids}
acceptable_states = [State.READY]
if bundle.metadata.allow_failed_dependencies:
acceptable_states.append(State.FAILED)
acceptable_states.append(State.KILLED)
else:
failed_uuids = [
uuid for uuid, state in parent_states.iteritems() if state == State.FAILED
]
killed_uuids = [
uuid for uuid, state in parent_states.iteritems() if state == State.KILLED
]
failure_message = ''
if failed_uuids:
failure_message += ' Parent bundles failed: %s' % ', '.join(failed_uuids)
if killed_uuids:
failure_message += ' Parent bundles were killed: %s' % ', '.join(killed_uuids)
if failure_message:
failure_message += ' (Please use the --allow-failed-dependencies flag to depend on results fo failed or killed bundles)'
bundles_to_fail.append((bundle, failure_message))
continue
if all(state in acceptable_states for state in parent_states.itervalues()):
bundles_to_stage.append(bundle)
for bundle, failure_message in bundles_to_fail:
logger.info('Failing bundle %s: %s', bundle.uuid, failure_message)
self._model.update_bundle(
bundle, {'state': State.FAILED, 'metadata': {'failure_message': failure_message}}
)
for bundle in bundles_to_stage:
logger.info('Staging %s', bundle.uuid)
self._model.update_bundle(bundle, {'state': State.STAGED})