def get_bundle_title_js(uuid): """ Returns Javascript code that updates the bundle title. """ bundle = local.model.get_bundle(uuid) check_bundles_have_read_permission(local.model, request.user, [uuid]) return template('title_setter_js', title=bundle.metadata.name)
def _netcurl_bundle(uuid, port, path=''): """ Forward an HTTP request into the specified port of the running bundle with uuid. Return the HTTP response from this bundle. """ check_bundles_have_read_permission(local.model, request.user, [uuid]) bundle = local.model.get_bundle(uuid) if bundle.state in State.FINAL_STATES: abort(http.client.FORBIDDEN, 'Cannot netcurl bundle, bundle already finalized.') try: request.path_shift(4) # shift away the routing parts of the URL headers_string = [ '{}: {}'.format(h, request.headers.get(h)) for h in request.headers.keys() ] message = "{} {} HTTP/1.1\r\n".format(request.method, request.path) message += "\r\n".join(headers_string) + "\r\n" message += "\r\n" message += request.body.read() info = local.download_manager.netcat(uuid, port, message) except Exception: print("{}".format(request.environ), file=sys.stderr) raise finally: request.path_shift(-4) # restore the URL return info
def _fetch_bundle_contents_info(uuid, path=''): depth = query_get_type(int, 'depth', default=0) if depth < 0: abort(httplib.BAD_REQUEST, "Depth must be at least 0") check_bundles_have_read_permission(local.model, request.user, [uuid]) return {'data': local.download_manager.get_target_info(uuid, path, depth)}
def head_target(self, target, num_lines): check_bundles_have_read_permission(self.model, self._current_user(), [target[0]]) path = self.get_target_path(target) lines = path_util.read_lines(path, num_lines) if lines == None: return None import base64 return map(base64.b64encode, lines)
def get_blob(uuid, path=''): """ API to download the contents of a bundle or a subpath within a bundle. For directories this method always returns a tarred and gzipped archive of the directory. For files, if the request has an Accept-Encoding header containing gzip, then the returned file is gzipped. """ check_bundles_have_read_permission(local.model, request.user, [uuid]) bundle = local.model.get_bundle(uuid) target_info = local.download_manager.get_target_info(uuid, path, 0) if target_info is None: abort(httplib.NOT_FOUND, 'Not found.') # Figure out the file name. if not path and bundle.metadata.name: filename = bundle.metadata.name else: filename = target_info['name'] if target_info['type'] == 'directory': # Always tar and gzip directories. filename = filename + '.tar.gz' fileobj = local.download_manager.stream_tarred_gzipped_directory( uuid, path) elif target_info['type'] == 'file': if not zip_util.path_is_archive( filename) and request_accepts_gzip_encoding(): # Let's gzip to save bandwidth. The browser will transparently decode # the file. filename = filename + '.gz' fileobj = local.download_manager.stream_file(uuid, path, gzipped=True) else: fileobj = local.download_manager.stream_file(uuid, path, gzipped=False) else: # Symlinks. abort(httplib.FORBIDDEN, 'Cannot download files of this type.') # Set headers. mimetype, _ = mimetypes.guess_type(filename, strict=False) response.set_header('Content-Type', mimetype or 'text/plain') if zip_util.get_archive_ext( filename) == '.gz' and request_accepts_gzip_encoding(): filename = zip_util.strip_archive_ext(filename) response.set_header('Content-Encoding', 'gzip') else: response.set_header('Content-Encoding', 'identity') response.set_header('Content-Disposition', 'filename="%s"' % filename) return fileobj
def _netcat_bundle(uuid, port): """ Send a raw bytestring into the specified port of the running bundle with uuid. Return the response from this bundle. """ check_bundles_have_read_permission(local.model, request.user, [uuid]) bundle = local.model.get_bundle(uuid) if bundle.state in State.FINAL_STATES: abort(httplib.FORBIDDEN, 'Cannot netcat bundle, bundle already finalized.') info = local.download_manager.netcat(uuid, port, request.json['message']) return {'data': info}
def _netcat_bundle(uuid, port): """ Send a raw bytestring into the specified port of the running bundle with uuid. Return the response from this bundle. """ check_bundles_have_read_permission(local.model, request.user, [uuid]) bundle = local.model.get_bundle(uuid) print(bundle.state) if bundle.state != State.RUNNING: abort(http.client.FORBIDDEN, 'Cannot netcat bundle, bundle not running.') info = local.download_manager.netcat(uuid, port, request.json['message']) return {'data': info}
def get_blob(uuid, path=''): """ API to download the contents of a bundle or a subpath within a bundle. For directories this method always returns a tarred and gzipped archive of the directory. For files, if the request has an Accept-Encoding header containing gzip, then the returned file is gzipped. """ check_bundles_have_read_permission(local.model, request.user, [uuid]) bundle = local.model.get_bundle(uuid) target_info = local.download_manager.get_target_info(uuid, path, 0) if target_info is None: abort(httplib.NOT_FOUND, 'Not found.') # Figure out the file name. if not path and bundle.metadata.name: filename = bundle.metadata.name else: filename = target_info['name'] if target_info['type'] == 'directory': # Always tar and gzip directories. filename = filename + '.tar.gz' fileobj = local.download_manager.stream_tarred_gzipped_directory(uuid, path) elif target_info['type'] == 'file': if not zip_util.path_is_archive(filename) and request_accepts_gzip_encoding(): # Let's gzip to save bandwidth. The browser will transparently decode # the file. filename = filename + '.gz' fileobj = local.download_manager.stream_file(uuid, path, gzipped=True) else: fileobj = local.download_manager.stream_file(uuid, path, gzipped=False) else: # Symlinks. abort(httplib.FORBIDDEN, 'Cannot download files of this type.') # Set headers. mimetype, _ = mimetypes.guess_type(filename, strict=False) response.set_header('Content-Type', mimetype or 'text/plain') if zip_util.get_archive_ext(filename) == '.gz' and request_accepts_gzip_encoding(): filename = zip_util.strip_archive_ext(filename) response.set_header('Content-Encoding', 'gzip') else: response.set_header('Content-Encoding', 'identity') response.set_header('Content-Disposition', 'filename="%s"' % filename) return fileobj
def _netcurl_bundle(uuid, port, path=''): """ Forward an HTTP request into the specified port of the running bundle with uuid. Return the HTTP response from this bundle. """ check_bundles_have_read_permission(local.model, request.user, [uuid]) bundle = local.model.get_bundle(uuid) if bundle.state in State.FINAL_STATES: abort(http.client.FORBIDDEN, 'Cannot netcurl bundle, bundle already finalized.') try: request.path_shift(4) # shift away the routing parts of the URL # Put the request headers into the message headers_string = [ '{}: {}'.format(h, request.headers.get(h)) for h in request.headers.keys() ] message = "{} {} HTTP/1.1\r\n".format(request.method, request.path) message += "\r\n".join(headers_string) + "\r\n" message += "\r\n" message += request.body.read().decode( ) # Assume bytes can be decoded to string bytestring = local.download_manager.netcat(uuid, port, message) # Parse the response class FakeSocket: def __init__(self, bytestring): self._file = BytesIO(bytestring) def makefile(self, *args, **kwargs): return self._file new_response = HTTPResponse(FakeSocket(bytestring)) new_response.begin() # Copy the headers over for k in new_response.headers: response.headers[k] = new_response.headers[k] # Return the response (which sends back the body) return new_response except Exception: logger.exception( "Failed to forward HTTP request for the bundle with uuid: {} for environment: {}" .format(uuid, request.environ)) raise finally: request.path_shift(-4) # restore the URL
def _netcat_bundle(uuid, port): """ Send a raw bytestring into the specified port of the running bundle with uuid. Return the response from this bundle. """ # Note that read permission is enough to hit the port (same for netcurl). # This allows users to host demos that the public can play with. In # general, this is not safe, since hitting a port can mutate what's # happening in the bundle. In the future, we might want to make people # explicitly expose ports to the world. check_bundles_have_read_permission(local.model, request.user, [uuid]) bundle = local.model.get_bundle(uuid) if bundle.state != State.RUNNING: abort(http.client.FORBIDDEN, 'Cannot netcat bundle, bundle not running.') return local.download_manager.netcat(uuid, port, request.json['message'])
def _fetch_bundle_contents_info(uuid, path=''): """ Fetch metadata of the bundle contents or a subpath within the bundle. Query parameters: - `depth`: recursively fetch subdirectory info up to this depth. Default is 0. Response format: ``` { "data": { "name": "<name of file or directory>", "link": "<string representing target if file is a symbolic link>", "type": "<file|directory|link>", "size": <size of file in bytes>, "perm": <unix permission integer>, "contents": [ { "name": ..., <each file of directory represented recursively with the same schema> }, ... ] } } ``` """ depth = query_get_type(int, 'depth', default=0) target = BundleTarget(uuid, path) if depth < 0: abort(http.client.BAD_REQUEST, "Depth must be at least 0") check_bundles_have_read_permission(local.model, request.user, [uuid]) try: info = local.download_manager.get_target_info(target, depth) # Object is not JSON serializable so submit its dict in API response # The client is responsible for deserializing it info['resolved_target'] = info['resolved_target'].__dict__ except NotFoundError as e: abort(http.client.NOT_FOUND, str(e)) except Exception as e: abort(http.client.BAD_REQUEST, str(e)) return {'data': info}
def _fetch_bundle_contents_info(uuid, path=''): """ Fetch metadata of the bundle contents or a subpath within the bundle. Query parameters: - `depth`: recursively fetch subdirectory info up to this depth. Default is 0. Response format: ``` { "data": { "name": "<name of file or directory>", "link": "<string representing target if file is a symbolic link>", "type": "<file|directory|link>", "size": <size of file in bytes>, "perm": <unix permission integer>, "contents": [ { "name": ..., <each file of directory represented recursively with the same schema> }, ... ] } } ``` """ depth = query_get_type(int, 'depth', default=0) if depth < 0: abort(httplib.BAD_REQUEST, "Depth must be at least 0") check_bundles_have_read_permission(local.model, request.user, [uuid]) info = local.download_manager.get_target_info(uuid, path, depth) if info is None: abort(httplib.NOT_FOUND, 'Bundle not found') return { 'data': info }
def get_target_info(self, target, depth): check_bundles_have_read_permission(self.model, self._current_user(), [target[0]]) path = self.get_target_path(target) return path_util.get_info(path, depth)
def _fetch_bundle_contents_blob(uuid, path=''): """ API to download the contents of a bundle or a subpath within a bundle. For directories this method always returns a tarred and gzipped archive of the directory. For files, if the request has an Accept-Encoding header containing gzip, then the returned file is gzipped. """ byte_range = get_request_range() head_lines = query_get_type(int, 'head', default=0) tail_lines = query_get_type(int, 'tail', default=0) max_line_length = query_get_type(int, 'max_line_length', default=128) check_bundles_have_read_permission(local.model, request.user, [uuid]) bundle = local.model.get_bundle(uuid) target_info = local.download_manager.get_target_info(uuid, path, 0) if target_info is None: abort(httplib.NOT_FOUND, 'Not found.') # Figure out the file name. if not path and bundle.metadata.name: filename = bundle.metadata.name else: filename = target_info['name'] if target_info['type'] == 'directory': if byte_range: abort(httplib.BAD_REQUEST, 'Range not supported for directory blobs.') if head_lines: abort(httplib.BAD_REQUEST, 'Head not supported for directory blobs.') # Always tar and gzip directories. filename = filename + '.tar.gz' fileobj = local.download_manager.stream_tarred_gzipped_directory( uuid, path) elif target_info['type'] == 'file': gzipped = False if not zip_util.path_is_archive( filename) and request_accepts_gzip_encoding(): # Let's gzip to save bandwidth. The browser will transparently decode # the file. filename = filename + '.gz' gzipped = True if byte_range and (head_lines or tail_lines): abort(httplib.BAD_REQUEST, 'Head and range not supported on the same request.') elif byte_range: start, end = byte_range fileobj = local.download_manager.read_file_section( uuid, path, start, end - start + 1, gzipped) elif head_lines or tail_lines: fileobj = local.download_manager.summarize_file( uuid, path, head_lines, tail_lines, max_line_length, None, gzipped) else: fileobj = local.download_manager.stream_file(uuid, path, gzipped) else: # Symlinks. abort(httplib.FORBIDDEN, 'Cannot download files of this type.') # Set headers. mimetype, _ = mimetypes.guess_type(filename, strict=False) response.set_header('Content-Type', mimetype or 'text/plain') if zip_util.get_archive_ext( filename) == '.gz' and request_accepts_gzip_encoding(): filename = zip_util.strip_archive_ext(filename) response.set_header('Content-Encoding', 'gzip') else: response.set_header('Content-Encoding', 'identity') response.set_header('Content-Disposition', 'filename="%s"' % filename) return fileobj
def open_target(self, target): check_bundles_have_read_permission(self.model, self._current_user(), [target[0]]) path = self.get_target_path(target) path_util.check_isfile(path, 'open_target') return open(path)
def mimic(self, old_inputs, old_output, new_inputs, new_output_name, worksheet_uuid, depth, shadow, dry_run): ''' old_inputs: list of bundle uuids old_output: bundle uuid that we produced new_inputs: list of bundle uuids that are analogous to old_inputs new_output_name: name of the bundle to create to be analogous to old_output (possibly None) worksheet_uuid: add newly created bundles to this worksheet depth: how far to do a BFS up from old_output. shadow: whether to add the new inputs right after all occurrences of the old inputs in worksheets. ''' check_worksheet_has_all_permission(self.model, self._current_user(), self.model.get_worksheet(worksheet_uuid, fetch_items=False)) #print 'old_inputs: %s, new_inputs: %s, old_output: %s, new_output_name: %s' % (old_inputs, new_inputs, old_output, new_output_name) # Build the graph (get all the infos). # If old_output is given, look at ancestors of old_output until we # reached some depth. If it's not given, we first get all the # descendants first, and then get their ancestors. infos = {} # uuid -> bundle info if old_output: bundle_uuids = [old_output] else: bundle_uuids = self.model.get_self_and_descendants(old_inputs, depth=depth) all_bundle_uuids = list(bundle_uuids) # should be infos.keys() in order for _ in range(depth): new_bundle_uuids = [] for bundle_uuid in bundle_uuids: if bundle_uuid in infos: continue # Already visited info = infos[bundle_uuid] = self.get_bundle_info(bundle_uuid) for dep in info['dependencies']: parent_uuid = dep['parent_uuid'] if parent_uuid not in infos: new_bundle_uuids.append(parent_uuid) all_bundle_uuids = new_bundle_uuids + all_bundle_uuids bundle_uuids = new_bundle_uuids # Make sure we have read access to all the bundles involved here. check_bundles_have_read_permission(self.model, self._current_user(), list(infos.keys())) # Now go recursively create the bundles. old_to_new = {} # old_uuid -> new_uuid downstream = set() # old_uuid -> whether we're downstream of an input (and actually needs to be mapped onto a new uuid) created_uuids = set() # set of uuids which were newly created plan = [] # sequence of (old, new) bundle infos to make for old, new in zip(old_inputs, new_inputs): old_to_new[old] = new downstream.add(old) # Return corresponding new_bundle_uuid def recurse(old_bundle_uuid): if old_bundle_uuid in old_to_new: #print old_bundle_uuid, 'cached' return old_to_new[old_bundle_uuid] # Don't have any more information (because we probably hit the maximum depth) if old_bundle_uuid not in infos: #print old_bundle_uuid, 'no information' return old_bundle_uuid # Get information about the old bundle. info = infos[old_bundle_uuid] new_dependencies = [{ 'parent_uuid': recurse(dep['parent_uuid']), 'parent_path': dep['parent_path'], 'child_uuid': dep['child_uuid'], # This is just a placeholder to do the equality test 'child_path': dep['child_path'] } for dep in info['dependencies']] # If there are no inputs or if we're downstream of any inputs, we need to make a new bundle. lone_output = (len(old_inputs) == 0 and old_bundle_uuid == old_output) downstream_of_inputs = any(dep['parent_uuid'] in downstream for dep in info['dependencies']) if lone_output or downstream_of_inputs: # Now create a new bundle that mimics the old bundle. # Only change the name if the output name is supplied. old_bundle_name = info['metadata']['name'] new_info = copy.deepcopy(info) new_metadata = new_info['metadata'] if new_output_name: if old_bundle_uuid == old_output: new_metadata['name'] = new_output_name else: # Just make up a name heuristically new_metadata['name'] = new_output_name + '-' + info['metadata']['name'] # Remove all the automatically generated keys cls = get_bundle_subclass(new_info['bundle_type']) for spec in cls.METADATA_SPECS: if spec.generated and spec.key in new_metadata: new_metadata.pop(spec.key) # Set the targets targets = [(dep['child_path'], (dep['parent_uuid'], dep['parent_path'])) for dep in new_dependencies] if dry_run: new_bundle_uuid = None else: if new_info['bundle_type'] not in ('make', 'run'): raise UsageError('Can\'t mimic %s since it is not make or run' % old_bundle_uuid) new_bundle_uuid = self._derive_bundle(new_info['bundle_type'], \ targets, new_info['command'], new_metadata, worksheet_uuid) new_info['uuid'] = new_bundle_uuid plan.append((info, new_info)) downstream.add(old_bundle_uuid) created_uuids.add(new_bundle_uuid) else: new_bundle_uuid = old_bundle_uuid old_to_new[old_bundle_uuid] = new_bundle_uuid # Cache it return new_bundle_uuid if old_output: recurse(old_output) else: # Don't have a particular output we're targetting, so just create # new versions of all the uuids. for uuid in all_bundle_uuids: recurse(uuid) # Add to worksheet if not dry_run: if shadow: # Add each new bundle in the "shadow" of the old_bundle (right after it). for old_bundle_uuid, new_bundle_uuid in old_to_new.items(): if new_bundle_uuid in created_uuids: # Only add novel bundles self.model.add_shadow_worksheet_items(old_bundle_uuid, new_bundle_uuid) else: def newline(): self.model.add_worksheet_item(worksheet_uuid, worksheet_util.markup_item('')) # A prelude of a bundle on a worksheet is the set of items that occur right before it (markup, directives, etc.) # Let W be the first worksheet containing the old_inputs[0]. # Add all items on that worksheet that appear in old_to_new along with their preludes. # For items not on this worksheet, add them at the end (instead of making them floating). if old_output: anchor_uuid = old_output elif len(old_inputs) > 0: anchor_uuid = old_inputs[0] host_worksheet_uuids = self.model.get_host_worksheet_uuids([anchor_uuid])[anchor_uuid] new_bundle_uuids_added = set() skipped = True # Whether there were items that we didn't include in the prelude (in which case we want to put '') if len(host_worksheet_uuids) > 0: # Choose a single worksheet. if worksheet_uuid in host_worksheet_uuids: # If current worksheet is one of them, favor that one. host_worksheet_uuid = worksheet_uuid else: # Choose an arbitrary one (in the future, have a better way of canonicalizing). host_worksheet_uuid = host_worksheet_uuids[0] # Fetch the worksheet worksheet_info = self.get_worksheet_info(host_worksheet_uuid, fetch_items=True) prelude_items = [] # The prelude that we're building up for item in worksheet_info['items']: (bundle_info, subworkheet_info, value_obj, item_type) = item just_added = False if item_type == worksheet_util.TYPE_BUNDLE: old_bundle_uuid = bundle_info['uuid'] if old_bundle_uuid in old_to_new: # Flush the prelude gathered so far. new_bundle_uuid = old_to_new[old_bundle_uuid] if new_bundle_uuid in created_uuids: # Only add novel bundles # Stand in for things skipped (this is important so directives have proper extent). if skipped: newline() # Add prelude and items for item2 in prelude_items: self.add_worksheet_item(worksheet_uuid, worksheet_util.convert_item_to_db(item2)) self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(new_bundle_uuid)) new_bundle_uuids_added.add(new_bundle_uuid) just_added = True if (item_type == worksheet_util.TYPE_MARKUP and value_obj != '') or item_type == worksheet_util.TYPE_DIRECTIVE: prelude_items.append(item) # Include in prelude skipped = False else: prelude_items = [] # Reset skipped = not just_added # Add the bundles that haven't been added yet for info, new_info in plan: new_bundle_uuid = new_info['uuid'] if new_bundle_uuid not in new_bundle_uuids_added: if skipped: newline() skipped = False self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(new_bundle_uuid)) return plan
def download_target(self, target, follow_symlinks): check_bundles_have_read_permission(self.model, self._current_user(), [target[0]]) # Don't need to download anything because it's already local. # Note that we can't really enforce follow_symlinks, but this is okay, # because we will follow them when we copy it from the target path. return (self.get_target_path(target), None)
def open_target_handle(self, target): check_bundles_have_read_permission(self.model, self._current_user(), [target[0]]) path = self.get_target_path(target) return open(path) if path and os.path.exists(path) else None
def check_target_has_read_permission(target): check_bundles_have_read_permission(local.model, request.user, [target[0]])
def _fetch_bundle_contents_blob(uuid, path=''): """ API to download the contents of a bundle or a subpath within a bundle. For directories, this method always returns a tarred and gzipped archive of the directory. For files, if the request has an Accept-Encoding header containing gzip, then the returned file is gzipped. Otherwise, the file is returned as-is. HTTP Request headers: - `Range: bytes=<start>-<end>`: fetch bytes from the range `[<start>, <end>)`. - `Accept-Encoding: <encoding>`: indicate that the client can accept encoding `<encoding>`. Currently only `gzip` encoding is supported. Query parameters: - `head`: number of lines to fetch from the beginning of the file. Default is 0, meaning to fetch the entire file. - `tail`: number of lines to fetch from the end of the file. Default is 0, meaning to fetch the entire file. - `max_line_length`: maximum number of characters to fetch from each line, if either `head` or `tail` is specified. Default is 128. HTTP Response headers (for single-file targets): - `Content-Disposition: inline; filename=<bundle name or target filename>` - `Content-Type: <guess of mimetype based on file extension>` - `Content-Encoding: [gzip|identity]` - `Target-Type: file` HTTP Response headers (for directories): - `Content-Disposition: attachment; filename=<bundle or directory name>.tar.gz` - `Content-Type: application/gzip` - `Content-Encoding: identity` - `Target-Type: directory` """ byte_range = get_request_range() head_lines = query_get_type(int, 'head', default=0) tail_lines = query_get_type(int, 'tail', default=0) truncation_text = query_get_type(str, 'truncation_text', default='') max_line_length = query_get_type(int, 'max_line_length', default=128) check_bundles_have_read_permission(local.model, request.user, [uuid]) target = BundleTarget(uuid, path) try: target_info = local.download_manager.get_target_info(target, 0) if target_info['resolved_target'] != target: check_bundles_have_read_permission( local.model, request.user, [target_info['resolved_target'].bundle_uuid]) target = target_info['resolved_target'] except NotFoundError as e: abort(http.client.NOT_FOUND, str(e)) except Exception as e: abort(http.client.BAD_REQUEST, str(e)) # Figure out the file name. bundle_name = local.model.get_bundle(target.bundle_uuid).metadata.name if not path and bundle_name: filename = bundle_name else: filename = target_info['name'] if target_info['type'] == 'directory': if byte_range: abort(http.client.BAD_REQUEST, 'Range not supported for directory blobs.') if head_lines or tail_lines: abort(http.client.BAD_REQUEST, 'Head and tail not supported for directory blobs.') # Always tar and gzip directories gzipped_stream = False # but don't set the encoding to 'gzip' mimetype = 'application/gzip' filename += '.tar.gz' fileobj = local.download_manager.stream_tarred_gzipped_directory( target) elif target_info['type'] == 'file': # Let's gzip to save bandwidth. # For simplicity, we do this even if the file is already a packed # archive (which should be relatively rare). # The browser will transparently decode the file. gzipped_stream = request_accepts_gzip_encoding() # Since guess_type() will interpret '.tar.gz' as an 'application/x-tar' file # with 'gzip' encoding, which would usually go into the Content-Encoding # header. But if the bundle contents is actually a packed archive, we don't # want the client to automatically decompress the file, so we don't want to # set the Content-Encoding header. Instead, if guess_type() detects an # archive, we just set mimetype to indicate an arbitrary binary file. mimetype, encoding = mimetypes.guess_type(filename, strict=False) if encoding is not None: mimetype = 'application/octet-stream' if byte_range and (head_lines or tail_lines): abort(http.client.BAD_REQUEST, 'Head and range not supported on the same request.') elif byte_range: start, end = byte_range fileobj = local.download_manager.read_file_section( target, start, end - start + 1, gzipped_stream) elif head_lines or tail_lines: fileobj = local.download_manager.summarize_file( target, head_lines, tail_lines, max_line_length, truncation_text, gzipped_stream) else: fileobj = local.download_manager.stream_file( target, gzipped_stream) else: # Symlinks. abort(http.client.FORBIDDEN, 'Cannot download files of this type (%s).' % target_info['type']) # Set headers. response.set_header('Content-Type', mimetype or 'text/plain') response.set_header('Content-Encoding', 'gzip' if gzipped_stream else 'identity') if target_info['type'] == 'file': response.set_header('Content-Disposition', 'inline; filename="%s"' % filename) else: response.set_header('Content-Disposition', 'attachment; filename="%s"' % filename) response.set_header('Target-Type', target_info['type']) return fileobj
def cat_target(self, target, out): check_bundles_have_read_permission(self.model, self._current_user(), [target[0]]) path = self.get_target_path(target) path_util.cat(path, out)
def _stage_bundles(self): """ Stages bundles by: 1) Failing any bundles that have any missing or failed dependencies. 2) Staging any bundles that have all ready dependencies. """ bundles = self._model.batch_get_bundles(state=State.CREATED) parent_uuids = set( dep.parent_uuid for bundle in bundles for dep in bundle.dependencies) parents = self._model.batch_get_bundles(uuid=parent_uuids) all_parent_states = {parent.uuid: parent.state for parent in parents} all_parent_uuids = set(all_parent_states) bundles_to_fail = [] bundles_to_stage = [] for bundle in bundles: parent_uuids = set(dep.parent_uuid for dep in bundle.dependencies) try: check_bundles_have_read_permission(self._model, self._model.get_user(bundle.owner_id), parent_uuids) except PermissionError as e: bundles_to_fail.append( (bundle, str(e)) ) continue missing_uuids = parent_uuids - all_parent_uuids if missing_uuids: bundles_to_fail.append( (bundle, 'Missing parent bundles: %s' % ', '.join(missing_uuids))) continue parent_states = {uuid: all_parent_states[uuid] for uuid in parent_uuids} acceptable_states = [State.READY] if bundle.metadata.allow_failed_dependencies: acceptable_states.append(State.FAILED) acceptable_states.append(State.KILLED) else: failed_uuids = [ uuid for uuid, state in parent_states.iteritems() if state == State.FAILED] if failed_uuids: bundles_to_fail.append( (bundle, 'Parent bundles failed: %s' % ', '.join(failed_uuids))) continue if all(state in acceptable_states for state in parent_states.itervalues()): bundles_to_stage.append(bundle) for bundle, failure_message in bundles_to_fail: logger.info('Failing bundle %s: %s', bundle.uuid, failure_message) self._model.update_bundle( bundle, {'state': State.FAILED, 'metadata': {'failure_message': failure_message}}) for bundle in bundles_to_stage: logger.info('Staging %s', bundle.uuid) self._model.update_bundle( bundle, {'state': State.STAGED})
def _stage_bundles(self): """ Stages bundles by: 1) Failing any bundles that have any missing or failed dependencies. 2) Staging any bundles that have all ready dependencies. """ bundles = self._model.batch_get_bundles(state=State.CREATED) parent_uuids = set(dep.parent_uuid for bundle in bundles for dep in bundle.dependencies) parents = self._model.batch_get_bundles(uuid=parent_uuids) all_parent_states = {parent.uuid: parent.state for parent in parents} all_parent_uuids = set(all_parent_states) bundles_to_fail = [] bundles_to_stage = [] for bundle in bundles: parent_uuids = set(dep.parent_uuid for dep in bundle.dependencies) try: check_bundles_have_read_permission( self._model, self._model.get_user(bundle.owner_id), parent_uuids ) except PermissionError as e: bundles_to_fail.append((bundle, str(e))) continue missing_uuids = parent_uuids - all_parent_uuids if missing_uuids: bundles_to_fail.append( (bundle, 'Missing parent bundles: %s' % ', '.join(missing_uuids)) ) continue parent_states = {uuid: all_parent_states[uuid] for uuid in parent_uuids} acceptable_states = [State.READY] if bundle.metadata.allow_failed_dependencies: acceptable_states.append(State.FAILED) acceptable_states.append(State.KILLED) else: failed_uuids = [ uuid for uuid, state in parent_states.iteritems() if state == State.FAILED ] killed_uuids = [ uuid for uuid, state in parent_states.iteritems() if state == State.KILLED ] failure_message = '' if failed_uuids: failure_message += ' Parent bundles failed: %s' % ', '.join(failed_uuids) if killed_uuids: failure_message += ' Parent bundles were killed: %s' % ', '.join(killed_uuids) if failure_message: failure_message += ' (Please use the --allow-failed-dependencies flag to depend on results fo failed or killed bundles)' bundles_to_fail.append((bundle, failure_message)) continue if all(state in acceptable_states for state in parent_states.itervalues()): bundles_to_stage.append(bundle) for bundle, failure_message in bundles_to_fail: logger.info('Failing bundle %s: %s', bundle.uuid, failure_message) self._model.update_bundle( bundle, {'state': State.FAILED, 'metadata': {'failure_message': failure_message}} ) for bundle in bundles_to_stage: logger.info('Staging %s', bundle.uuid) self._model.update_bundle(bundle, {'state': State.STAGED})