def run_daemon(klass, conf_file, section_name='', once=False, **kwargs): """ Loads settings from conf, then instantiates daemon "klass" and runs the daemon with the specified once kwarg. The section_name will be derived from the daemon "klass" if not provided (e.g. ObjectReplicator => object-replicator). :param klass: Class to instantiate, subclass of common.daemon.Daemon :param conf_file: Path to configuration file :param section_name: Section name from conf file to load config from :param once: Passed to daemon run method """ # very often the config section_name is based on the class name # the None singleton will be passed through to readconf as is if section_name is '': section_name = sub(r'([a-z])([A-Z])', r'\1-\2', klass.__name__).lower() conf = utils.readconf(conf_file, section_name, log_name=kwargs.get('log_name')) # once on command line (i.e. daemonize=false) will over-ride config once = once or not utils.config_true_value(conf.get('daemonize', 'true')) # pre-configure logger if 'logger' in kwargs: logger = kwargs.pop('logger') else: logger = utils.get_logger(conf, conf.get('log_name', section_name), log_to_console=kwargs.pop('verbose', False), log_route=section_name) # disable fallocate if desired if utils.config_true_value(conf.get('disable_fallocate', 'no')): utils.disable_fallocate() # set utils.FALLOCATE_RESERVE if desired reserve = int(conf.get('fallocate_reserve', 0)) if reserve > 0: utils.FALLOCATE_RESERVE = reserve # By default, disable eventlet printing stacktraces eventlet_debug = utils.config_true_value(conf.get('eventlet_debug', 'no')) eventlet.debug.hub_exceptions(eventlet_debug) # Ensure TZ environment variable exists to avoid stat('/etc/localtime') on # some platforms. This locks in reported times to the timezone in which # the server first starts running in locations that periodically change # timezones. os.environ['TZ'] = time.strftime("%z", time.gmtime()) try: klass(conf).run(once=once, **kwargs) except KeyboardInterrupt: logger.info('User quit') logger.info('Exited')
def __init__(self, app, conf): self.app = app self.conf = conf self.logger = get_logger(conf, log_route='tempauth') self.log_headers = config_true_value(conf.get('log_headers', 'f')) self.reseller_prefix = conf.get('reseller_prefix', 'AUTH').strip() if self.reseller_prefix and self.reseller_prefix[-1] != '_': self.reseller_prefix += '_' self.logger.set_statsd_prefix( 'tempauth.%s' % (self.reseller_prefix if self.reseller_prefix else 'NONE', )) self.auth_prefix = conf.get('auth_prefix', '/auth/') if not self.auth_prefix or not self.auth_prefix.strip('/'): self.logger.warning('Rewriting invalid auth prefix "%s" to ' '"/auth/" (Non-empty auth prefix path ' 'is required)' % self.auth_prefix) self.auth_prefix = '/auth/' if self.auth_prefix[0] != '/': self.auth_prefix = '/' + self.auth_prefix if self.auth_prefix[-1] != '/': self.auth_prefix += '/' self.token_life = int(conf.get('token_life', 86400)) self.allow_overrides = config_true_value( conf.get('allow_overrides', 't')) self.storage_url_scheme = conf.get('storage_url_scheme', 'default') self.users = {} for conf_key in conf: if conf_key.startswith('user_') or conf_key.startswith('user64_'): account, username = conf_key.split('_', 1)[1].split('_') if conf_key.startswith('user64_'): # Because trailing equal signs would screw up config file # parsing, we auto-pad with '=' chars. account += '=' * (len(account) % 4) account = base64.b64decode(account) username += '=' * (len(username) % 4) username = base64.b64decode(username) values = conf[conf_key].split() if not values: raise ValueError('%s has no key set' % conf_key) key = values.pop(0) if values and ('://' in values[-1] or '$HOST' in values[-1]): url = values.pop() else: url = '$HOST/v1/%s%s' % (self.reseller_prefix, account) self.users[account + ':' + username] = { 'key': key, 'url': url, 'groups': values }
def __init__(self, app, conf): self.app = app self.conf = conf self.logger = osd_utils.get_logger(conf, log_route='keystoneauth') self.reseller_prefix = conf.get('reseller_prefix', 'AUTH_').strip() if self.reseller_prefix and self.reseller_prefix[-1] != '_': self.reseller_prefix += '_' self.operator_roles = conf.get('operator_roles', 'admin, swiftoperator').lower() self.reseller_admin_role = conf.get('reseller_admin_role', 'ResellerAdmin').lower() config_is_admin = conf.get('is_admin', "false").lower() self.is_admin = osd_utils.config_true_value(config_is_admin) config_overrides = conf.get('allow_overrides', 't').lower() self.allow_overrides = osd_utils.config_true_value(config_overrides)
def _initrp(conf_path, app_section, *args, **kwargs): try: conf = appconfig(conf_path, name=app_section) except Exception as e: raise ConfigFileError("Error trying to load config from %s: %s" % (conf_path, e)) validate_configuration() # pre-configure logger log_name = conf.get('log_name', app_section) if 'logger' in kwargs: logger = kwargs.pop('logger') else: logger = get_logger(conf, log_name, log_to_console=kwargs.pop('verbose', False), log_route='wsgi') # disable fallocate if desired if config_true_value(conf.get('disable_fallocate', 'no')): disable_fallocate() conf.update({'llport': kwargs.pop('llport', 61014)}) monkey_patch_mimetools() return (conf, logger, log_name)
def __init__(self, app, conf, logger=None): self.app = app self.log_hdrs = config_true_value( conf.get('access_log_headers', conf.get('log_headers', 'no'))) log_hdrs_only = list_from_csv(conf.get('access_log_headers_only', '')) self.log_hdrs_only = [x.title() for x in log_hdrs_only] # The leading access_* check is in case someone assumes that # log_statsd_valid_http_methods behaves like the other log_statsd_* # settings. self.valid_methods = conf.get( 'access_log_statsd_valid_http_methods', conf.get('log_statsd_valid_http_methods', 'GET,HEAD,POST,PUT,DELETE,COPY,OPTIONS')) self.valid_methods = [ m.strip().upper() for m in self.valid_methods.split(',') if m.strip() ] access_log_conf = {} for key in ('log_facility', 'log_name', 'log_level', 'log_udp_host', 'log_udp_port', 'log_statsd_host', 'log_statsd_port', 'log_statsd_default_sample_rate', 'log_statsd_sample_rate_factor', 'log_statsd_metric_prefix'): value = conf.get('access_' + key, conf.get(key, None)) if value: access_log_conf[key] = value self.access_logger = logger or get_logger( access_log_conf, log_route='proxy-access', section='proxy-access', fmt="%(asctime)s %(message)s") self.access_logger.set_statsd_prefix('proxy-server') self.reveal_sensitive_prefix = int( conf.get('reveal_sensitive_prefix', MAX_HEADER_SIZE))
def get_or_head_response(self, req, resp_headers, resp_iter): resp_body = ''.join(resp_iter) try: segments = json.loads(resp_body) except ValueError: segments = [] etag = md5() content_length = 0 for seg_dict in segments: etag.update(seg_dict['hash']) if config_true_value(seg_dict.get('sub_slo')): override_bytes_from_content_type(seg_dict, logger=self.slo.logger) content_length += int(seg_dict['bytes']) response_headers = [(h, v) for h, v in resp_headers if h.lower() not in ('etag', 'content-length')] response_headers.append(('Content-Length', str(content_length))) response_headers.append(('Etag', '"%s"' % etag.hexdigest())) if req.method == 'HEAD': return self._manifest_head_response(req, response_headers) else: return self._manifest_get_response(req, content_length, response_headers, segments)
def slo_hook(source_req, source_resp, sink_req): x_slo = source_resp.headers.get('X-Static-Large-Object') if (config_true_value(x_slo) and source_req.params.get('multipart-manifest') != 'get'): source_resp = SloGetContext(self).get_or_head_response( source_req, source_resp.headers.items(), source_resp.app_iter) return inner_hook(source_req, source_resp, sink_req)
def handle_slo_get_or_head(self, req, start_response): """ Takes a request and a start_response callable and does the normal WSGI thing with them. Returns an iterator suitable for sending up the WSGI chain. :param req: swob.Request object; is a GET or HEAD request aimed at what may be a static large object manifest (or may not). :param start_response: WSGI start_response callable """ resp_iter = self._app_call(req.environ) # make sure this response is for a static large object manifest for header, value in self._response_headers: if (header.lower() == 'x-static-large-object' and config_true_value(value)): break else: # Not a static large object manifest. Just pass it through. start_response(self._response_status, self._response_headers, self._response_exc_info) return resp_iter # Handle pass-through request for the manifest itself if req.params.get('multipart-manifest') == 'get': new_headers = [] for header, value in self._response_headers: if header.lower() == 'content-type': new_headers.append( ('Content-Type', 'application/json; charset=utf-8')) else: new_headers.append((header, value)) self._response_headers = new_headers start_response(self._response_status, self._response_headers, self._response_exc_info) return resp_iter if self._need_to_refetch_manifest(req): req.environ['swift.non_client_disconnect'] = True close_if_possible(resp_iter) del req.environ['swift.non_client_disconnect'] get_req = make_subrequest( req.environ, method='GET', headers={'x-auth-token': req.headers.get('x-auth-token')}, agent=('%(orig)s ' + 'SLO MultipartGET'), swift_source='SLO') resp_iter = self._app_call(get_req.environ) # Any Content-Range from a manifest is almost certainly wrong for the # full large object. resp_headers = [(h, v) for h, v in self._response_headers if not h.lower() == 'content-range'] response = self.get_or_head_response(req, resp_headers, resp_iter) return response(req.environ, start_response)
def __init__(self, conf, logger): libraryUtils.OSDLoggerImpl("object-library").initialize_logger() self.logger = logger self.filesystems = conf.get('filesystems', '/export') self.disk_chunk_size = int(conf.get('disk_chunk_size', 65536)) self.keep_cache_size = int(conf.get('keep_cache_size', 5242880)) self.mount_check = config_true_value(conf.get('mount_check', 'true')) self.reclaim_age = int(conf.get('reclaim_age', ONE_WEEK)) threads_for_read = int(conf.get('threads_for_read', '8')) threads_for_write = int(conf.get('threads_for_write', '4')) self.threadpools_write = defaultdict( lambda: ThreadPool(nthreads=threads_for_write)) self.object_server_id = get_service_id('object') self.object_lib = ObjectLibrary(threads_for_write, threads_for_read)
def run_server(conf, logger, sock, global_conf=None): # Ensure TZ environment variable exists to avoid stat('/etc/localtime') on # some platforms. This locks in reported times to the timezone in which # the server first starts running in locations that periodically change # timezones. os.environ['TZ'] = time.strftime("%z", time.gmtime()) wsgi.HttpProtocol.default_request_version = "HTTP/1.0" # Turn off logging requests by the underlying WSGI software. wsgi.HttpProtocol.log_request = lambda *a: None # Redirect logging other messages by the underlying WSGI software. wsgi.HttpProtocol.log_message = \ lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a) wsgi.WRITE_TIMEOUT = int(conf.get('client_timeout') or 60) eventlet.hubs.use_hub(get_hub()) eventlet.patcher.monkey_patch(all=False, socket=True) eventlet_debug = config_true_value(conf.get('eventlet_debug', 'no')) eventlet.debug.hub_exceptions(eventlet_debug) #TODO: deveshg: this will be removed when submitting the code to QA for testing #eventlet.debug.hub_blocking_detection(int(conf.get('eventlet_detection', 1)), conf.get('eventlet_detection_timeout', 30)) # utils.LogAdapter stashes name in server; fallback on unadapted loggers if not global_conf: if hasattr(logger, 'server'): log_name = logger.server else: log_name = logger.name global_conf = {'log_name': log_name} app = loadapp(conf['__file__'], global_conf=global_conf) max_clients = int(conf.get('max_clients', '1024')) pool = RestrictedGreenPool(size=max_clients) try: pool.spawn_n(getattr(app, 'add_green_thread', lambda: True)) func = getattr(app, 'add_process', lambda: True) func() wsgi.server(sock, app, NullLogger(), custom_pool=pool, capitalize_response_headers=False) except socket.error as err: if err[0] != errno.EINVAL: raise pool.waitall()
def get_slo_segments(self, obj_name, req): """ Performs a swob.Request and returns the SLO manifest's segments. :raises HTTPServerError: on unable to load obj_name or on unable to load the SLO manifest data. :raises HTTPBadRequest: on not an SLO manifest :raises HTTPNotFound: on SLO manifest not found :returns: SLO manifest's segments """ vrs, account, _junk = req.split_path(2, 3, True) new_env = req.environ.copy() new_env['REQUEST_METHOD'] = 'GET' del (new_env['wsgi.input']) new_env['QUERY_STRING'] = 'multipart-manifest=get' new_env['CONTENT_LENGTH'] = 0 new_env['HTTP_USER_AGENT'] = \ '%s MultipartDELETE' % new_env.get('HTTP_USER_AGENT') new_env['swift.source'] = 'SLO' new_env['PATH_INFO'] = ( '/%s/%s/%s' % (vrs, account, obj_name.lstrip('/'))).encode('utf-8') resp = Request.blank('', new_env).get_response(self.app) if resp.is_success: if config_true_value(resp.headers.get('X-Static-Large-Object')): try: return json.loads(resp.body) except ValueError: raise HTTPServerError('Unable to load SLO manifest') else: raise HTTPBadRequest('Not an SLO manifest') elif resp.status_int == HTTP_NOT_FOUND: raise HTTPNotFound('SLO manifest not found') elif resp.status_int == HTTP_UNAUTHORIZED: raise HTTPUnauthorized('401 Unauthorized') else: raise HTTPServerError('Unable to load SLO manifest or segment.')
def __init__(self, conf, memcache=None, logger=None, account_ring=None, container_ring=None, object_ring=None): if conf is None: conf = {} if logger is None: self.logger = get_logger(conf, log_route='proxy-server') else: self.logger = logger libraryUtils.OSDLoggerImpl("proxy-monitoring").initialize_logger() create_recovery_file('proxy-server') self.ongoing_operation_list = [] self.stop_service_flag = False osd_dir = conf.get('osd_dir', '/export/.osd_meta_config') static_config_file = conf.get('static_config_file', \ '/opt/HYDRAstor/objectStorage/configFiles/static_proxy-server.conf') #swift_dir = conf.get('swift_dir', '/etc/swift') static_conf = self.readconf(static_config_file) self.logger.debug("Static config parameters:%s" % (static_conf)) self.node_timeout = int(conf.get('node_timeout', 10)) self.recoverable_node_timeout = int( conf.get('recoverable_node_timeout', self.node_timeout)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.client_timeout = int(conf.get('client_timeout', 60)) self.put_queue_depth = int(static_conf.get('put_queue_depth', 5)) self.retry_count = int(static_conf.get('retry_count', 3)) self.request_retry_time_service_unavailable = int(\ static_conf.get('request_retry_time_service_unavailable', 100)) self.request_retry_time_component_transfer = int(\ static_conf.get('request_retry_time_component_transfer', 50)) self.object_chunk_size = int(conf.get('object_chunk_size', 65536)) self.client_chunk_size = int(conf.get('client_chunk_size', 65536)) self.trans_id_suffix = conf.get('trans_id_suffix', '') self.post_quorum_timeout = float(conf.get('post_quorum_timeout', 0.5)) #self.error_suppression_interval = \ # int(conf.get('error_suppression_interval', 60)) #self.error_suppression_limit = \ # int(conf.get('error_suppression_limit', 10)) self.recheck_container_existence = \ int(conf.get('recheck_container_existence', 60)) self.recheck_account_existence = \ int(conf.get('recheck_account_existence', 60)) self.allow_account_management = \ config_true_value(static_conf.get('allow_account_management', 'no')) self.object_post_as_copy = \ config_true_value(conf.get('object_post_as_copy', 'false')) self.object_ring = object_ring or ObjectRing(osd_dir, self.logger, \ self.node_timeout) self.container_ring = container_ring or \ ContainerRing(osd_dir, self.logger, self.node_timeout) self.account_ring = account_ring or AccountRing(osd_dir, self.logger, \ self.node_timeout) self.memcache = memcache mimetypes.init(mimetypes.knownfiles + [os.path.join(osd_dir, 'mime.types')]) self.account_autocreate = \ config_true_value(static_conf.get('account_autocreate', 'yes')) #self.expiring_objects_account = \ # (conf.get('auto_create_account_prefix') or '.') + \ # (conf.get('expiring_objects_account_name') or 'expiring_objects') #self.expiring_objects_container_divisor = \ # int(conf.get('expiring_objects_container_divisor') or 86400) self.max_containers_per_account = \ int(conf.get('max_containers_per_account') or 10000000) self.max_containers_whitelist = [ a.strip() for a in conf.get('max_containers_whitelist', '').split(',') if a.strip() ] self.deny_host_headers = [ host.strip() for host in conf.get('deny_host_headers', '').split(',') if host.strip() ] #self.rate_limit_after_segment = \ # int(conf.get('rate_limit_after_segment', 10)) #self.rate_limit_segments_per_sec = \ # int(conf.get('rate_limit_segments_per_sec', 1)) #self.log_handoffs = config_true_value(conf.get('log_handoffs', 'true')) #self.cors_allow_origin = [ # a.strip() # for a in conf.get('cors_allow_origin', '').split(',') # if a.strip()] #self.strict_cors_mode = config_true_value( # conf.get('strict_cors_mode', 't')) self.node_timings = {} self.timing_expiry = int(conf.get('timing_expiry', 300)) #self.sorting_method = conf.get('sorting_method', 'shuffle').lower() #self.max_large_object_get_time = float( # conf.get('max_large_object_get_time', '86400')) #value = conf.get('request_node_count', '2 * replicas').lower().split() #if len(value) == 1: # value = int(value[0]) # self.request_node_count = lambda replicas: value #elif len(value) == 3 and value[1] == '*' and value[2] == 'replicas': # value = int(value[0]) # self.request_node_count = lambda replicas: value * replicas #else: # raise ValueError( # 'Invalid request_node_count value: %r' % ''.join(value)) #try: # self._read_affinity = read_affinity = conf.get('read_affinity', '') # self.read_affinity_sort_key = affinity_key_function(read_affinity) #except ValueError as err: # # make the message a little more useful # raise ValueError("Invalid read_affinity value: %r (%s)" % # (read_affinity, err.message)) #try: # write_affinity = conf.get('write_affinity', '') # self.write_affinity_is_local_fn \ # = affinity_locality_predicate(write_affinity) #except ValueError as err: # # make the message a little more useful # raise ValueError("Invalid write_affinity value: %r (%s)" % # (write_affinity, err.message)) #value = conf.get('write_affinity_node_count', # '2 * replicas').lower().split() #if len(value) == 1: # value = int(value[0]) # self.write_affinity_node_count = lambda replicas: value #elif len(value) == 3 and value[1] == '*' and value[2] == 'replicas': # value = int(value[0]) # self.write_affinity_node_count = lambda replicas: value * replicas #else: # raise ValueError( # 'Invalid write_affinity_node_count value: %r' % ''.join(value)) # swift_owner_headers are stripped by the account and container # controllers; we should extend header stripping to object controller # when a privileged object header is implemented. swift_owner_headers = conf.get( 'swift_owner_headers', 'x-container-read, x-container-write, ' 'x-container-sync-key, x-container-sync-to, ' 'x-account-meta-temp-url-key, x-account-meta-temp-url-key-2, ' 'x-account-access-control') self.swift_owner_headers = [ name.strip().title() for name in swift_owner_headers.split(',') if name.strip() ] # Initialization was successful, so now apply the client chunk size # parameter as the default read / write buffer size for the network # sockets. # # NOTE WELL: This is a class setting, so until we get set this on a # per-connection basis, this affects reading and writing on ALL # sockets, those between the proxy servers and external clients, and # those between the proxy servers and the other internal servers. # # ** Because it affects the client as well, currently, we use the # client chunk size as the govenor and not the object chunk size. socket._fileobject.default_bufsize = self.client_chunk_size self.expose_info = config_true_value(conf.get('expose_info', 'yes')) self.disallowed_sections = list_from_csv( conf.get('disallowed_sections')) self.admin_key = conf.get('admin_key', None) register_osd_info( max_file_size=constraints.MAX_FILE_SIZE, max_meta_name_length=constraints.MAX_META_NAME_LENGTH, max_meta_value_length=constraints.MAX_META_VALUE_LENGTH, max_meta_count=constraints.MAX_META_COUNT, account_listing_limit=constraints.ACCOUNT_LISTING_LIMIT, container_listing_limit=constraints.CONTAINER_LISTING_LIMIT, max_account_name_length=constraints.MAX_ACCOUNT_NAME_LENGTH, max_container_name_length=constraints.MAX_CONTAINER_NAME_LENGTH, max_object_name_length=constraints.MAX_OBJECT_NAME_LENGTH, non_allowed_headers=constraints.NON_ALLOWED_HEADERS) self.proxy_port = int(static_conf.get('bind_port', 61005)) self.__ll_port = int(conf.get('llport', 61014)) self.max_bulk_delete_entries = int(conf.get(\ 'max_bulk_delete_entries', 1000)) #unblock new requests which was blocked due to proxy service stop self.__request_unblock() hostname = socket.gethostname() self.__server_id = hostname + "_" + str( self.__ll_port) + "_proxy-server" # Start sending health to local leader self.logger.info("Loading health monitoring library") self.health_instance = healthMonitoring(self.__get_node_ip(hostname), \ self.proxy_port, self.__ll_port, self.__server_id, True) self.logger.info("Loaded health monitoring library") remove_recovery_file('proxy-server')
original_syslog_handler = logging.handlers.SysLogHandler def fake_syslog_handler(): for attr in dir(original_syslog_handler): if attr.startswith('LOG'): setattr(FakeLogger, attr, copy.copy(getattr(logging.handlers.SysLogHandler, attr))) FakeLogger.priority_map = \ copy.deepcopy(logging.handlers.SysLogHandler.priority_map) logging.handlers.SysLogHandler = FakeLogger if config_true_value(get_config('unit_test').get('fake_syslog', 'False')): fake_syslog_handler() class MockTrue(object): """ Instances of MockTrue evaluate like True Any attr accessed on an instance of MockTrue will return a MockTrue instance. Any method called on an instance of MockTrue will return a MockTrue instance. >>> thing = MockTrue() >>> thing True >>> thing == True # True == True True
def handle_multipart_put(self, req, start_response): """ Will handle the PUT of a SLO manifest. Heads every object in manifest to check if is valid and if so will save a manifest generated from the user input. Uses WSGIContext to call self and start_response and returns a WSGI iterator. :params req: a swob.Request with an obj in path :raises: HttpException on errors """ try: vrs, account, container, obj = req.split_path(1, 4, True) self.logger.info("Received manifest file: %s for upload" % (req.path)) except ValueError: return self.app(req.environ, start_response) if req.content_length > self.max_manifest_size: raise HTTPRequestEntityTooLarge("Manifest File > %d bytes" % self.max_manifest_size) if req.headers.get('X-Copy-From'): raise HTTPMethodNotAllowed( 'Multipart Manifest PUTs cannot be COPY requests') if req.content_length is None and \ req.headers.get('transfer-encoding', '').lower() != 'chunked': raise HTTPLengthRequired(request=req) parsed_data = parse_input(req.body_file.read(self.max_manifest_size)) problem_segments = [] if len(parsed_data) > self.max_manifest_segments: raise HTTPRequestEntityTooLarge( 'Number of segments must be <= %d' % self.max_manifest_segments) total_size = 0 out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS) if not out_content_type: out_content_type = 'text/plain' data_for_storage = [] slo_etag = md5() for index, seg_dict in enumerate(parsed_data): obj_name = seg_dict['path'] if isinstance(obj_name, unicode): obj_name = obj_name.encode('utf-8') obj_path = '/'.join(['', vrs, account, obj_name.lstrip('/')]) try: seg_size = int(seg_dict['size_bytes']) except (ValueError, TypeError): raise HTTPBadRequest('Invalid Manifest File') if seg_size < self.min_segment_size and \ (index == 0 or index < len(parsed_data) - 1): raise HTTPBadRequest( 'Each segment, except the last, must be at least ' '%d bytes.' % self.min_segment_size) new_env = req.environ.copy() new_env['PATH_INFO'] = obj_path new_env['REQUEST_METHOD'] = 'HEAD' new_env['swift.source'] = 'SLO' del (new_env['wsgi.input']) del (new_env['QUERY_STRING']) new_env['CONTENT_LENGTH'] = 0 new_env['HTTP_USER_AGENT'] = \ '%s MultipartPUT' % req.environ.get('HTTP_USER_AGENT') head_seg_resp = \ Request.blank(obj_path, new_env).get_response(self) if head_seg_resp.is_success: total_size += seg_size if seg_size != head_seg_resp.content_length: problem_segments.append([quote(obj_name), 'Size Mismatch']) if seg_dict['etag'] == head_seg_resp.etag: slo_etag.update(seg_dict['etag']) else: problem_segments.append([quote(obj_name), 'Etag Mismatch']) if head_seg_resp.last_modified: last_modified = head_seg_resp.last_modified else: # shouldn't happen last_modified = datetime.now() last_modified_formatted = \ last_modified.strftime('%Y-%m-%dT%H:%M:%S.%f') seg_data = { 'name': '/' + seg_dict['path'].lstrip('/'), 'bytes': seg_size, 'hash': seg_dict['etag'], 'content_type': head_seg_resp.content_type, 'last_modified': last_modified_formatted } if config_true_value( head_seg_resp.headers.get('X-Static-Large-Object')): seg_data['sub_slo'] = True data_for_storage.append(seg_data) else: problem_segments.append( [quote(obj_name), head_seg_resp.status]) if problem_segments: resp_body = get_response_body(out_content_type, {}, problem_segments) raise HTTPBadRequest(resp_body, content_type=out_content_type) env = req.environ if not env.get('CONTENT_TYPE'): guessed_type, _junk = mimetypes.guess_type(req.path_info) env['CONTENT_TYPE'] = guessed_type or 'application/octet-stream' env['swift.content_type_overridden'] = True env['CONTENT_TYPE'] += ";swift_bytes=%d" % total_size env['HTTP_X_STATIC_LARGE_OBJECT'] = 'True' json_data = json.dumps(data_for_storage) env['CONTENT_LENGTH'] = str(len(json_data)) env['wsgi.input'] = StringIO(json_data) slo_put_context = SloPutContext(self, slo_etag) return slo_put_context.handle_slo_put(req, start_response)
def _segment_listing_iterator(self, req, version, account, segments, recursion_depth=1): for seg_dict in segments: if config_true_value(seg_dict.get('sub_slo')): override_bytes_from_content_type(seg_dict, logger=self.slo.logger) # We handle the range stuff here so that we can be smart about # skipping unused submanifests. For example, if our first segment is a # submanifest referencing 50 MiB total, but self.first_byte falls in # the 51st MiB, then we can avoid fetching the first submanifest. # # If we were to make SegmentedIterable handle all the range # calculations, we would be unable to make this optimization. total_length = sum(int(seg['bytes']) for seg in segments) if self.first_byte is None: self.first_byte = 0 if self.last_byte is None: self.last_byte = total_length - 1 for seg_dict in segments: seg_length = int(seg_dict['bytes']) if self.first_byte >= seg_length: # don't need any bytes from this segment self.first_byte = max(self.first_byte - seg_length, -1) self.last_byte = max(self.last_byte - seg_length, -1) continue if self.last_byte < 0: # no bytes are needed from this or any future segment break if config_true_value(seg_dict.get('sub_slo')): # do this check here so that we can avoid fetching this last # manifest before raising the exception if recursion_depth >= self.max_slo_recursion_depth: raise ListingIterError("Max recursion depth exceeded") sub_path = get_valid_utf8_str(seg_dict['name']) sub_cont, sub_obj = split_path(sub_path, 2, 2, True) sub_segments = self._fetch_sub_slo_segments( req, version, account, sub_cont, sub_obj) for sub_seg_dict, sb, eb in self._segment_listing_iterator( req, version, account, sub_segments, recursion_depth=recursion_depth + 1): yield sub_seg_dict, sb, eb else: if isinstance(seg_dict['name'], unicode): seg_dict['name'] = seg_dict['name'].encode("utf-8") seg_length = int(seg_dict['bytes']) yield (seg_dict, (None if self.first_byte <= 0 else self.first_byte), (None if self.last_byte >= seg_length - 1 else self.last_byte)) self.first_byte = max(self.first_byte - seg_length, -1) self.last_byte = max(self.last_byte - seg_length, -1)