def get_socket(conf, default_port=61005): """Bind socket to bind ip:port in conf :param conf: Configuration dict to read settings from :param default_port: port to use if not specified in conf :returns : a socket object as returned from socket.listen or ssl.wrap_socket if conf specifies cert_file """ bind_addr = (conf.get('bind_ip', '0.0.0.0'), int(conf.get('bind_port', default_port))) address_family = [ addr[0] for addr in socket.getaddrinfo( bind_addr[0], bind_addr[1], socket.AF_UNSPEC, socket.SOCK_STREAM) if addr[0] in (socket.AF_INET, socket.AF_INET6) ][0] sock = None bind_timeout = int(conf.get('bind_timeout', 30)) so_rcvbuf = conf.get('so_rcvbuf', None) retry_until = time.time() + bind_timeout warn_ssl = False while not sock and time.time() < retry_until: try: sock = listen(bind_addr, backlog=int(conf.get('backlog', 4096)), family=address_family) if 'cert_file' in conf: warn_ssl = True sock = ssl.wrap_socket(sock, certfile=conf['cert_file'], keyfile=conf['key_file']) except socket.error as err: if err.args[0] != errno.EADDRINUSE: raise sleep(0.1) if not sock: raise Exception( _('Could not bind to %s:%s ' 'after trying for %s seconds') % (bind_addr[0], bind_addr[1], bind_timeout)) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # in my experience, sockets can hang around forever without keepalive sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) if so_rcvbuf: sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, int(so_rcvbuf)) sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) if hasattr(socket, 'TCP_KEEPIDLE'): sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 600) if warn_ssl: ssl_warning_message = _('WARNING: SSL should only be enabled for ' 'testing purposes. Use external SSL ' 'termination for a production deployment.') get_logger(conf).warning(ssl_warning_message) print(ssl_warning_message) return sock
def __init__(self, app, conf, logger=None): self.app = app self.log_hdrs = config_true_value( conf.get('access_log_headers', conf.get('log_headers', 'no'))) log_hdrs_only = list_from_csv(conf.get('access_log_headers_only', '')) self.log_hdrs_only = [x.title() for x in log_hdrs_only] # The leading access_* check is in case someone assumes that # log_statsd_valid_http_methods behaves like the other log_statsd_* # settings. self.valid_methods = conf.get( 'access_log_statsd_valid_http_methods', conf.get('log_statsd_valid_http_methods', 'GET,HEAD,POST,PUT,DELETE,COPY,OPTIONS')) self.valid_methods = [ m.strip().upper() for m in self.valid_methods.split(',') if m.strip() ] access_log_conf = {} for key in ('log_facility', 'log_name', 'log_level', 'log_udp_host', 'log_udp_port', 'log_statsd_host', 'log_statsd_port', 'log_statsd_default_sample_rate', 'log_statsd_sample_rate_factor', 'log_statsd_metric_prefix'): value = conf.get('access_' + key, conf.get(key, None)) if value: access_log_conf[key] = value self.access_logger = logger or get_logger( access_log_conf, log_route='proxy-access', section='proxy-access', fmt="%(asctime)s %(message)s") self.access_logger.set_statsd_prefix('proxy-server') self.reveal_sensitive_prefix = int( conf.get('reveal_sensitive_prefix', MAX_HEADER_SIZE))
def _initrp(conf_path, app_section, *args, **kwargs): try: conf = appconfig(conf_path, name=app_section) except Exception as e: raise ConfigFileError("Error trying to load config from %s: %s" % (conf_path, e)) validate_configuration() # pre-configure logger log_name = conf.get('log_name', app_section) if 'logger' in kwargs: logger = kwargs.pop('logger') else: logger = get_logger(conf, log_name, log_to_console=kwargs.pop('verbose', False), log_route='wsgi') # disable fallocate if desired if config_true_value(conf.get('disable_fallocate', 'no')): disable_fallocate() conf.update({'llport': kwargs.pop('llport', 61014)}) monkey_patch_mimetools() return (conf, logger, log_name)
def __init__(self, app, conf): self.app = app self.filters = [ IPfilter(conf), RequestFilter(conf), BucketFilter(conf) ] self.logger = utils.get_logger(conf, log_route='ceilometer')
def run_daemon(klass, conf_file, section_name='', once=False, **kwargs): """ Loads settings from conf, then instantiates daemon "klass" and runs the daemon with the specified once kwarg. The section_name will be derived from the daemon "klass" if not provided (e.g. ObjectReplicator => object-replicator). :param klass: Class to instantiate, subclass of common.daemon.Daemon :param conf_file: Path to configuration file :param section_name: Section name from conf file to load config from :param once: Passed to daemon run method """ # very often the config section_name is based on the class name # the None singleton will be passed through to readconf as is if section_name is '': section_name = sub(r'([a-z])([A-Z])', r'\1-\2', klass.__name__).lower() conf = utils.readconf(conf_file, section_name, log_name=kwargs.get('log_name')) # once on command line (i.e. daemonize=false) will over-ride config once = once or not utils.config_true_value(conf.get('daemonize', 'true')) # pre-configure logger if 'logger' in kwargs: logger = kwargs.pop('logger') else: logger = utils.get_logger(conf, conf.get('log_name', section_name), log_to_console=kwargs.pop('verbose', False), log_route=section_name) # disable fallocate if desired if utils.config_true_value(conf.get('disable_fallocate', 'no')): utils.disable_fallocate() # set utils.FALLOCATE_RESERVE if desired reserve = int(conf.get('fallocate_reserve', 0)) if reserve > 0: utils.FALLOCATE_RESERVE = reserve # By default, disable eventlet printing stacktraces eventlet_debug = utils.config_true_value(conf.get('eventlet_debug', 'no')) eventlet.debug.hub_exceptions(eventlet_debug) # Ensure TZ environment variable exists to avoid stat('/etc/localtime') on # some platforms. This locks in reported times to the timezone in which # the server first starts running in locations that periodically change # timezones. os.environ['TZ'] = time.strftime("%z", time.gmtime()) try: klass(conf).run(once=once, **kwargs) except KeyboardInterrupt: logger.info('User quit') logger.info('Exited')
def setUp(self): self.gl_map_object = GLmap() self.conf = {'CLIENT_TIMEOUT': 10} self.communicator = "None_for_now" self.service_id = "HN0101_61014_container-server" self.logger = get_logger({}, log_route='recovery') self.__trans_path = "/export/HN0101_61014_transaction_journal" self.__cont_path = "/export/HN0101_61014_container_journal" osd.containerService.container_recovery.final_recovery_status_list = list( ) osd.containerService.container_recovery.list_of_tuple = queue( maxsize=0) osd.containerService.container_recovery.Thread = Monitor()
def __init__(self, app, conf): self.app = app self.conf = conf self.forbidden_chars = self.conf.get('forbidden_chars', FORBIDDEN_CHARS) self.maximum_length = int(self.conf.get('maximum_length', MAX_LENGTH)) self.forbidden_regexp = self.conf.get('forbidden_regexp', FORBIDDEN_REGEXP) if self.forbidden_regexp: self.forbidden_regexp_compiled = re.compile(self.forbidden_regexp) else: self.forbidden_regexp_compiled = None self.logger = get_logger(self.conf, log_route='name_check')
def __init__(self, conf, global_map_object, service_id, communicator_object, Request_handler, logger=None, source_service_obj=None): self.logger = logger or get_logger(conf, log_route='container-recovery') libraryUtils.OSDLoggerImpl("container-library").initialize_logger() self.logger.info("running RecoveryMgr ") self.communicator_object = communicator_object self.Request_handler = Request_handler self.conf = conf self.__trans_port = int(conf['child_port']) self.global_map_object = global_map_object self.source_service_obj = source_service_obj self.service_id = service_id self.id = self.service_id.split('_')[0] self.local_leader_id = self.service_id.split('_')[1] self.__cont_journal_path = ''.join('.' + self.id +"_" +self.local_leader_id \ + "_container_journal") self.__trans_journal_path = ''.join('.' + self.id + "_" +self.local_leader_id \ + "_transaction_journal") self.__cont_path = EXPORT + self.__cont_journal_path self.__trans_path = EXPORT + self.__trans_journal_path self.logger.debug("Initiaization entries: %s %s %s %s %s %s " % \ (self.global_map_object, \ self.service_id, self.__cont_journal_path, \ self.__trans_journal_path, self.__cont_path, self.__trans_path)) #get object gl version from global leader self._object_gl_version = self.__get_object_gl_version() self.logger.debug("Object version received: %s" % self._object_gl_version) #Map from gl self.map_gl = self.global_map_object.dest_comp_map() self.logger.debug("Map from Gl Received %s" % self.map_gl) #start heart beat self.logger.debug("Made heart beat thread") #self.heart_beat_thread.spawn(self.start_heart_beat, self.Request_handler) self.heart_beat_gthread = eventlet.greenthread.spawn( self.start_heart_beat, self.Request_handler) self.logger.debug("Started heart beat thread func")
def __init__(self, app, conf): self.app = app self.conf = conf self.logger = get_logger(conf, log_route='tempauth') self.log_headers = config_true_value(conf.get('log_headers', 'f')) self.reseller_prefix = conf.get('reseller_prefix', 'AUTH').strip() if self.reseller_prefix and self.reseller_prefix[-1] != '_': self.reseller_prefix += '_' self.logger.set_statsd_prefix( 'tempauth.%s' % (self.reseller_prefix if self.reseller_prefix else 'NONE', )) self.auth_prefix = conf.get('auth_prefix', '/auth/') if not self.auth_prefix or not self.auth_prefix.strip('/'): self.logger.warning('Rewriting invalid auth prefix "%s" to ' '"/auth/" (Non-empty auth prefix path ' 'is required)' % self.auth_prefix) self.auth_prefix = '/auth/' if self.auth_prefix[0] != '/': self.auth_prefix = '/' + self.auth_prefix if self.auth_prefix[-1] != '/': self.auth_prefix += '/' self.token_life = int(conf.get('token_life', 86400)) self.allow_overrides = config_true_value( conf.get('allow_overrides', 't')) self.storage_url_scheme = conf.get('storage_url_scheme', 'default') self.users = {} for conf_key in conf: if conf_key.startswith('user_') or conf_key.startswith('user64_'): account, username = conf_key.split('_', 1)[1].split('_') if conf_key.startswith('user64_'): # Because trailing equal signs would screw up config file # parsing, we auto-pad with '=' chars. account += '=' * (len(account) % 4) account = base64.b64decode(account) username += '=' * (len(username) % 4) username = base64.b64decode(username) values = conf[conf_key].split() if not values: raise ValueError('%s has no key set' % conf_key) key = values.pop(0) if values and ('://' in values[-1] or '$HOST' in values[-1]): url = values.pop() else: url = '$HOST/v1/%s%s' % (self.reseller_prefix, account) self.users[account + ':' + username] = { 'key': key, 'url': url, 'groups': values }
def __init__(self, app, conf): self.app = app self.conf = conf self.logger = osd_utils.get_logger(conf, log_route='keystoneauth') self.reseller_prefix = conf.get('reseller_prefix', 'AUTH_').strip() if self.reseller_prefix and self.reseller_prefix[-1] != '_': self.reseller_prefix += '_' self.operator_roles = conf.get('operator_roles', 'admin, swiftoperator').lower() self.reseller_admin_role = conf.get('reseller_admin_role', 'ResellerAdmin').lower() config_is_admin = conf.get('is_admin', "false").lower() self.is_admin = osd_utils.config_true_value(config_is_admin) config_overrides = conf.get('allow_overrides', 't').lower() self.allow_overrides = osd_utils.config_true_value(config_overrides)
def __init__(self, app, conf, logger=None): self.app = app self.logger = logger or get_logger(conf) self.unit_time_period = int( conf.get('bandwidth_controlled_unit_time_period', 60)) self.bw_control = str(conf.get('bandwidth_control', 'Disable')) self.next_unit_time = time.time() + self.unit_time_period self.account_limit = 30000 if not int(conf.get('bandwidth_controlled_account_limit', 0)) \ else int(conf.get('bandwidth_controlled_account_limit', 30000)) self.def_max_bandwidth_read_limit = int( conf.get('default_max_bandwidth_read_limit', 0)) self.def_max_bandwidth_write_limit = int( conf.get('default_max_bandwidth_write_limit', 0)) #creating the cache with self.account_limit+1024 maximum entries of accounts self.cache = BWInfoCache(self.account_limit+1024, self.next_unit_time, \ self.unit_time_period, self.logger)
def __init__(self, conf): self.logger = utils.get_logger(conf, log_route='ceilometer') parser = ConfigParser() parser.read( conf.get( 'ip_filter_config_path', '/opt/HYDRAstor/objectStorage/configFiles/osd_ip_filter.conf')) self.ips_list = [ ip.strip() for ip in parser.get('ip_filter', 'internal_ip_list').split('\n') ] self.subnet_list = [ subnet.strip() for subnet in parser.get( 'ip_filter', 'internal_skip_list').split('\n') ] self.ip_headers = [ header.strip() for header in parser.get('ip_filter', 'ip_headers').split('\n') ]
def __init__(self, app, conf, min_segment_size=DEFAULT_MIN_SEGMENT_SIZE, max_manifest_segments=DEFAULT_MAX_MANIFEST_SEGMENTS, max_manifest_size=DEFAULT_MAX_MANIFEST_SIZE): self.conf = conf self.app = app # self.logger = get_logger(conf, log_route='slo') self.max_manifest_segments = max_manifest_segments self.max_manifest_size = max_manifest_size self.min_segment_size = min_segment_size self.max_get_time = int(self.conf.get('max_get_time', 86400)) self.rate_limit_after_segment = int( self.conf.get('rate_limit_after_segment', '10')) self.rate_limit_segments_per_sec = int( self.conf.get('rate_limit_segments_per_sec', '0')) # self.bulk_deleter = Bulk(app, {}, logger=self.logger) self.objs_to_delete = [] self.logger = get_logger(self.conf, log_route='slo')
def __init__(self, app, conf): self.app = app self.publish_incoming_bytes = conf.get("publish_incoming_bytes", True) self.publish_outgoing_bytes = conf.get("publish_outgoing_bytes", True) self.publish_on_error = conf.get("publish_on_error", False) self.enable_filters = conf.get("enable_filters", True) self.error_on_status = [ status.strip() for status in conf.get("error_on_status", '').split('\n') ] self.logger = utils.get_logger(conf, log_route='ceilometer') self.metadata_headers = [ h.strip().replace('-', '_').lower() for h in conf.get("metadata_headers", "").split(",") if h.strip() ] service.prepare_service([]) self.pipeline_manager = pipeline.setup_pipeline() self.reseller_prefix = conf.get('reseller_prefix', 'AUTH_') if self.reseller_prefix and self.reseller_prefix[-1] != '_': self.reseller_prefix += '_'
def __init__(self, conf, dictionary_new, communicator_obj, \ service_component_map, service_id, gl_communication_obj=None, \ logger=None, object_gl_version=None, gl_map=None): self.logger = logger or get_logger({}, log_route='recovery-container') self.pile_size = len(dictionary_new.keys()) self.logger.info("Started Recovery after restructuring...") self.dictionary_new = dictionary_new #destination map of gl self.Map_from_gl = gl_map self.logger.info("map from gl in recovery %s" % self.Map_from_gl) self.conf = conf self.container_node_list = \ [ {'ip':"%s" % key.split(':')[0], 'port':"%s" % \ key.split(':')[1]}for key \ in dictionary_new.keys()] self.logger.debug("Container node list:%s ,Dictionary passed : %s" % \ (self.container_node_list, self.dictionary_new)) self.service_component_map = service_component_map self.communicator_obj = communicator_obj self.service_id = service_id self.gl_communication_obj = gl_communication_obj self.__object_gl_version = object_gl_version
def __init__(self, app, conf, logger=None): self.app = app if logger: self.logger = logger else: self.logger = get_logger(conf, log_route='ratelimit') self.account_ratelimit = float(conf.get('account_ratelimit', 0)) self.max_sleep_time_seconds = \ float(conf.get('max_sleep_time_seconds', 60)) self.log_sleep_time_seconds = \ float(conf.get('log_sleep_time_seconds', 0)) self.clock_accuracy = int(conf.get('clock_accuracy', 1000)) self.rate_buffer_seconds = int(conf.get('rate_buffer_seconds', 5)) self.ratelimit_whitelist = \ [acc.strip() for acc in conf.get('account_whitelist', '').split(',') if acc.strip()] self.ratelimit_blacklist = \ [acc.strip() for acc in conf.get('account_blacklist', '').split(',') if acc.strip()] self.memcache_client = None self.container_ratelimits = interpret_conf_limits( conf, 'container_ratelimit_') self.container_listing_ratelimits = interpret_conf_limits( conf, 'container_listing_ratelimit_')
def __init__(self, app, conf): self.app = app self.logger = get_logger(conf, log_route='gatekeeper') self.inbound_condition = make_exclusion_test(inbound_exclusions) self.outbound_condition = make_exclusion_test(outbound_exclusions)
def __init__(self, conf): self.conf = conf self.logger = utils.get_logger(conf, log_route='daemon')
def __init__(self, conf, memcache=None, logger=None, account_ring=None, container_ring=None, object_ring=None): if conf is None: conf = {} if logger is None: self.logger = get_logger(conf, log_route='proxy-server') else: self.logger = logger libraryUtils.OSDLoggerImpl("proxy-monitoring").initialize_logger() create_recovery_file('proxy-server') self.ongoing_operation_list = [] self.stop_service_flag = False osd_dir = conf.get('osd_dir', '/export/.osd_meta_config') static_config_file = conf.get('static_config_file', \ '/opt/HYDRAstor/objectStorage/configFiles/static_proxy-server.conf') #swift_dir = conf.get('swift_dir', '/etc/swift') static_conf = self.readconf(static_config_file) self.logger.debug("Static config parameters:%s" % (static_conf)) self.node_timeout = int(conf.get('node_timeout', 10)) self.recoverable_node_timeout = int( conf.get('recoverable_node_timeout', self.node_timeout)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.client_timeout = int(conf.get('client_timeout', 60)) self.put_queue_depth = int(static_conf.get('put_queue_depth', 5)) self.retry_count = int(static_conf.get('retry_count', 3)) self.request_retry_time_service_unavailable = int(\ static_conf.get('request_retry_time_service_unavailable', 100)) self.request_retry_time_component_transfer = int(\ static_conf.get('request_retry_time_component_transfer', 50)) self.object_chunk_size = int(conf.get('object_chunk_size', 65536)) self.client_chunk_size = int(conf.get('client_chunk_size', 65536)) self.trans_id_suffix = conf.get('trans_id_suffix', '') self.post_quorum_timeout = float(conf.get('post_quorum_timeout', 0.5)) #self.error_suppression_interval = \ # int(conf.get('error_suppression_interval', 60)) #self.error_suppression_limit = \ # int(conf.get('error_suppression_limit', 10)) self.recheck_container_existence = \ int(conf.get('recheck_container_existence', 60)) self.recheck_account_existence = \ int(conf.get('recheck_account_existence', 60)) self.allow_account_management = \ config_true_value(static_conf.get('allow_account_management', 'no')) self.object_post_as_copy = \ config_true_value(conf.get('object_post_as_copy', 'false')) self.object_ring = object_ring or ObjectRing(osd_dir, self.logger, \ self.node_timeout) self.container_ring = container_ring or \ ContainerRing(osd_dir, self.logger, self.node_timeout) self.account_ring = account_ring or AccountRing(osd_dir, self.logger, \ self.node_timeout) self.memcache = memcache mimetypes.init(mimetypes.knownfiles + [os.path.join(osd_dir, 'mime.types')]) self.account_autocreate = \ config_true_value(static_conf.get('account_autocreate', 'yes')) #self.expiring_objects_account = \ # (conf.get('auto_create_account_prefix') or '.') + \ # (conf.get('expiring_objects_account_name') or 'expiring_objects') #self.expiring_objects_container_divisor = \ # int(conf.get('expiring_objects_container_divisor') or 86400) self.max_containers_per_account = \ int(conf.get('max_containers_per_account') or 10000000) self.max_containers_whitelist = [ a.strip() for a in conf.get('max_containers_whitelist', '').split(',') if a.strip() ] self.deny_host_headers = [ host.strip() for host in conf.get('deny_host_headers', '').split(',') if host.strip() ] #self.rate_limit_after_segment = \ # int(conf.get('rate_limit_after_segment', 10)) #self.rate_limit_segments_per_sec = \ # int(conf.get('rate_limit_segments_per_sec', 1)) #self.log_handoffs = config_true_value(conf.get('log_handoffs', 'true')) #self.cors_allow_origin = [ # a.strip() # for a in conf.get('cors_allow_origin', '').split(',') # if a.strip()] #self.strict_cors_mode = config_true_value( # conf.get('strict_cors_mode', 't')) self.node_timings = {} self.timing_expiry = int(conf.get('timing_expiry', 300)) #self.sorting_method = conf.get('sorting_method', 'shuffle').lower() #self.max_large_object_get_time = float( # conf.get('max_large_object_get_time', '86400')) #value = conf.get('request_node_count', '2 * replicas').lower().split() #if len(value) == 1: # value = int(value[0]) # self.request_node_count = lambda replicas: value #elif len(value) == 3 and value[1] == '*' and value[2] == 'replicas': # value = int(value[0]) # self.request_node_count = lambda replicas: value * replicas #else: # raise ValueError( # 'Invalid request_node_count value: %r' % ''.join(value)) #try: # self._read_affinity = read_affinity = conf.get('read_affinity', '') # self.read_affinity_sort_key = affinity_key_function(read_affinity) #except ValueError as err: # # make the message a little more useful # raise ValueError("Invalid read_affinity value: %r (%s)" % # (read_affinity, err.message)) #try: # write_affinity = conf.get('write_affinity', '') # self.write_affinity_is_local_fn \ # = affinity_locality_predicate(write_affinity) #except ValueError as err: # # make the message a little more useful # raise ValueError("Invalid write_affinity value: %r (%s)" % # (write_affinity, err.message)) #value = conf.get('write_affinity_node_count', # '2 * replicas').lower().split() #if len(value) == 1: # value = int(value[0]) # self.write_affinity_node_count = lambda replicas: value #elif len(value) == 3 and value[1] == '*' and value[2] == 'replicas': # value = int(value[0]) # self.write_affinity_node_count = lambda replicas: value * replicas #else: # raise ValueError( # 'Invalid write_affinity_node_count value: %r' % ''.join(value)) # swift_owner_headers are stripped by the account and container # controllers; we should extend header stripping to object controller # when a privileged object header is implemented. swift_owner_headers = conf.get( 'swift_owner_headers', 'x-container-read, x-container-write, ' 'x-container-sync-key, x-container-sync-to, ' 'x-account-meta-temp-url-key, x-account-meta-temp-url-key-2, ' 'x-account-access-control') self.swift_owner_headers = [ name.strip().title() for name in swift_owner_headers.split(',') if name.strip() ] # Initialization was successful, so now apply the client chunk size # parameter as the default read / write buffer size for the network # sockets. # # NOTE WELL: This is a class setting, so until we get set this on a # per-connection basis, this affects reading and writing on ALL # sockets, those between the proxy servers and external clients, and # those between the proxy servers and the other internal servers. # # ** Because it affects the client as well, currently, we use the # client chunk size as the govenor and not the object chunk size. socket._fileobject.default_bufsize = self.client_chunk_size self.expose_info = config_true_value(conf.get('expose_info', 'yes')) self.disallowed_sections = list_from_csv( conf.get('disallowed_sections')) self.admin_key = conf.get('admin_key', None) register_osd_info( max_file_size=constraints.MAX_FILE_SIZE, max_meta_name_length=constraints.MAX_META_NAME_LENGTH, max_meta_value_length=constraints.MAX_META_VALUE_LENGTH, max_meta_count=constraints.MAX_META_COUNT, account_listing_limit=constraints.ACCOUNT_LISTING_LIMIT, container_listing_limit=constraints.CONTAINER_LISTING_LIMIT, max_account_name_length=constraints.MAX_ACCOUNT_NAME_LENGTH, max_container_name_length=constraints.MAX_CONTAINER_NAME_LENGTH, max_object_name_length=constraints.MAX_OBJECT_NAME_LENGTH, non_allowed_headers=constraints.NON_ALLOWED_HEADERS) self.proxy_port = int(static_conf.get('bind_port', 61005)) self.__ll_port = int(conf.get('llport', 61014)) self.max_bulk_delete_entries = int(conf.get(\ 'max_bulk_delete_entries', 1000)) #unblock new requests which was blocked due to proxy service stop self.__request_unblock() hostname = socket.gethostname() self.__server_id = hostname + "_" + str( self.__ll_port) + "_proxy-server" # Start sending health to local leader self.logger.info("Loading health monitoring library") self.health_instance = healthMonitoring(self.__get_node_ip(hostname), \ self.proxy_port, self.__ll_port, self.__server_id, True) self.logger.info("Loaded health monitoring library") remove_recovery_file('proxy-server')
def __init__(self, config, logger, communicator, recovery_stat, service_id): self.__recovery_status = recovery_stat self.__serv_id = service_id self.__mount_check = True self.__root = "/export" self.logger = logger or get_logger(config, log_route='object-recovery') self.__recovery_handler_obj = \ RecoveryHandler(self.__serv_id, self.logger) self._communicator_obj = communicator gl_info_obj = communicator.get_gl_info() retry_count = 0 while retry_count < 3: retry_count += 1 self._con_obj = communicator.connector(IoType.EVENTIO, gl_info_obj) if self._con_obj: self.logger.debug("CONN object success") break self.logger.debug("conn obj:%s, retry_count:%s " % (self._con_obj, retry_count)) self._network_fail_status_list = [ Resp.BROKEN_PIPE, Resp.CONNECTION_RESET_BY_PEER, Resp.INVALID_SOCKET ] self._success_status_list = [Resp.SUCCESS] #flags self._final_recovery_status_list = [] self._strm_ack_flag = False self._final_status = True self._finish_recovery_flag = False # return variables code ret = self._communicator_obj.recv_pro_start_monitoring( \ self.__serv_id, self._con_obj) if ret.status.get_status_code() in self._success_status_list: self.logger.info("Strm status successfull") else: gl_info_obj = self._communicator_obj.get_gl_info() retry_count = 0 while retry_count < 3: retry_count += 1 self._con_obj = self._communicator_obj.connector( IoType.EVENTIO, gl_info_obj) if self._con_obj: self.logger.debug("CONN object success") ret = self._communicator_obj.recv_pro_start_monitoring( self.__serv_id, self._con_obj) if ret.status.get_status_code( ) in self._success_status_list: break else: self.logger.error("Start Monitoring Failure") sys.exit(130) self.logger.debug("conn obj:%s, retry_count:%s " % (self._con_obj, retry_count)) # set flag, main thread will read this flag #TODO Start sending health to local leader # return variables code self.logger.debug("Initialized Object Recovery")
description="Recovery process option parser") parser.add_argument("server_id", help="Service id from Global leader.") args = parser.parse_args() __service_id = args.server_id #Change Process name of Script. script_prefix = "RECOVERY_" proc_name = script_prefix + __service_id procname.setprocname(proc_name) recovery_status = RecoveryStatus() conf = { 'log_requests': 'false', 'log_failure': 'true', 'log_level': 'debug', 'log_facility': 'LOG_LOCAL6', 'log_name': 'object_recovery' } #create logger obj logger_obj = get_logger(conf, log_route='object-recovery') logger_obj.info("Service received is : %s of type %s " % \ (__service_id, type(__service_id))) #create communication object communicator_obj = Req(logger_obj) recover = RecoveryObject(conf, logger_obj, communicator_obj, recovery_status, __service_id) recover.recover_data() logger_obj.info("Object Recovery finished")
def __init__(self, conf): self.logger = utils.get_logger(conf, log_route='ceilometer')
REST_SERVERS = [s for s in ALL_SERVERS if s not in MAIN_SERVERS] GRACEFUL_SHUTDOWN_SERVERS = MAIN_SERVERS + ['auth-server'] START_ONCE_SERVERS = REST_SERVERS # These are servers that match a type (account-*, container-*, object-*) but # don't use that type-server.conf file and instead use their own. STANDALONE_SERVERS = REST_SERVERS KILL_WAIT = 60 # seconds to wait for servers to die (by default) WARNING_WAIT = 3 # seconds to wait after message that may just be a warning MAX_DESCRIPTORS = 32768 MAX_MEMORY = (1024 * 1024 * 1024) * 2 # 2 GB MAX_PROCS = 8192 # workers * disks * threads_per_disk, can get high script_logger = get_logger({}, section='script') def setup_env(): """Try to increase resource limits of the OS. Move PYTHON_EGG_CACHE to /tmp """ try: resource.setrlimit(resource.RLIMIT_NOFILE, (MAX_DESCRIPTORS, MAX_DESCRIPTORS)) except ValueError: print _("WARNING: Unable to modify file descriptor limit. " "Running as non-root?") script_logger.error("WARNING: Unable to modify file descriptor limit. Running as non-root?") try: resource.setrlimit(resource.RLIMIT_DATA,
def __init__(self, app, conf): self.app = app self.logger = get_logger(conf, log_route='catch-errors') self.trans_id_suffix = conf.get('trans_id_suffix', '')
args = parser.parse_args() acc_service_id = args.server_id #Change Process name of Script. script_prefix = "RECOVERY_" proc_name = script_prefix + acc_service_id procname.setprocname(proc_name) #create config object CONFIG_PATH = "/opt/HYDRAstor/objectStorage/configFiles/account-server.conf" config = ConfigParser.ConfigParser() config.read(CONFIG_PATH) conf_obj = dict(config.items('DEFAULT')) #create logger obj logger_obj = get_logger(conf_obj, log_route='recovery') logger_obj.info("Parsed aguments:%s, configfile: %s, proc %s" % \ (acc_service_id, conf_obj, proc_name)) #create communication object communicator_obj = Req(logger_obj) #create recovery obj and start recovery rec_data_obj = RecoveryData() retry = 0 rec_obj = Recovery(conf_obj, logger_obj, acc_service_id, \ communicator_obj, rec_data_obj) while retry < MAX_RETRY_COUNT: if rec_obj.connect_gl(): logger_obj.info("Starting Recovery Process...") # create green thread for recovery instance
CONFIG_PATH = \ "/opt/HYDRAstor/objectStorage/configFiles/container-recovery.conf" config = ConfigParser.ConfigParser() config.read(CONFIG_PATH) config_dict = dict(config.items('DEFAULT')) #Global variables for intermediated response handling list_of_tuple = queue(maxsize=1000) finish_recovery_flag = False final_status = True final_recovery_status_list = list() list_to_gl = [] clean_journal_flag = True #logger initialize logger_obj = get_logger(config_dict, log_route='recovery') logger_obj.info("Parsed aguments:%s, configfile: %s, proc %s" % \ (service_id_, config_dict, proc_name)) #Get Global map and retry 2 times. logger_obj.info("Start global map request") Request_handler_ = Req(logger_obj, timeout=0) gl_info_obj = Request_handler_.get_gl_info() retry_count = 0 while retry_count < 3: retry_count += 1 communicator_object_ = Request_handler_.connector( IoType.EVENTIO, gl_info_obj) if communicator_object_: logger_obj.debug("CONN object success")